diff options
Diffstat (limited to 'tool')
224 files changed, 10645 insertions, 4394 deletions
diff --git a/tool/annocheck/Dockerfile b/tool/annocheck/Dockerfile index 138adc48de..d1fb1839c9 100644 --- a/tool/annocheck/Dockerfile +++ b/tool/annocheck/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/fedora:latest +FROM ghcr.io/ruby/fedora:latest RUN dnf -y install annobin-annocheck WORKDIR /work diff --git a/tool/annocheck/Dockerfile-copy b/tool/annocheck/Dockerfile-copy index 0a79f3a50a..d437f27387 100644 --- a/tool/annocheck/Dockerfile-copy +++ b/tool/annocheck/Dockerfile-copy @@ -1,4 +1,4 @@ -FROM docker.io/fedora:latest +FROM ghcr.io/ruby/fedora:latest ARG IN_DIR RUN dnf -y install annobin-annocheck diff --git a/tool/auto-style.rb b/tool/auto-style.rb new file mode 100755 index 0000000000..3b93c8c317 --- /dev/null +++ b/tool/auto-style.rb @@ -0,0 +1,284 @@ +#!/usr/bin/env ruby +# Usage: +# auto-style.rb oldrev newrev [pushref] + +require 'shellwords' +require 'tmpdir' +ENV['LC_ALL'] = 'C' + +class Git + attr_reader :depth + + def initialize(oldrev, newrev, branch = nil) + @oldrev = oldrev + @newrev = !newrev || newrev.empty? ? 'HEAD' : newrev + @branch = branch + + return unless oldrev + + # GitHub may not fetch github.event.pull_request.base.sha at checkout + git('log', '--format=%H', '-1', @oldrev, out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=1', 'origin', @oldrev) + git('log', '--format=%H', '-1', "#@newrev~99", out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=100', 'origin', @newrev) + + with_clean_env do + @revs = {} + IO.popen(['git', 'log', '--format=%H %s', "#{@oldrev}..#{@newrev}"]) do |f| + f.each do |line| + line.chomp! + rev, subj = line.split(' ', 2) + @revs[rev] = subj + end + end + @depth = @revs.size + end + end + + # ["foo/bar.c", "baz.h", ...] + def updated_paths + with_clean_env do + IO.popen(['git', 'diff', '--name-only', @oldrev, @newrev], &:readlines).each(&:chomp!) + end + end + + # [0, 1, 4, ...] + def updated_lines(file) # NOTE: This doesn't work well on pull requests, so not used anymore + lines = [] + revs = @revs.map {|rev, subj| rev unless subj.start_with?("Revert ")}.compact + revs_pattern = /\A(?:#{revs.join('|')}) / + with_clean_env { IO.popen(['git', 'blame', '-l', '--', file], &:readlines) }.each_with_index do |line, index| + if revs_pattern =~ line + lines << index + end + end + lines + end + + def commit(log, *files) + git('add', *files) + git('commit', '-m', log) + end + + def push + git('push', 'origin', @branch) if @branch + end + + def diff + git('--no-pager', 'diff') + end + + private + + def git(*args, **opts) + cmd = ['git', *args] + puts "+ #{cmd.shelljoin}" + ret = with_clean_env { system(*cmd, **opts) } + unless ret or opts[:err] + abort "Failed to run: #{cmd}" + end + ret + end + + def with_clean_env + git_dir = ENV.delete('GIT_DIR') # this overcomes '-C' or pwd + yield + ensure + ENV['GIT_DIR'] = git_dir if git_dir + end +end + +DEFAULT_GEM_LIBS = %w[ + bundler + cmath + csv + e2mmap + fileutils + forwardable + ipaddr + irb + logger + matrix + mutex_m + ostruct + prime + rdoc + rexml + rss + scanf + shell + sync + thwait + tracer + webrick +] + +DEFAULT_GEM_EXTS = %w[ + bigdecimal + date + dbm + digest + etc + fcntl + fiddle + gdbm + io/console + io/nonblock + json + openssl + psych + racc + sdbm + stringio + strscan + zlib +] + +IGNORED_FILES = [ + # default gems whose master is GitHub + %r{\Abin/(?!erb)\w+\z}, + *(DEFAULT_GEM_LIBS + DEFAULT_GEM_EXTS).flat_map { |lib| + [ + %r{\Alib/#{lib}/}, + %r{\Alib/#{lib}\.gemspec\z}, + %r{\Alib/#{lib}\.rb\z}, + %r{\Atest/#{lib}/}, + ] + }, + *DEFAULT_GEM_EXTS.flat_map { |ext| + [ + %r{\Aext/#{ext}/}, + %r{\Atest/#{ext}/}, + ] + }, + + # vendoring (ccan) + %r{\Accan/}, + + # vendoring (io/) + %r{\Aext/io/}, + + # vendoring (nkf) + %r{\Aext/nkf/nkf-utf8/}, + + # vendoring (onigmo) + %r{\Aenc/}, + %r{\Ainclude/ruby/onigmo\.h\z}, + %r{\Areg.+\.(c|h)\z}, + + # explicit or implicit `c-file-style: "linux"` + %r{\Aaddr2line\.c\z}, + %r{\Amissing/}, + %r{\Astrftime\.c\z}, + %r{\Avsnprintf\.c\z}, + + # to respect the original statements of licenses + %r{\ALEGAL\z}, + + # trailing spaces could be intentional in TRICK code + %r{\Asample/trick[^/]*/}, +] + +DIFFERENT_STYLE_FILES = %w[ + addr2line.c io_buffer.c prism*.c scheduler.c +] + +def adjust_styles(files) + trailing = eofnewline = expandtab = indent = false + + edited_files = files.select do |f| + src = File.binread(f) rescue next + eofnewline = eofnewline0 = true if src.sub!(/(?<!\A|\n)\z/, "\n") + + trailing0 = false + expandtab0 = false + indent0 = false + + src.gsub!(/^.*$/).with_index do |line, lineno| + trailing = trailing0 = true if line.sub!(/[ \t]+$/, '') + line + end + + if f.end_with?('.c') || f.end_with?('.h') || f == 'insns.def' + # If and only if unedited lines did not have tab indentation, prevent introducing tab indentation to the file. + expandtab_allowed = src.each_line.with_index.all? do |line, lineno| + !line.start_with?("\t") + end + + if expandtab_allowed + src.gsub!(/^.*$/).with_index do |line, lineno| + if line.start_with?("\t") # last-committed line with hard tabs + expandtab = expandtab0 = true + line.sub(/\A\t+/) { |tabs| ' ' * (8 * tabs.length) } + else + line + end + end + end + end + + if File.fnmatch?("*.[chy]", f, File::FNM_PATHNAME) && + !DIFFERENT_STYLE_FILES.any? {|pat| File.fnmatch?(pat, f, File::FNM_PATHNAME)} + indent0 = true if src.gsub!(/^\w+\([^\n]*?\)\K[ \t]*(?=\{( *\\)?$)/, '\1' "\n") + indent0 = true if src.gsub!(/^([ \t]*)\}\K[ \t]*(?=else\b.*?( *\\)?$)/, '\2' "\n" '\1') + indent0 = true if src.gsub!(/^[ \t]*\}\n\K\n+(?=[ \t]*else\b)/, '') + indent ||= indent0 + end + + if trailing0 or eofnewline0 or expandtab0 or indent0 + File.binwrite(f, src) + true + end + end + if edited_files.empty? + return + else + msg = [('remove trailing spaces' if trailing), + ('append newline at EOF' if eofnewline), + ('expand tabs' if expandtab), + ('adjust indents' if indent), + ].compact + message = "* #{msg.join(', ')}. [ci skip]" + if expandtab + message += "\nPlease consider using misc/expand_tabs.rb as a pre-commit hook." + end + return message, edited_files + end +end + +oldrev, newrev, pushref = ARGV +if (dry_run = oldrev == '-n') or oldrev == '--' + _, *updated_files = ARGV + git = Git.new(nil, nil) +else + unless dry_run = pushref.nil? + branch = IO.popen(['git', 'rev-parse', '--symbolic', '--abbrev-ref', pushref], &:read).strip + end + git = Git.new(oldrev, newrev, branch) + + updated_files = git.updated_paths +end + +files = updated_files.select {|l| + /^\d/ !~ l and /\.bat\z/ !~ l and + (/\A(?:config|[Mm]akefile|GNUmakefile|README)/ =~ File.basename(l) or + /\A\z|\.(?:[chsy]|\d+|e?rb|tmpl|bas[eh]|z?sh|in|ma?k|def|src|trans|rdoc|ja|en|el|sed|awk|p[ly]|scm|mspec|html|rs)\z/ =~ File.extname(l)) +} +files.select! {|n| File.file?(n) } +files.reject! do |f| + IGNORED_FILES.any? { |re| f.match(re) } +end + +if files.empty? + puts "No files are an auto-style target:\n#{updated_files.join("\n")}" +elsif !(message, edited_files = adjust_styles(files)) + puts "All edited lines are formatted well:\n#{files.join("\n")}" +else + if dry_run + git.diff + abort message + else + git.commit(message, *edited_files) + git.push + end +end diff --git a/tool/auto_review_pr.rb b/tool/auto_review_pr.rb new file mode 100755 index 0000000000..07c98c7e0a --- /dev/null +++ b/tool/auto_review_pr.rb @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'uri' +require_relative './sync_default_gems' + +class GitHubAPIClient + def initialize(token) + @token = token + end + + def get(path) + response = Net::HTTP.get_response(URI("https://api.github.com#{path}"), { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end + + def post(path, body = {}) + body = JSON.dump(body) + response = Net::HTTP.post(URI("https://api.github.com#{path}"), body, { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + 'Content-Type' => 'application/json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end +end + +class AutoReviewPR + REPO = 'ruby/ruby' + + COMMENT_USER = 'github-actions[bot]' + COMMENT_PREFIX = 'The following files are maintained in the following upstream repositories:' + COMMENT_SUFFIX = 'Please file a pull request to the above instead. Thank you!' + + def initialize(client) + @client = client + end + + def review(pr_number) + # Fetch the list of files changed by the PR + changed_files = @client.get("/repos/#{REPO}/pulls/#{pr_number}/files").map { it.fetch(:filename) } + + # Build a Hash: { upstream_repo => files, ... } + upstream_repos = SyncDefaultGems::Repository.group(changed_files) + upstream_repos.delete(nil) # exclude no-upstream files + upstream_repos.delete('prism') if changed_files.include?('prism_compile.c') # allow prism changes in this case + if upstream_repos.empty? + puts "Skipped: The PR ##{pr_number} doesn't have upstream repositories." + return + end + + # Check if the PR is already reviewed + existing_comments = @client.get("/repos/#{REPO}/issues/#{pr_number}/comments") + existing_comments.map! { [it.fetch(:user).fetch(:login), it.fetch(:body)] } + if existing_comments.any? { |user, comment| user == COMMENT_USER && comment.start_with?(COMMENT_PREFIX) } + puts "Skipped: The PR ##{pr_number} already has an automated review comment." + return + end + + # Post a comment + comment = format_comment(upstream_repos) + result = @client.post("/repos/#{REPO}/issues/#{pr_number}/comments", { body: comment }) + puts "Success: #{JSON.pretty_generate(result)}" + end + + private + + # upstream_repos: { upstream_repo => files, ... } + def format_comment(upstream_repos) + comment = +'' + comment << "#{COMMENT_PREFIX}\n\n" + + upstream_repos.each do |upstream_repo, files| + comment << "* https://github.com/ruby/#{upstream_repo}\n" + files.each do |file| + comment << " * #{file}\n" + end + end + + comment << "\n#{COMMENT_SUFFIX}" + comment + end +end + +pr_number = ARGV[0] || abort("Usage: #{$0} <pr_number>") +client = GitHubAPIClient.new(ENV.fetch('GITHUB_TOKEN')) + +AutoReviewPR.new(client).review(pr_number) diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb index 53c374e973..91ac5628f1 100644 --- a/tool/bundler/dev_gems.rb +++ b/tool/bundler/dev_gems.rb @@ -3,17 +3,18 @@ source "https://rubygems.org" gem "test-unit", "~> 3.0" +gem "test-unit-ruby-core" gem "rake", "~> 13.1" gem "rb_sys" gem "turbo_tests", "~> 2.2.3" -gem "parallel_tests", "~> 4.7" +gem "parallel_tests", "~> 4.10.1" gem "parallel", "~> 1.19" gem "rspec-core", "~> 3.12" gem "rspec-expectations", "~> 3.12" gem "rspec-mocks", "~> 3.12" -gem "uri", "~> 0.13.0" +gem "rubygems-generate_index", "~> 1.1" group :doc do - gem "nronn", "~> 0.11.1", platform: :ruby + gem "ronn-ng", "~> 0.10.1", platform: :ruby end diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock new file mode 100644 index 0000000000..832127fb4c --- /dev/null +++ b/tool/bundler/dev_gems.rb.lock @@ -0,0 +1,132 @@ +GEM + remote: https://rubygems.org/ + specs: + compact_index (0.15.0) + diff-lcs (1.6.2) + kramdown (2.5.1) + rexml (>= 3.3.9) + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + mini_portile2 (2.8.9) + mustache (1.1.1) + nokogiri (1.19.0) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.19.0-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-java) + racc (~> 1.4) + nokogiri (1.19.0-x64-mingw-ucrt) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-gnu) + racc (~> 1.4) + parallel (1.27.0) + parallel_tests (4.10.1) + parallel + power_assert (3.0.1) + racc (1.8.1) + racc (1.8.1-java) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rexml (3.4.4) + ronn-ng (0.10.1) + kramdown (~> 2, >= 2.1) + kramdown-parser-gfm (~> 1, >= 1.0.1) + mustache (~> 1) + nokogiri (~> 1, >= 1.14.3) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + test-unit (3.7.7) + power_assert + test-unit-ruby-core (1.0.14) + test-unit (>= 3.7.2) + turbo_tests (2.2.5) + parallel_tests (>= 3.3.0, < 5) + rspec (>= 3.10) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm-linux + arm64-darwin + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86-linux + x86_64-darwin + x86_64-linux + +DEPENDENCIES + parallel (~> 1.19) + parallel_tests (~> 4.10.1) + rake (~> 13.1) + rb_sys + ronn-ng (~> 0.10.1) + rspec-core (~> 3.12) + rspec-expectations (~> 3.12) + rspec-mocks (~> 3.12) + rubygems-generate_index (~> 1.1) + test-unit (~> 3.0) + test-unit-ruby-core + turbo_tests (~> 2.2.3) + +CHECKSUMS + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + kramdown (2.5.1) sha256=87bbb6abd9d3cebe4fc1f33e367c392b4500e6f8fa19dd61c0972cf4afe7368c + kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 + mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289 + mustache (1.1.1) sha256=90891fdd50b53919ca334c8c1031eada1215e78d226d5795e523d6123a2717d0 + nokogiri (1.19.0) sha256=e304d21865f62518e04f2bf59f93bd3a97ca7b07e7f03952946d8e1c05f45695 + nokogiri (1.19.0-aarch64-linux-gnu) sha256=11a97ecc3c0e7e5edcf395720b10860ef493b768f6aa80c539573530bc933767 + nokogiri (1.19.0-arm-linux-gnu) sha256=572a259026b2c8b7c161fdb6469fa2d0edd2b61cd599db4bbda93289abefbfe5 + nokogiri (1.19.0-arm64-darwin) sha256=0811dfd936d5f6dd3f6d32ef790568bf29b2b7bead9ba68866847b33c9cf5810 + nokogiri (1.19.0-java) sha256=5f3a70e252be641d8a4099f7fb4cc25c81c632cb594eec9b4b8f2ca8be4374f3 + nokogiri (1.19.0-x64-mingw-ucrt) sha256=05d7ed2d95731edc9bef2811522dc396df3e476ef0d9c76793a9fca81cab056b + nokogiri (1.19.0-x86_64-darwin) sha256=1dad56220b603a8edb9750cd95798bffa2b8dd9dd9aa47f664009ee5b43e3067 + nokogiri (1.19.0-x86_64-linux-gnu) sha256=f482b95c713d60031d48c44ce14562f8d2ce31e3a9e8dd0ccb131e9e5a68b58c + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parallel_tests (4.10.1) sha256=df05458c691462b210f7a41fc2651d4e4e8a881e8190e6d1e122c92c07735d70 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + ronn-ng (0.10.1) sha256=4eeb0185c0fbfa889efed923b5b50e949cd869e7d82ac74138acd0c9c7165ec0 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + test-unit-ruby-core (1.0.14) sha256=d2e997796c9c5c5e8e31ac014f83a473ff5c2523a67cfa491b08893e12d43d22 + turbo_tests (2.2.5) sha256=3fa31497d12976d11ccc298add29107b92bda94a90d8a0a5783f06f05102509f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb index 4d0b21060a..a9b6fda11b 100644 --- a/tool/bundler/rubocop_gems.rb +++ b/tool/bundler/rubocop_gems.rb @@ -4,7 +4,8 @@ source "https://rubygems.org" gem "rubocop", ">= 1.52.1", "< 2" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock new file mode 100644 index 0000000000..d0c3120e11 --- /dev/null +++ b/tool/bundler/rubocop_gems.rb.lock @@ -0,0 +1,159 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) + ast (~> 2.4.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) + rake + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.82.1) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.48.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + stringio (3.2.0) + test-unit (3.7.7) + power_assert + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + irb + minitest (~> 5.1) + rake + rake-compiler + rb_sys + rspec + rubocop (>= 1.52.1, < 2) + test-unit + +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.82.1) sha256=09f1a6a654a960eda767aebea33e47603080f8e9c9a3f019bf9b94c9cab5e273 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb index 20c1ecd827..f7bc34cf5e 100644 --- a/tool/bundler/standard_gems.rb +++ b/tool/bundler/standard_gems.rb @@ -4,7 +4,8 @@ source "https://rubygems.org" gem "standard", "~> 1.0" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock new file mode 100644 index 0000000000..f3792f8611 --- /dev/null +++ b/tool/bundler/standard_gems.rb.lock @@ -0,0 +1,179 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) + ast (~> 2.4.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) + rake + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.81.7) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + rubocop-performance (1.26.1) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (1.13.0) + standard (1.52.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.81.7) + standard-custom (~> 1.0.0) + standard-performance (~> 1.8) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.9.0) + lint_roller (~> 1.1) + rubocop-performance (~> 1.26.0) + stringio (3.2.0) + test-unit (3.7.7) + power_assert + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + irb + minitest (~> 5.1) + rake + rake-compiler + rb_sys + rspec + standard (~> 1.0) + test-unit + +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.81.7) sha256=6fb5cc298c731691e2a414fe0041a13eb1beed7bab23aec131da1bcc527af094 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834 + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + standard (1.52.0) sha256=ec050e63228e31fabe40da3ef96da7edda476f7acdf3e7c2ad47b6e153f6a076 + standard-custom (1.0.2) sha256=424adc84179a074f1a2a309bb9cf7cd6bfdb2b6541f20c6bf9436c0ba22a652b + standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb index 2d66c98dbd..ddc19e2939 100644 --- a/tool/bundler/test_gems.rb +++ b/tool/bundler/test_gems.rb @@ -2,14 +2,17 @@ source "https://rubygems.org" -gem "rack", "~> 3.0" -gem "rackup", "~> 2.1" -gem "base64" -gem "webrick", "~> 1.9" +gem "rack", "~> 3.1" gem "rack-test", "~> 2.1" gem "compact_index", "~> 0.15.0" gem "sinatra", "~> 4.1" gem "rake", "~> 13.1" gem "builder", "~> 3.2" gem "rb_sys" +gem "fiddle" gem "rubygems-generate_index", "~> 1.1" +gem "concurrent-ruby" +gem "psych" +gem "etc", platforms: [:ruby, :windows] +gem "open3" +gem "shellwords" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock new file mode 100644 index 0000000000..fdffc1f09d --- /dev/null +++ b/tool/bundler/test_gems.rb.lock @@ -0,0 +1,106 @@ +GEM + remote: https://rubygems.org/ + specs: + base64 (0.3.0) + builder (3.3.0) + compact_index (0.15.0) + concurrent-ruby (1.3.6) + date (3.5.1) + date (3.5.1-java) + etc (1.4.6) + fiddle (1.1.8) + jar-dependencies (0.5.5) + logger (1.7.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + open3 (0.2.1) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + rack (3.2.4) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + ruby2_keywords (0.0.5) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + shellwords (0.2.2) + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) + tilt (~> 2.0) + stringio (3.2.0) + tilt (2.6.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + builder (~> 3.2) + compact_index (~> 0.15.0) + concurrent-ruby + etc + fiddle + open3 + psych + rack (~> 3.1) + rack-test (~> 2.1) + rake (~> 13.1) + rb_sys + rubygems-generate_index (~> 1.1) + shellwords + sinatra (~> 4.1) + +CHECKSUMS + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + etc (1.4.6) sha256=0f7e9e7842ea5e3c3bd9bc81746ebb8c65ea29e4c42a93520a0d638129c7de01 + fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mustermann (3.0.4) sha256=85fadcb6b3c6493a8b511b42426f904b7f27b282835502233dd154daab13aa22 + open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + rack (3.2.4) sha256=5d74b6f75082a643f43c1e76b419c40f0e5527fcfee1e669ac1e6b73c0ccb6f6 + rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac + rack-session (2.1.1) sha256=0b6dc07dea7e4b583f58a48e8b806d4c9f1c6c9214ebc202ec94562cbea2e4e9 + rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94 + sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + tilt (2.6.1) sha256=35a99bba2adf7c1e362f5b48f9b581cce4edfba98117e34696dde6d308d84770 + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb index b8525c2e90..8d12c5adde 100644 --- a/tool/bundler/vendor_gems.rb +++ b/tool/bundler/vendor_gems.rb @@ -2,16 +2,16 @@ source "https://rubygems.org" -gem "fileutils", "1.7.3" -gem "molinillo", github: "cocoapods/molinillo" -gem "net-http", "0.6.0" -gem "net-http-persistent", "4.0.4" +gem "fileutils", "1.8.0" +gem "molinillo", github: "cocoapods/molinillo", ref: "1d62d7d5f448e79418716dc779a4909509ccda2a" +gem "net-http", "0.7.0" # net-http-0.8.0 is broken with JRuby +gem "net-http-persistent", "4.0.6" gem "net-protocol", "0.2.2" -gem "optparse", "0.6.0" -gem "pub_grub", github: "jhawthorn/pub_grub" -gem "resolv", "0.6.0" +gem "optparse", "0.8.0" +gem "pub_grub", github: "jhawthorn/pub_grub", ref: "df6add45d1b4d122daff2f959c9bd1ca93d14261" +gem "resolv", "0.6.2" gem "securerandom", "0.4.1" -gem "timeout", "0.4.3" -gem "thor", "1.3.2" +gem "timeout", "0.4.4" +gem "thor", "1.4.0" gem "tsort", "0.2.0" -gem "uri", "1.0.2" +gem "uri", "1.1.1" diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock new file mode 100644 index 0000000000..cc7886e60b --- /dev/null +++ b/tool/bundler/vendor_gems.rb.lock @@ -0,0 +1,75 @@ +GIT + remote: https://github.com/cocoapods/molinillo.git + revision: 1d62d7d5f448e79418716dc779a4909509ccda2a + ref: 1d62d7d5f448e79418716dc779a4909509ccda2a + specs: + molinillo (0.8.0) + +GIT + remote: https://github.com/jhawthorn/pub_grub.git + revision: df6add45d1b4d122daff2f959c9bd1ca93d14261 + ref: df6add45d1b4d122daff2f959c9bd1ca93d14261 + specs: + pub_grub (0.5.0) + +GEM + remote: https://rubygems.org/ + specs: + connection_pool (2.5.4) + fileutils (1.8.0) + net-http (0.7.0) + uri + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + net-protocol (0.2.2) + timeout + optparse (0.8.0) + resolv (0.6.2) + securerandom (0.4.1) + thor (1.4.0) + timeout (0.4.4) + tsort (0.2.0) + uri (1.1.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + fileutils (= 1.8.0) + molinillo! + net-http (= 0.7.0) + net-http-persistent (= 4.0.6) + net-protocol (= 0.2.2) + optparse (= 0.8.0) + pub_grub! + resolv (= 0.6.2) + securerandom (= 0.4.1) + thor (= 1.4.0) + timeout (= 0.4.4) + tsort (= 0.2.0) + uri (= 1.1.1) + +CHECKSUMS + connection_pool (2.5.4) sha256=e9e1922327416091f3f6542f5f4446c2a20745276b9aa796dd0bb2fd0ea1e70a + fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02 + molinillo (0.8.0) + net-http (0.7.0) sha256=4db7d9f558f8ffd4dcf832d0aefd02320c569c7d4f857def49e585069673a425 + net-http-persistent (4.0.6) sha256=2abb3a04438edf6cb9e0e7e505969605f709eda3e3c5211beadd621a2c84dd5d + net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 + optparse (0.8.0) sha256=ef6b7fbaf7ec331474f325bc08dd5622e6e1e651007a5341330ee4b08ce734f0 + pub_grub (0.5.0) + resolv (0.6.2) sha256=61efe545cedddeb1b14f77e51f85c85ca66af5098fdbf567fadf32c34590fb14 + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + thor (1.4.0) sha256=8763e822ccb0f1d7bee88cde131b19a65606657b847cc7b7b4b82e772bcd8a3d + timeout (0.4.4) sha256=f0f6f970104b82427cd990680f539b6bbb8b1e55efa913a55c6492935e4e0edb + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + +BUNDLED WITH + 4.0.0.dev diff --git a/tool/commit-email.rb b/tool/commit-email.rb new file mode 100755 index 0000000000..c887f8783e --- /dev/null +++ b/tool/commit-email.rb @@ -0,0 +1,372 @@ +#!/usr/bin/env ruby + +require "optparse" +require "nkf" +require "shellwords" + +CommitEmailInfo = Struct.new( + :author, + :author_email, + :revision, + :entire_sha256, + :date, + :log, + :branch, + :diffs, + :added_files, :deleted_files, :updated_files, + :added_dirs, :deleted_dirs, :updated_dirs, +) + +class GitInfoBuilder + GitCommandFailure = Class.new(RuntimeError) + + def initialize(repo_path) + @repo_path = repo_path + end + + def build(oldrev, newrev, refname) + diffs = build_diffs(oldrev, newrev) + + info = CommitEmailInfo.new + info.author = git_show(newrev, format: '%an') + info.author_email = normalize_email(git_show(newrev, format: '%aE')) + info.revision = newrev[0...10] + info.entire_sha256 = newrev + info.date = Time.at(Integer(git_show(newrev, format: '%at'))) + info.log = git_show(newrev, format: '%B') + info.branch = git('rev-parse', '--symbolic', '--abbrev-ref', refname).strip + info.diffs = diffs + info.added_files = find_files(diffs, status: :added) + info.deleted_files = find_files(diffs, status: :deleted) + info.updated_files = find_files(diffs, status: :modified) + info.added_dirs = [] # git does not deal with directory + info.deleted_dirs = [] # git does not deal with directory + info.updated_dirs = [] # git does not deal with directory + info + end + + private + + # Force git-svn email address to @ruby-lang.org to avoid email bounce by invalid email address. + def normalize_email(email) + if email.match(/\A[^@]+@\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/) # git-svn + svn_user, _ = email.split('@', 2) + "#{svn_user}@ruby-lang.org" + else + email + end + end + + def find_files(diffs, status:) + files = [] + diffs.each do |path, values| + if values.keys.first == status + files << path + end + end + files + end + + # SVN version: + # { + # "filename" => { + # "[modified|added|deleted|copied|property_changed]" => { + # type: "[modified|added|deleted|copied|property_changed]", + # body: "diff body", # not implemented because not used + # added: Integer, + # deleted: Integer, + # } + # } + # } + def build_diffs(oldrev, newrev) + diffs = {} + + numstats = git('diff', '--numstat', oldrev, newrev).lines.map { |l| l.strip.split("\t", 3) } + git('diff', '--name-status', oldrev, newrev).each_line do |line| + status, path, _newpath = line.strip.split("\t", 3) + diff = build_diff(path, numstats) + + case status + when 'A' + diffs[path] = { added: { type: :added, **diff } } + when 'M' + diffs[path] = { modified: { type: :modified, **diff } } + when 'C' + diffs[path] = { copied: { type: :copied, **diff } } + when 'D' + diffs[path] = { deleted: { type: :deleted, **diff } } + when /\AR/ # R100 (which does not exist in git.ruby-lang.org's git 2.1.4) + # TODO: implement something + else + $stderr.puts "unexpected git diff status: #{status}" + end + end + + diffs + end + + def build_diff(path, numstats) + diff = { added: 0, deleted: 0 } # :body not implemented because not used + line = numstats.find { |(_added, _deleted, file, *)| file == path } + return diff if line.nil? + + added, deleted, _ = line + if added + diff[:added] = Integer(added) + end + if deleted + diff[:deleted] = Integer(deleted) + end + diff + end + + def git_show(revision, format:) + git('show', '--no-show-signature', "--pretty=#{format}", '--no-patch', revision).strip + end + + def git(*args) + command = ['git', '-C', @repo_path, *args] + output = with_gitenv { IO.popen(command, external_encoding: 'UTF-8', &:read) } + unless $?.success? + raise GitCommandFailure, "failed to execute '#{command.join(' ')}':\n#{output}" + end + output + end + + def with_gitenv + orig = ENV.to_h.dup + begin + ENV.delete('GIT_DIR') + yield + ensure + ENV.replace(orig) + end + end +end + +CommitEmailOptions = Struct.new(:error_to, :viewer_uri) + +CommitEmail = Module.new +class << CommitEmail + SENDMAIL = ENV.fetch('SENDMAIL', '/usr/sbin/sendmail') + private_constant :SENDMAIL + + def parse(args) + options = CommitEmailOptions.new + + opts = OptionParser.new do |opts| + opts.separator('') + + opts.on('-e', '--error-to [TO]', + 'Add [TO] to to address when error is occurred') do |to| + options.error_to = to + end + + opts.on('--viewer-uri [URI]', + 'Use [URI] as URI of revision viewer') do |uri| + options.viewer_uri = uri + end + + opts.on_tail('--help', 'Show this message') do + puts opts + exit + end + end + + return opts.parse(args), options + end + + def main(repo_path, to, rest) + args, options = parse(rest) + + infos = args.each_slice(3).flat_map do |oldrev, newrev, refname| + revisions = IO.popen(['git', 'log', '--no-show-signature', '--reverse', '--pretty=%H', "#{oldrev}^..#{newrev}"], &:read).lines.map(&:strip) + revisions[0..-2].zip(revisions[1..-1]).map do |old, new| + GitInfoBuilder.new(repo_path).build(old, new, refname) + end + end + + infos.each do |info| + next if info.branch.start_with?('notes/') + puts "#{info.branch}: #{info.revision} (#{info.author})" + + from = make_from(name: info.author, email: "noreply@ruby-lang.org") + sendmail(to, from, make_mail(to, from, info, viewer_uri: options.viewer_uri)) + end + end + + def sendmail(to, from, mail) + IO.popen([*SENDMAIL.shellsplit, to], 'w') do |f| + f.print(mail) + end + unless $?.success? + raise "Failed to run `#{SENDMAIL} #{to}` with: '#{mail}'" + end + end + + private + + def b_encode(str) + NKF.nkf('-WwM', str) + end + + def make_body(info, viewer_uri:) + body = +'' + body << "#{info.author}\t#{format_time(info.date)}\n" + body << "\n" + body << " New Revision: #{info.revision}\n" + body << "\n" + body << " #{viewer_uri}#{info.revision}\n" + body << "\n" + body << " Log:\n" + body << info.log.lstrip.gsub(/^\t*/, ' ').rstrip + body << "\n\n" + body << added_dirs(info) + body << added_files(info) + body << deleted_dirs(info) + body << deleted_files(info) + body << modified_dirs(info) + body << modified_files(info) + [body.rstrip].pack('M') + end + + def format_time(time) + time.strftime('%Y-%m-%d %X %z (%a, %d %b %Y)') + end + + def changed_items(title, type, items) + rv = '' + unless items.empty? + rv << " #{title} #{type}:\n" + rv << items.collect {|item| " #{item}\n"}.join('') + end + rv + end + + def changed_files(title, files) + changed_items(title, 'files', files) + end + + def added_files(info) + changed_files('Added', info.added_files) + end + + def deleted_files(info) + changed_files('Removed', info.deleted_files) + end + + def modified_files(info) + changed_files('Modified', info.updated_files) + end + + def changed_dirs(title, files) + changed_items(title, 'directories', files) + end + + def added_dirs(info) + changed_dirs('Added', info.added_dirs) + end + + def deleted_dirs(info) + changed_dirs('Removed', info.deleted_dirs) + end + + def modified_dirs(info) + changed_dirs('Modified', info.updated_dirs) + end + + def changed_dirs_info(info, uri) + (info.added_dirs.collect do |dir| + " Added: #{dir}\n" + end + info.deleted_dirs.collect do |dir| + " Deleted: #{dir}\n" + end + info.updated_dirs.collect do |dir| + " Modified: #{dir}\n" + end).join("\n") + end + + def diff_info(info, uri) + info.diffs.collect do |key, values| + [ + key, + values.collect do |type, value| + case type + when :added + rev = "?revision=#{info.revision}&view=markup" + when :modified, :property_changed + prev_revision = (info.revision.is_a?(Integer) ? info.revision - 1 : "#{info.revision}^") + rev = "?r1=#{info.revision}&r2=#{prev_revision}&diff_format=u" + when :deleted, :copied + rev = '' + else + raise "unknown diff type: #{value[:type]}" + end + + link = [uri, key.sub(/ .+/, '') || ''].join('/') + rev + + desc = '' + + [desc, link] + end + ] + end + end + + def make_header(to, from, info) + <<~EOS + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: #{from} + To: #{to} + Subject: #{make_subject(info)} + EOS + end + + def make_subject(info) + subject = +'' + subject << "#{info.revision}" + subject << " (#{info.branch})" + subject << ': ' + subject << info.log.lstrip.lines.first.to_s.strip + b_encode(subject) + end + + # https://tools.ietf.org/html/rfc822#section-4.1 + # https://tools.ietf.org/html/rfc822#section-6.1 + # https://tools.ietf.org/html/rfc822#appendix-D + # https://tools.ietf.org/html/rfc2047 + def make_from(name:, email:) + if name.ascii_only? + escaped_name = name.gsub(/["\\\n]/) { |c| "\\#{c}" } + %Q["#{escaped_name}" <#{email}>] + else + escaped_name = "=?UTF-8?B?#{NKF.nkf('-WwMB', name)}?=" + %Q[#{escaped_name} <#{email}>] + end + end + + def make_mail(to, from, info, viewer_uri:) + make_header(to, from, info) + make_body(info, viewer_uri: viewer_uri) + end +end + +repo_path, to, *rest = ARGV +begin + CommitEmail.main(repo_path, to, rest) +rescue StandardError => e + $stderr.puts "#{e.class}: #{e.message}" + $stderr.puts e.backtrace + + _, options = CommitEmail.parse(rest) + to = options.error_to + CommitEmail.sendmail(to, to, <<-MAIL) +From: #{to} +To: #{to} +Subject: Error + +#{$!.class}: #{$!.message} +#{$@.join("\n")} +MAIL + exit 1 +end diff --git a/tool/downloader.rb b/tool/downloader.rb index 3a91ea0b93..39ebf44a83 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -1,39 +1,14 @@ # Used by configure and make to download or update mirrored Ruby and GCC -# files. This will use HTTPS if possible, falling back to HTTP. +# files. + +# -*- frozen-string-literal: true -*- require 'fileutils' require 'open-uri' require 'pathname' -begin - require 'net/https' -rescue LoadError - https = 'http' -else - https = 'https' - - # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old - # versions do not. so, patching OpenSSL::X509::Store#add_file instead. - class OpenSSL::X509::Store - alias orig_add_file add_file - def add_file(pems) - Array(pems).each do |pem| - if File.directory?(pem) - add_path pem - else - orig_add_file pem - end - end - end - end - # since open-uri internally checks ssl_ca_cert using File.directory?, - # allow to accept an array. - class <<File - alias orig_directory? directory? - def File.directory? files - files.is_a?(Array) ? false : orig_directory?(files) - end - end -end +verbose, $VERBOSE = $VERBOSE, nil +require 'net/https' +$VERBOSE = verbose class Downloader def self.find(dlname) @@ -42,47 +17,37 @@ class Downloader end end - def self.https=(https) - @@https = https - end - - def self.https? - @@https == 'https' - end - - def self.https - @@https - end - def self.get_option(argv, options) false end class GNU < self - def self.download(name, *rest) - if https? - begin - super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest) - rescue => e - m1, m2 = e.message.split("\n", 2) - STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" - super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) - end + Mirrors = %w[ + https://raw.githubusercontent.com/autotools-mirror/autoconf/refs/heads/master/build-aux/ + https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master + ] + + def self.download(name, *rest, **options) + Mirrors.each_with_index do |url, i| + super("#{url}/#{name}", name, *rest, **options) + rescue => e + raise if i + 1 == Mirrors.size # no more URLs + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" else - super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) + return end end end class RubyGems < self - def self.download(name, dir = nil, since = true, options = {}) + def self.download(name, dir = nil, since = true, **options) require 'rubygems' - options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? options[:ignore_http_client_errors] = true end - super("https://rubygems.org/downloads/#{name}", name, dir, since, options) + super("https://rubygems.org/downloads/#{name}", name, dir, since, **options) end end @@ -107,16 +72,13 @@ class Downloader end end - def self.download(name, dir = nil, since = true, options = {}) - options = options.dup - unicode_beta = options.delete(:unicode_beta) + def self.download(name, dir = nil, since = true, unicode_beta: nil, **options) name_dir_part = name.sub(/[^\/]+$/, '') if unicode_beta == 'YES' if INDEX.size == 0 - index_options = options.dup - index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file + cache_save = false # TODO: make sure caching really doesn't work for index file index_data = File.read(under(dir, "index.html")) rescue nil - index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options) + index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, cache_save: cache_save, **options) INDEX[:index] = File.read(index_file) since = true unless INDEX[:index] == index_data end @@ -125,7 +87,7 @@ class Downloader beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] # make sure we always check for new versions of files, # because they can easily change in the beta period - super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options) + super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, **options) else index_file = Pathname.new(under(dir, name_dir_part+'index.html')) if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ @@ -133,7 +95,7 @@ class Downloader "Remove all files in this directory and in .downloaded-cache/ " + "because they may be leftovers from the beta period." end - super(UNICODE_PUBLIC+name, name, dir, since, options) + super(UNICODE_PUBLIC+name, name, dir, since, **options) end end end @@ -198,62 +160,44 @@ class Downloader # Example usage: # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', # 'UnicodeData.txt', 'enc/unicode/data' - def self.download(url, name, dir = nil, since = true, options = {}) - options = options.dup + def self.download(url, name, dir = nil, since = true, + cache_save: ENV["CACHE_SAVE"] != "no", cache_dir: nil, + ignore_http_client_errors: nil, + dryrun: nil, verbose: false, **options) url = URI(url) - dryrun = options.delete(:dryrun) - if name file = Pathname.new(under(dir, name)) else name = File.basename(url.path) end - cache_save = options.delete(:cache_save) { - ENV["CACHE_SAVE"] != "no" - } - cache = cache_file(url, name, options.delete(:cache_dir)) + cache = cache_file(url, name, cache_dir) file ||= cache if since.nil? and file.exist? - if $VERBOSE + if verbose $stdout.puts "#{file} already exists" $stdout.flush end - if cache_save - save_cache(cache, file, name) - end return file.to_path end if dryrun puts "Download #{url} into #{file}" return end - if link_cache(cache, file, name, $VERBOSE) + if link_cache(cache, file, name, verbose: verbose) return file.to_path end - if !https? and URI::HTTPS === url - warn "*** using http instead of https ***" - url.scheme = 'http' - url = URI(url.to_s) - end - if $VERBOSE + if verbose $stdout.print "downloading #{name} ... " $stdout.flush end mtime = nil - ignore_http_client_errors = options.delete(:ignore_http_client_errors) options = options.merge(http_options(file, since.nil? ? true : since)) begin - data = with_retry(10) do - data = url.read(options) - if mtime = data.meta["last-modified"] - mtime = Time.httpdate(mtime) - end - data - end + data = with_retry(10) {url.read(options)} rescue OpenURI::HTTPError => http_error case http_error.message when /^304 / # 304 Not Modified - if $VERBOSE + if verbose $stdout.puts "#{name} not modified" $stdout.flush end @@ -277,6 +221,10 @@ class Downloader return file.to_path end raise + else + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end end dest = (cache_save && cache && !cache.exist? ? cache : file) dest.parent.mkpath @@ -288,7 +236,7 @@ class Downloader if mtime dest.utime(mtime, mtime) end - if $VERBOSE + if verbose $stdout.puts "done" $stdout.flush end @@ -306,20 +254,24 @@ class Downloader dir ? File.join(dir, File.basename(name)) : name end + def self.default_cache_dir + if cache_dir = ENV['CACHE_DIR'] + return cache_dir unless cache_dir.empty? + end + ".downloaded-cache" + end + def self.cache_file(url, name, cache_dir = nil) case cache_dir when false return nil when nil - cache_dir = ENV['CACHE_DIR'] - if !cache_dir or cache_dir.empty? - cache_dir = ".downloaded-cache" - end + cache_dir = default_cache_dir end Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) end - def self.link_cache(cache, file, name, verbose = false) + def self.link_cache(cache, file, name, verbose: false) return false unless cache and cache.exist? return true if cache.eql?(file) if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] @@ -391,8 +343,6 @@ class Downloader private_class_method :with_retry end -Downloader.https = https.freeze - if $0 == __FILE__ since = true options = {} @@ -460,7 +410,7 @@ if $0 == __FILE__ end ARGV.shift end - $VERBOSE = true + options[:verbose] = true if dl args.each do |name| dir = destdir @@ -479,10 +429,10 @@ if $0 == __FILE__ end name = "#{prefix}/#{name}" end - dl.download(name, dir, since, options) + dl.download(name, dir, since, **options) end else abort "usage: #{$0} url name" unless args.size == 2 - Downloader.download(args[0], args[1], destdir, since, options) + Downloader.download(args[0], args[1], destdir, since, **options) end end diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index 9d49f427bb..a89390ad8f 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -12,6 +12,9 @@ # You can get source file for gperf. After this, simply make ruby. # Or directly run: # tool/enc-unicode.rb --header data_dir emoji_data_dir > enc/unicode/<VERSION>/name2ctype.h +# +# There are Makefile rules that automate steps above: `make update-unicode` and +# `make enc/unicode/<VERSION>/name2ctype.h`. while arg = ARGV.shift case arg @@ -143,7 +146,8 @@ def define_posix_props(data) data['Space'] = data['White_Space'] data['Blank'] = data['Space_Separator'] + [0x0009] data['Cntrl'] = data['Cc'] - data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + + data['Connector_Punctuation'] + data['Join_Control'] data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - data['Surrogate'] - data['Unassigned'] data['Print'] = data['Graph'] + data['Space_Separator'] @@ -161,14 +165,24 @@ def parse_scripts(data, categories) names = {} files.each do |file| data_foreach(file[:fn]) do |line| + # Parse Unicode data files and store code points and properties. if /^# Total (?:code points|elements): / =~ line data[current] = cps categories[current] = file[:title] (names[file[:title]] ||= []) << current cps = [] - elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line - current = $3 + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w(?:[\w\s;]*\w)?)/ =~ line + # $1: The first hexadecimal code point or the start of a range. + # $2: The end code point of the range, if present. + # If there's no range (just a single code point), $2 is nil. + # $3: The property or other info. + # Example: + # line = "0915..0939 ; InCB; Consonant # Lo [37] DEVANAGARI LETTER KA..DEVANAGARI LETTER HA" + # $1 = "0915" + # $2 = "0939" + # $3 = "InCB; Consonant" $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + current = $3.gsub(/\W+/, '_') end end end @@ -486,7 +500,11 @@ end output.ifdef :USE_UNICODE_PROPERTIES do props.each do |name| i += 1 - name = normalize_propname(name) + name = if name.start_with?('InCB') + name.downcase.gsub(/_/, '=') + else + normalize_propname(name) + end name_to_index[name] = i puts "%-40s %3d" % [name + ',', i] end diff --git a/tool/extlibs.rb b/tool/extlibs.rb index 887cac61eb..cef6712833 100755 --- a/tool/extlibs.rb +++ b/tool/extlibs.rb @@ -43,7 +43,7 @@ class ExtLibs end def do_download(url, cache_dir) - Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) + Downloader.download(url, nil, nil, nil, cache_dir: cache_dir) end def do_checksum(cache, chksums) diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb index f50bda360a..127ea236f3 100755 --- a/tool/fetch-bundled_gems.rb +++ b/tool/fetch-bundled_gems.rb @@ -1,4 +1,4 @@ -#!ruby -an +#!ruby -alnF\s+|#.* BEGIN { require 'fileutils' require_relative 'lib/colorize' @@ -21,27 +21,27 @@ BEGIN { n, v, u, r = $F next unless n -next if n =~ /^#/ next if bundled_gems&.all? {|pat| !File.fnmatch?(pat, n)} -if File.directory?(n) - puts "updating #{color.notice(n)} ..." - system("git", "fetch", "--all", chdir: n) or abort -else +unless File.exist?("#{n}/.git") puts "retrieving #{color.notice(n)} ..." - system(*%W"git clone #{u} #{n}") or abort + system(*%W"git clone --depth=1 --no-tags #{u} #{n}") or abort end if r puts "fetching #{color.notice(r)} ..." system("git", "fetch", "origin", r, chdir: n) or abort + c = r +else + c = ["v#{v}", v].find do |c| + puts "fetching #{color.notice(c)} ..." + system("git", "fetch", "origin", "refs/tags/#{c}:refs/tags/#{c}", chdir: n) + end or abort end -c = r || "v#{v}" checkout = %w"git -c advice.detachedHead=false checkout" -print %[checking out #{color.notice(c)} (v=#{color.info(v)}] -print %[, r=#{color.info(r)}] if r -puts ") ..." +info = %[, r=#{color.info(r)}] if r +puts "checking out #{color.notice(c)} (v=#{color.info(v)}#{info}) ..." unless system(*checkout, c, "--", chdir: n) abort if r or !system(*checkout, v, "--", chdir: n) end diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb index 6200e78a56..2de8379606 100755 --- a/tool/file2lastrev.rb +++ b/tool/file2lastrev.rb @@ -98,6 +98,7 @@ ok = true data.sub!(/(?<!\A|\n)\z/, "\n") @output.write(data, overwrite: true, create_only: create_only) rescue => e + next if @suppress_not_found and VCS::NotFoundError === e warn "#{File.basename(Program)}: #{e.message}" ok = false end diff --git a/tool/format-release b/tool/format-release index 72fc173000..8bb6154243 100755 --- a/tool/format-release +++ b/tool/format-release @@ -9,6 +9,7 @@ end require "open-uri" require "yaml" +require_relative "./ruby-version" Diffy::Diff.default_options.merge!( include_diff_info: true, @@ -30,10 +31,9 @@ class Tarball def gz?; @url.end_with?('.gz'); end def zip?; @url.end_with?('.zip'); end - def bz2?; @url.end_with?('.bz2'); end def xz?; @url.end_with?('.xz'); end - def ext; @url[/(?:zip|tar\.(?:gz|bz2|xz))\z/]; end + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end def to_md <<eom @@ -55,25 +55,10 @@ eom unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" - info = YAML.load(URI(uri).read) + info = YAML.unsafe_load(URI(uri).read) if info.size != 1 raise "unexpected info.yml '#{uri}'" end @@ -88,9 +73,10 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) stat = `git -C #{rubydir} diff -l0 --shortstat #{prev_tag}..#{tag}` files_changed, insertions, deletions = stat.scan(/\d+/) end @@ -184,7 +170,7 @@ eom if /\.0(?:-\w+)?\z/ =~ ver # preview, rc, or first release entry <<= <<eom - tag: v#{ver.tr('.-', '_')} + tag: #{RubyVersion.tag(ver)} stats: files_changed: #{files_changed} insertions: #{insertions} @@ -196,34 +182,25 @@ eom url: gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip - bz2: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.bz2 xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz size: gz: #{ary.find{|x|x.gz? }.size} zip: #{ary.find{|x|x.zip?}.size} - bz2: #{ary.find{|x|x.bz2?}&.size} xz: #{ary.find{|x|x.xz? }.size} sha1: gz: #{ary.find{|x|x.gz? }.sha1} zip: #{ary.find{|x|x.zip?}.sha1} - bz2: #{ary.find{|x|x.bz2?}&.sha1} xz: #{ary.find{|x|x.xz? }.sha1} sha256: gz: #{ary.find{|x|x.gz? }.sha256} zip: #{ary.find{|x|x.zip?}.sha256} - bz2: #{ary.find{|x|x.bz2?}&.sha256} xz: #{ary.find{|x|x.xz? }.sha256} sha512: gz: #{ary.find{|x|x.gz? }.sha512} zip: #{ary.find{|x|x.zip?}.sha512} - bz2: #{ary.find{|x|x.bz2?}&.sha512} xz: #{ary.find{|x|x.xz? }.sha512} eom - if ver.start_with?("3.") - entry = entry.gsub(/ bz2: .*\n/, "") - end - if data.include?("\n- version: #{ver}\n") # update existing entry data.sub!(/\n- version: #{ver}\n(^ .*\n)*\n/, "\n#{entry}\n") diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb index fd429dab37..daa1a1f235 100644 --- a/tool/lib/_tmpdir.rb +++ b/tool/lib/_tmpdir.rb @@ -4,11 +4,11 @@ template = "rubytest." # Assume the directory by these environment variables are safe. base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| next unless tmp and tmp.size <= 50 and File.directory?(tmp) - # On macOS, the default TMPDIR is very long, inspite of UNIX socket - # path length is limited. + # On macOS, the default TMPDIR is very long, in spite of UNIX socket + # path length being limited. # # Also Rubygems creates its own temporary directory per tests, and - # some tests copy the full path of gemhome there. In that caes, the + # some tests copy the full path of gemhome there. In that case, the # path contains both temporary names twice, and can exceed path name # limit very easily. tmp diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb new file mode 100644 index 0000000000..9ad5ea220b --- /dev/null +++ b/tool/lib/bundle_env.rb @@ -0,0 +1,4 @@ +ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_PATH__SYSTEM"] = "true" +ENV["BUNDLE_WITHOUT"] = "lint doc" diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index d870925d05..d2ed61a508 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -12,14 +12,18 @@ module BundledGem "singleton", # prime "ipaddr", # rinda "forwardable", # prime, rinda - "strscan" # rexml + "strscan", # rexml + "psych" # rdoc ] module_function def unpack(file, *rest) pkg = Gem::Package.new(file) - prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)} + prepare_test(pkg.spec, *rest) do |dir| + pkg.extract_files(dir) + FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)}) + end puts "Unpacked #{file}" rescue Gem::Package::FormatError, Errno::ENOENT puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 62ada2bdca..e29a0e3c25 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -75,9 +75,18 @@ module Test require_relative 'envutil' require 'pp' begin - require '-test-/asan' + require '-test-/sanitizers' rescue LoadError + # in test-unit-ruby-core gem + def sanitizers + nil + end + else + def sanitizers + Test::Sanitizers + end end + module_function :sanitizers nil.pretty_inspect @@ -97,11 +106,14 @@ module Test end def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, - success: nil, **opt) + success: nil, failed: nil, gems: false, **opt) args = Array(args).dup - args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') + unless gems.nil? + args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems") + end stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) - desc = FailDesc[status, message, stderr] + desc = failed[status, message, stderr] if failed + desc ||= FailDesc[status, message, stderr] if block_given? raise "test_stdout ignored, use block only or without block" if test_stdout != [] raise "test_stderr ignored, use block only or without block" if test_stderr != [] @@ -159,7 +171,7 @@ module Test pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # ASAN has the same problem - its shadow memory greatly increases memory usage # (plus asan has better ways to detect memory leaks than this assertion) - pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if defined?(Test::ASAN) && Test::ASAN.enabled? + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled? require_relative 'memory_status' raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) @@ -291,9 +303,34 @@ module Test def separated_runner(token, out = nil) include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + out = out ? IO.new(out, 'w') : STDOUT + + # avoid method redefinitions + out_write = out.method(:write) + integer_to_s = Integer.instance_method(:to_s) + array_pack = Array.instance_method(:pack) + marshal_dump = Marshal.method(:dump) + assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set) + assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get) + Test::Unit::Assertions.module_eval do + @_assertions = 0 + + undef _assertions= + define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)}) + + undef _assertions + define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)}) + end + # assume Method#call and UnboundMethod#bind_call need to work as the original + at_exit { - out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" + assertions = assertions_ivar_get.call(:@_assertions) + out_write.call <<~OUT + <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}> + #{array_pack.bind_call([marshal_dump.call($!)], 'm0')} + </error id="#{token}"> + OUT } if defined?(Test::Unit::Runner) Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) @@ -327,7 +364,16 @@ eom args = args.dup args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + # power_assert 3 requires ruby 3.1 or later + args << "-W:no-experimental" if RUBY_VERSION < "3.1." stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + + if sanitizers&.lsan_enabled? + # LSAN may output messages like the following line into stderr. We should ignore it. + # ==276855==Running thread 276851 was not suspended. False leaks are possible. + # See https://github.com/google/sanitizers/issues/1479 + stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "") + end ensure if res_c res_c.close @@ -338,15 +384,16 @@ eom end raise if $! abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + marshal_error = nil assert(!abort, FailDesc[status, nil, stderr]) - self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i - begin - res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m")) + res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do + self._assertions += $1.to_i + res = Marshal.load($2.unpack1("m")) or next rescue => marshal_error ignore_stderr = nil res = nil - end - if res and !(SystemExit === res) + else + next if SystemExit === res if bt = res.backtrace bt.each do |l| l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} @@ -358,7 +405,7 @@ eom raise res end - # really is it succeed? + # really did it succeed? unless ignore_stderr # the body of assert_separately must not output anything to detect error assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) @@ -369,9 +416,17 @@ eom # Run Ractor-related test without influencing the main test suite def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) - return unless defined?(Ractor) + omit unless defined?(Ractor) + + # https://bugs.ruby-lang.org/issues/21262 + shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value) + shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join) + + if require + require = [require] unless require.is_a?(Array) + require = require.map {|r| "require #{r.inspect}"}.join("\n") + end - require = "require #{require.inspect}" if require if require_relative dir = File.dirname(caller_locations[0,1][0].absolute_path) full_path = File.expand_path(require_relative, dir) @@ -379,6 +434,8 @@ eom end assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{shim_value} + #{shim_join} #{require} previous_verbose = $VERBOSE $VERBOSE = nil @@ -489,19 +546,15 @@ eom case expected when String assert = :assert_equal - when Regexp - assert = :assert_match else - raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" + assert_respond_to(expected, :===) + assert = :assert_match end - ex = m = nil - EnvUtil.with_default_internal(expected.encoding) do - ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do - yield - end - m = ex.message + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield end + m = ex.message msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} if assert == :assert_equal @@ -514,6 +567,43 @@ eom ex end + # :call-seq: + # assert_raise_kind_of(*args, &block) + # + #Tests if the given block raises one of the given exceptions or + #sub exceptions of the given exceptions. If the last argument + #is a String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise SystemCallErr do + # Dir.chdir(__FILE__) #Raises Errno::ENOTDIR, so assertion succeeds + # end + def assert_raise_kind_of(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + raise e unless exp.include? Test::Unit::PendedError + rescue *exp => e + pass + rescue Exception => e + flunk(message(msg) {"#{mu_pp(exp)} family exception expected, not #{mu_pp(e)}"}) + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} family expected but nothing was raised"}) + end + end + e + end + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: # :call-seq: @@ -633,7 +723,7 @@ eom def assert_warning(pat, msg = nil) result = nil - stderr = EnvUtil.with_default_internal(pat.encoding) { + stderr = EnvUtil.with_default_internal(of: pat) { EnvUtil.verbose_warning { result = yield } @@ -647,17 +737,15 @@ eom assert_warning(*args) {$VERBOSE = false; yield} end - def assert_deprecated_warning(mesg = /deprecated/) + def assert_deprecated_warning(mesg = /deprecated/, &block) assert_warning(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end - def assert_deprecated_warn(mesg = /deprecated/) + def assert_deprecated_warn(mesg = /deprecated/, &block) assert_warn(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end @@ -794,6 +882,9 @@ eom rescue # Constants may be defined but not implemented, e.g., mingw. else + unless Process.clock_getres(clk) < 1.0e-03 + next # needs msec precision + end PERFORMANCE_CLOCK = clk end end diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb new file mode 100644 index 0000000000..56b420a546 --- /dev/null +++ b/tool/lib/dump.gdb @@ -0,0 +1,17 @@ +set height 0 +set width 0 +set confirm off + +echo \n>>> Threads\n\n +info threads + +echo \n>>> Machine level backtrace\n\n +thread apply all info stack full + +echo \n>>> Dump Ruby level backtrace (if possible)\n\n +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +echo ">>> Finish\n" +detach +quit diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb new file mode 100644 index 0000000000..ed9cb89010 --- /dev/null +++ b/tool/lib/dump.lldb @@ -0,0 +1,13 @@ +script print("\n>>> Threads\n\n") +thread list + +script print("\n>>> Machine level backtrace\n\n") +thread backtrace all + +script print("\n>>> Dump Ruby level backtrace (if possible)\n\n") +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +script print(">>> Finish\n") +detach +quit diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index 7b8aa99a39..6089605056 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -79,6 +79,72 @@ module EnvUtil end module_function :timeout + class Debugger + @list = [] + + attr_accessor :name + + def self.register(name, &block) + @list << new(name, &block) + end + + def initialize(name, &block) + @name = name + instance_eval(&block) + end + + def usable?; false; end + + def start(pid, *args) end + + def dump(pid, timeout: 60, reprieve: timeout&.div(4)) + dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err) + rescue Errno::ENOENT + return + else + return unless dpid + [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig| + begin + return EnvUtil.timeout(t) {Process.wait(dpid)} + rescue Timeout::Error + Process.kill(sig, dpid) + end + end + true + end + + # sudo -n: --non-interactive + PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM) + + def spawn(*args, **opts) + super(*PRECOMMAND, *args, **opts) + end + + register("gdb") do + class << self + def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end + def start(pid, *args, **opts) + spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts) + end + def command_file(file) "--command=#{file}"; end + end + end + + register("lldb") do + class << self + def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end + def start(pid, *args, **opts) + spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts) + end + def command_file(file) ["--source", file]; end + end + end + + def self.search + @debugger ||= @list.find(&:usable?) + end + end + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) reprieve = apply_timeout_scale(reprieve) if reprieve @@ -94,17 +160,12 @@ module EnvUtil pgroup = pid end - lldb = true if /darwin/ =~ RUBY_PLATFORM - + dumped = false while signal = signals.shift - if lldb and [:ABRT, :KILL].include?(signal) - lldb = false - # sudo -n: --non-interactive - # lldb -p: attach - # -o: run command - system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit]) - true + if !dumped and [:ABRT, :KILL].include?(signal) + Debugger.search&.dump(pid) + dumped = true end begin @@ -166,8 +227,8 @@ module EnvUtil args = [args] if args.kind_of?(String) # use the same parser as current ruby - if args.none? { |arg| arg.start_with?("--parser=") } - current_parser = RUBY_DESCRIPTION =~ /prism/i ? "prism" : "parse.y" + if (args.none? { |arg| arg.start_with?("--parser=") } and + /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read)) args = ["--parser=#{current_parser}"] + args end pid = spawn(child_env, *precommand, rubybin, *args, opt) @@ -217,6 +278,12 @@ module EnvUtil end module_function :invoke_ruby + def current_parser + features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}] + features&.split&.include?("+PRISM") ? "prism" : "parse.y" + end + module_function :current_parser + def verbose_warning class << (stderr = "".dup) alias write concat @@ -233,6 +300,21 @@ module EnvUtil end module_function :verbose_warning + if defined?(Warning.[]=) + def deprecation_warning + previous_deprecated = Warning[:deprecated] + Warning[:deprecated] = true + yield + ensure + Warning[:deprecated] = previous_deprecated + end + else + def deprecation_warning + yield + end + end + module_function :deprecation_warning + def default_warning $VERBOSE = false yield @@ -279,7 +361,8 @@ module EnvUtil end module_function :without_gc - def with_default_external(enc) + def with_default_external(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_external = enc } yield ensure @@ -287,7 +370,8 @@ module EnvUtil end module_function :with_default_external - def with_default_internal(enc) + def with_default_internal(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_internal = enc } yield ensure diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb index 70a2469db2..1893e07657 100644 --- a/tool/lib/gem_env.rb +++ b/tool/lib/gem_env.rb @@ -1,2 +1 @@ -ENV['GEM_HOME'] = gem_home = File.expand_path('.bundle') -ENV['GEM_PATH'] = [gem_home, File.expand_path('../../../.bundle', __FILE__)].uniq.join(File::PATH_SEPARATOR) +ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__) diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb index 69aeb2c254..69df9a64b8 100644 --- a/tool/lib/leakchecker.rb +++ b/tool/lib/leakchecker.rb @@ -112,7 +112,7 @@ class LeakChecker } unless fd_leaked.empty? unless @@try_lsof == false - @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + @@try_lsof |= system(*%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) end end h.each {|fd, list| diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb index 60632523a8..429e5f6a1d 100644 --- a/tool/lib/memory_status.rb +++ b/tool/lib/memory_status.rb @@ -20,48 +20,68 @@ module Memory data.scan(pat) {|k, v| keys << k.downcase.intern} when /mswin|mingw/ =~ RUBY_PLATFORM - require 'fiddle/import' - require 'fiddle/types' - - module Win32 - extend Fiddle::Importer - dlload "kernel32.dll", "psapi.dll" - include Fiddle::Win32Types - typealias "SIZE_T", "size_t" - - PROCESS_MEMORY_COUNTERS = struct [ - "DWORD cb", - "DWORD PageFaultCount", - "SIZE_T PeakWorkingSetSize", - "SIZE_T WorkingSetSize", - "SIZE_T QuotaPeakPagedPoolUsage", - "SIZE_T QuotaPagedPoolUsage", - "SIZE_T QuotaPeakNonPagedPoolUsage", - "SIZE_T QuotaNonPagedPoolUsage", - "SIZE_T PagefileUsage", - "SIZE_T PeakPagefileUsage", - ] - - typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" - - extern "HANDLE GetCurrentProcess()", :stdcall - extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall - - module_function - def memory_info - size = PROCESS_MEMORY_COUNTERS.size - data = PROCESS_MEMORY_COUNTERS.malloc - data.cb = size - data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + keys.push(:size, :rss, :peak) + + begin + require 'fiddle/import' + require 'fiddle/types' + rescue LoadError + # Fallback to PowerShell command to get memory information for current process + def self.read_status + cmd = [ + "powershell.exe", "-NoProfile", "-Command", + "Get-Process -Id #{$$} | " \ + "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }" + ] + + IO.popen(cmd, "r", err: [:child, :out]) do |out| + if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read + yield :size, $1.to_i + yield :rss, $2.to_i + yield :peak, $3.to_i + end + end + end + else + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end end - end - keys.push(:size, :rss, :peak) - def self.read_status - if info = Win32.memory_info - yield :size, info.PagefileUsage - yield :rss, info.WorkingSetSize - yield :peak, info.PeakWorkingSetSize + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end end end when (require_relative 'find_executable' diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb new file mode 100644 index 0000000000..7b889163b0 --- /dev/null +++ b/tool/lib/test/jobserver.rb @@ -0,0 +1,47 @@ +module Test + module JobServer + end +end + +class << Test::JobServer + def connect(makeflags = ENV["MAKEFLAGS"]) + return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags + begin + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end + rescue + r&.close + nil + else + return r, w + end + end + + def acquire_possible(r, w, max) + return unless tokens = r.read_nonblock(max - 1, exception: false) + if (jobs = tokens.size) > 0 + jobserver, w = w, nil + at_exit do + jobserver.print(tokens) + jobserver.close + end + end + return jobs + 1 + rescue Errno::EBADF + ensure + r&.close + w&.close + end + + def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"]) + if max > 1 and (r, w = connect(makeflags)) + acquire_possible(r, w, max) + end + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 30f30df62e..2663b7b76a 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -19,6 +19,7 @@ require_relative '../envutil' require_relative '../colorize' require_relative '../leakchecker' require_relative '../test/unit/testcase' +require_relative '../test/jobserver' require 'optparse' # See Test::Unit @@ -249,6 +250,8 @@ module Test end module Parallel # :nodoc: all + attr_accessor :prefix + def process_args(args = []) return @options if @options options = super @@ -260,27 +263,8 @@ module Test def non_options(files, options) @jobserver = nil - makeflags = ENV.delete("MAKEFLAGS") - if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags - begin - if fifo = $3 - fifo.gsub!(/\\(?=.)/, '') - r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) - w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) - else - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) - end - rescue - r.close if r - nil - else - r.close_on_exec = true - w.close_on_exec = true - @jobserver = [r, w] - options[:parallel] ||= 256 # number of tokens to acquire first - end + if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS")) + options[:parallel] ||= 256 # number of tokens to acquire first end @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) super @@ -298,7 +282,7 @@ module Test opts.separator "parallel test options:" - options[:retry] = true + options[:retry] = false opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| options[:testing] = true & t # For testing @@ -370,8 +354,12 @@ module Test @io.puts(*args) end - def run(task,type) - @file = File.basename(task, ".rb") + def run(task, type, base = nil) + if base + @file = task.delete_prefix(base).chomp(".rb") + else + @file = File.basename(task, ".rb") + end @real_file = task begin puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" @@ -415,6 +403,7 @@ module Test end def kill + EnvUtil::Debugger.search&.dump(@pid) signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV Process.kill(signal, @pid) warn "worker #{to_s} does not respond; #{signal} is sent" @@ -597,7 +586,7 @@ module Test worker.quit worker = launch_worker end - worker.run(task, type) + worker.run(task, type, (@prefix unless @options[:job_status] == :replace)) @test_count += 1 jobs_status(worker) @@ -1292,10 +1281,15 @@ module Test parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| options[:repeat_count] = n end + options[:keep_repeating] = false + parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n| + options[:keep_repeating] = true + end end def _run_anything(type) @repeat_count = @options[:repeat_count] + @keep_repeating = @options[:keep_repeating] super end end @@ -1617,7 +1611,7 @@ module Test [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, t, @test_count.fdiv(t), @assertion_count.fdiv(t)] end while @repeat_count && @@current_repeat_count < @repeat_count && - report.empty? && failures.zero? && errors.zero? + (@keep_repeating || report.empty? && failures.zero? && errors.zero?) output.sync = old_sync if sync @@ -1856,6 +1850,7 @@ module Test @force_standalone = force_standalone @runner = Runner.new do |files, options| base = options[:base_directory] ||= default_dir + @runner.prefix = base ? (base + "/") : nil files << default_dir if files.empty? and default_dir @to_run = files yield self if block_given? diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index fe3351107e..19581fc3ab 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -798,33 +798,37 @@ EOT MIN_MEASURABLE = 1.0 / MIN_HZ def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) - require 'benchmark' - wait = EnvUtil.apply_timeout_scale(wait) if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" end - tms = Benchmark.measure(msg || '') do - if stop - th = Thread.start {sleep wait; stop.call} - yield - th.join - else - begin - Timeout.timeout(wait) {yield} - rescue Timeout::Error - end + + t0, r0 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error end end - max = pct * tms.real + t1, r1 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + total = t1.utime - t0.utime + t1.stime - t0.stime + t1.cutime - t0.cutime + t1.cstime - t0.cstime + real = r1 - r0 + + max = pct * real min_measurable = MIN_MEASURABLE min_measurable *= 1.30 # add a little (30%) to account for misc. overheads if max < min_measurable max = min_measurable end - assert_operator tms.total, :<=, max, msg + assert_operator total, :<=, max, msg end def assert_is_minus_zero(f) diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb index 3894f9c8e8..26c9763c13 100644 --- a/tool/lib/vcs.rb +++ b/tool/lib/vcs.rb @@ -51,23 +51,9 @@ module DebugPOpen end using DebugPOpen module DebugSystem - def system(*args) + def system(*args, exception: true, **opts) VCS.dump(args, "args: ") if $DEBUG - exception = false - opts = Hash.try_convert(args[-1]) - if RUBY_VERSION >= "2.6" - unless opts - opts = {} - args << opts - end - exception = opts.fetch(:exception) {opts[:exception] = true} - elsif opts - exception = opts.delete(:exception) {true} - args.pop if opts.empty? - end - ret = super(*args) - raise "Command failed with status (#$?): #{args[0]}" if exception and !ret - ret + super(*args, exception: exception, **opts) end end @@ -271,155 +257,6 @@ class VCS code end - class SVN < self - register(".svn") - COMMAND = ENV['SVN'] || 'svn' - - def self.revision_name(rev) - "r#{rev}" - end - - def _get_revisions(path, srcdir = nil) - if srcdir and self.class.local_path?(path) - path = File.join(srcdir, path) - end - if srcdir - info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") - info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s - end - info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") - _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) - modified = info_xml[/<date>([^<>]*)/, 1] - branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] - [Integer(last), Integer(changed), modified, branch] - end - - def self.search_root(path) - return unless local_path?(path) - parent = File.realpath(path) - begin - parent = File.dirname(wkdir = parent) - return wkdir if File.directory?(wkdir + "/.svn") - end until parent == wkdir - end - - def get_info - @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") - end - - def url - @url ||= begin - url = get_info[/<root>(.*)<\/root>/, 1] - @url = URI.parse(url+"/") if url - end - end - - def wcroot - @wcroot ||= begin - info = get_info - @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] - @wcroot ||= self.class.search_root(@srcdir) - end - end - - def branch(name) - return trunk if name == "trunk" - url + "branches/#{name}" - end - - def tag(name) - url + "tags/#{name}" - end - - def trunk - url + "trunk" - end - alias master trunk - - def branch_list(pat) - IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| - f.each do |line| - line.chomp! - line.chomp!('/') - yield(line) if File.fnmatch?(pat, line) - end - end - end - - def grep(pat, tag, *files, &block) - cmd = %W"#{COMMAND} cat" - files.map! {|n| File.join(tag, n)} if tag - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files]) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end - - def export(revision, url, dir, keep_temp = false) - if @srcdir and (rootdir = wcroot) - srcdir = File.realpath(@srcdir) - rootdir << "/" - if srcdir.start_with?(rootdir) - subdir = srcdir[rootdir.size..-1] - subdir = nil if subdir.empty? - FileUtils.mkdir_p(svndir = dir+"/.svn") - FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) - system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false - FileUtils.rm_rf(svndir) unless keep_temp - if subdir - tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") - File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") - FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) - begin - Dir.rmdir("#{dir}/#{subdir}") - end until (subdir = File.dirname(subdir)) == '.' - FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) - Dir.rmdir(tmpdir) - end - return self - end - end - IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| - pipe.each {|line| /^A/ =~ line or yield line} - end - self if $?.success? - end - - def after_export(dir) - super - FileUtils.rm_rf(dir+"/.svn") - end - - def branch_beginning(url) - # `--limit` of svn-log is useless in this case, because it is - # applied before `--search`. - rev = IO.pread(%W[ #{COMMAND} log --xml - --search=matz --search-and=has\ started - -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] - rev.to_i if rev - end - - def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path) - range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') - IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -r#{range} #{url}") do |r| - IO.copy_stream(r, path) - end - end - - def commit - args = %W"#{COMMAND} commit" - if dryrun? - VCS.dump(args, "commit: ") - return true - end - system(*args) - end - end - class GIT < self register(".git") do |path, dir| SAFE_DIRECTORIES ||= @@ -479,7 +316,7 @@ class VCS last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref, err: w]]).rstrip w.close unless r.eof? - raise "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" + raise VCS::NotFoundError, "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" end end log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) @@ -533,15 +370,6 @@ class VCS rev[0, 10] end - def revision_handler(rev) - case rev - when Integer - SVN - else - super - end - end - def without_gitconfig envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| h[v] = ENV.delete(v) @@ -616,32 +444,26 @@ class VCS def export(revision, url, dir, keep_temp = false) system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return - (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir)) + GIT.new(File.expand_path(dir)) end def branch_beginning(url) - cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H + year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i + cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25 --author=matz --committer=matz --grep=started\\.$ - #{url.to_str} -- version.h include/ruby/version.h]) + #{url} -- version.h include/ruby/version.h])[/.*/] end - def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil) - svn = nil + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true) from, to = [from, to].map do |rev| rev or next - if Integer === rev - svn = true - rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -n1 --format=format:%H" << - "--grep=^ *git-svn-id: .*@#{rev} ") - end rev unless rev.empty? end - unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from)) + unless from&.match?(/./) or (from = branch_beginning(url))&.match?(/./) warn "no starting commit found", uplevel: 1 from = nil end - if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", chdir: @srcdir, exception: false) system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", chdir: @srcdir, exception: false) @@ -654,22 +476,18 @@ class VCS else arg = ["--since=25 Dec 00:00:00", to] end - writer = - if svn - format_changelog_as_svn(path, arg) - else - if base_url == true - remote, = upstream - if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) - remote.chomp! - # hack to redirect git.r-l.o to github - remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') - remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') - end - base_url = remote - end - format_changelog(path, arg, base_url) + if base_url == true + env = CHANGELOG_ENV + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') end + base_url = remote + end + writer = changelog_formatter(path, arg, base_url) if !path or path == '-' writer[$stdout] else @@ -678,9 +496,10 @@ class VCS end LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' + CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} - def format_changelog(path, arg, base_url = nil) - env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + def changelog_formatter(path, arg, base_url = nil) + env = CHANGELOG_ENV cmd = %W[#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] @@ -692,17 +511,32 @@ class VCS cmd << date cmd.concat(arg) proc do |w| - w.print "-*- coding: utf-8 -*-\n\n" - w.print "base-url = #{base_url}\n\n" if base_url + w.print "-*- coding: utf-8 -*-\n" + w.print "\n""base-url = #{base_url}\n" if base_url + + begin + ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true) + .grep_v(/^ *(?:#|$)/) + .to_h {|v| [v, true]} + ignore_revs = nil if ignore_revs.empty? + rescue Errno::ENOENT + end + cmd_pipe(env, cmd, chdir: @srcdir) do |r| - while s = r.gets("\ncommit ") + r.gets(sep = "commit ") + sep = "\n" + sep + while s = r.gets(sep, chomp: true) h, s = s.split(/^$/, 2) + if ignore_revs&.key?(h[/\A\h{40}/]) + next + end next if /^Author: *dependabot\[bot\]/ =~ h h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') fix = $1 + next if /\A *skip\Z/ =~ fix s = s.lines fix.each_line do |x| next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') @@ -739,7 +573,7 @@ class VCS next end end - message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] + message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] from = [1, n-2].max to = [s.size-1, n+2].min s.each_with_index do |e, i| @@ -761,6 +595,10 @@ class VCS s = s.join('') end + s.gsub!(%r[(?!<\w)([-\w]+/[-\w]+)(?:@(\h{8,40})|#(\d{5,}))\b]) do + path = defined?($2) ? "commit/#{$2}" : "pull/#{$3}" + "[#$&](https://github.com/#{$1}/#{path})" + end if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s issue = "#{$1}pull/" s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} @@ -768,28 +606,7 @@ class VCS s.gsub!(/ +\n/, "\n") s.sub!(/^Notes:/, ' \&') - w.print h, s - end - end - end - end - - def format_changelog_as_svn(path, arg) - cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B" - cmd.concat(arg) - proc do |w| - sep = "-"*72 + "\n" - w.print sep - cmd_pipe(cmd) do |r| - while s = r.gets("\0") - s.chomp!("\0") - author, time, s = s.split("\n", 3) - s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '') - rev = $1 - time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)") - lines = s.count("\n") + 1 - lines = "#{lines} line#{lines == 1 ? '' : 's'}" - w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep + w.print sep, h, s end end end @@ -826,46 +643,6 @@ class VCS end end - class GITSVN < GIT - def self.revision_name(rev) - SVN.revision_name(rev) - end - - def last_changed_revision - rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] - com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp - return rev, com - end - - def commit(opts = {}) - rev, com = last_changed_revision - head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp - - commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") - commits.each_with_index do |l, i| - r, a, c = l.split(' ') - dcommit = [COMMAND, "svn", "dcommit"] - dcommit.insert(-2, "-n") if dryrun? - dcommit << "--add-author-from" unless a == c - dcommit << r - system(*dcommit) or return false - system(COMMAND, "checkout", head) or return false - system(COMMAND, "rebase") or return false - end - - if rev - old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] - old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] - 3.times do - sleep 2 - system(*%W"#{COMMAND} pull --no-edit --rebase") - break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) - end - end - true - end - end - class Null < self def get_revisions(path, srcdir = nil) @modified ||= Time.now - 10 diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md index 3c3cb1df45..f71118a913 100644 --- a/tool/lrama/NEWS.md +++ b/tool/lrama/NEWS.md @@ -1,5 +1,400 @@ # NEWS for Lrama +## Lrama 0.7.1 (2025-12-24) + +### Optimize IELR + +Optimized performance to a level that allows for IELR testing in practical applications. + +https://github.com/ruby/lrama/pull/595 +https://github.com/ruby/lrama/pull/605 +https://github.com/ruby/lrama/pull/685 +https://github.com/ruby/lrama/pull/700 + +### Introduce counterexamples timeout + +Counterexample searches can sometimes take a long time, so we've added a timeout to abort the process after a set period. The current limits are: + +* 10 seconds per case +* 120 seconds total (cumulative) + +Please note that these are hard-coded and cannot be modified by the user in the current version. + +https://github.com/ruby/lrama/pull/623 + +### Optimize Counterexamples + +Optimized counterexample search performance. + +https://github.com/ruby/lrama/pull/607 +https://github.com/ruby/lrama/pull/610 +https://github.com/ruby/lrama/pull/614 +https://github.com/ruby/lrama/pull/622 +https://github.com/ruby/lrama/pull/627 +https://github.com/ruby/lrama/pull/629 +https://github.com/ruby/lrama/pull/659 + +### Support parameterized rule's arguments include inline + +Allow to use %inline directive with Parameterized rules arguments. When an inline rule is used as an argument to a Parameterized rule, it expands inline at the point of use. + +```yacc +%rule %inline op : '+' + | '-' + ; +%% +operation : op? + ; +``` + +This expands to: + +```yacc +operation : /* empty */ + | '+' + | '-' + ; +``` + +https://github.com/ruby/lrama/pull/637 + +### Render conflicts of each state on output file + +Added token information for conflicts in the output file. +These information are useful when a state has many actions. + +``` +State 1 + + 4 class: keyword_class • tSTRING "end" + 5 $@1: ε • [tSTRING] + 7 class: keyword_class • $@1 tSTRING '!' "end" $@2 + 8 $@3: ε • [tSTRING] + 10 class: keyword_class • $@3 tSTRING '?' "end" $@4 + + Conflict on tSTRING. shift/reduce($@1) + Conflict on tSTRING. shift/reduce($@3) + Conflict on tSTRING. reduce($@1)/reduce($@3) + + tSTRING shift, and go to state 6 + + tSTRING reduce using rule 5 ($@1) + tSTRING reduce using rule 8 ($@3) + + $@1 go to state 7 + $@3 go to state 8 +``` + +https://github.com/ruby/lrama/pull/541 + +### Render the origin of conflicted tokens on output file + +For example, for the grammar file like below: + +``` +%% + +program: expr + ; + +expr: expr '+' expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where `"plus"` (`'+'`) look ahead tokens come from: + +``` +State 6 + + 2 expr: expr • "plus" expr + 2 | expr "plus" expr • ["end of file", "plus"] + + Conflict on "plus". shift/reduce(expr) + "plus" comes from state 0 goto by expr + "plus" comes from state 5 goto by expr +``` + +state 0 and state 5 look like below: + +``` +State 0 + + 0 $accept: • program "end of file" + 1 program: • expr + 2 expr: • expr "plus" expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + program go to state 2 + expr go to state 3 + +State 5 + + 2 expr: • expr "plus" expr + 2 | expr "plus" • expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + expr go to state 6 +``` + +https://github.com/ruby/lrama/pull/726 + +### Render precedences usage information on output file + +For example, for the grammar file like below: + +``` +%left tPLUS +%right tUPLUS + +%% + +program: expr ; + +expr: tUPLUS expr + | expr tPLUS expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where these precedences are used to resolve conflicts: + +``` +Precedences + precedence on "unary+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + precedence on "+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + state 8. Conflict between reduce by "expr -> expr tPLUS expr" and shift "+" resolved as reduce (%left "+"). +``` + +https://github.com/ruby/lrama/pull/741 + +### Add support for reporting Rule Usage Frequency + +Support to report rule usage frequency statistics for analyzing grammar characteristics. +Run `exe/lrama --report=rules` to show how frequently each terminal and non-terminal symbol is used in the grammar rules. + +```console +$ exe/lrama --report=rules sample/calc.y +Rule Usage Frequency + 0 tSTRING (4 times) + 1 keyword_class (3 times) + 2 keyword_end (3 times) + 3 '+' (2 times) + 4 string (2 times) + 5 string_1 (2 times) + 6 '!' (1 times) + 7 '-' (1 times) + 8 '?' (1 times) + 9 EOI (1 times) + 10 class (1 times) + 11 program (1 times) + 12 string_2 (1 times) + 13 strings_1 (1 times) + 14 strings_2 (1 times) + 15 tNUMBER (1 times) +``` + +This feature provides insights into the language characteristics by showing: +- Which symbols are most frequently used in the grammar +- The distribution of terminal and non-terminal usage +- Potential areas for grammar optimization or refactoring + +The frequency statistics help developers understand the grammar structure and can be useful for: +- Grammar complexity analysis +- Performance optimization hints +- Language design decisions +- Documentation and educational purposes + +https://github.com/ruby/lrama/pull/677 + +### Render Split States information on output file + +For example, for the grammar file like below: + +``` +%token a +%token b +%token c +%define lr.type ielr + +%precedence tLOWEST +%precedence a +%precedence tHIGHEST + +%% + +S: a A B a + | b A B b + ; + +A: a C D E + ; + +B: c + | // empty + ; + +C: D + ; + +D: a + ; + +E: a + | %prec tHIGHEST // empty + ; + +%% +``` + +Lrama generates output file which describes where which new states are created when IELR is enabled: + +``` +Split States + + State 19 is split from state 4 + State 20 is split from state 9 + State 21 is split from state 14 +``` + +https://github.com/ruby/lrama/pull/624 + +### Add ioption support to the Standard library + +Support `ioption` (inline option) rule, which is expanded inline without creating intermediate rules. + +Unlike the regular `option` rule that generates a separate rule, `ioption` directly expands at the point of use: + +```yacc +program: ioption(number) expr + +// Expanded inline to: + +program: expr + | number expr +``` + +This differs from the regular `option` which would generate: + +```yacc +program: option(number) expr + +// Expanded to: + +program: option_number expr +option_number: %empty + | number +``` + +The `ioption` rule provides more compact grammar generation by avoiding intermediate rule creation, which can be beneficial for reducing the parser's rule count and potentially improving performance. + +This feature is inspired by Menhir's standard library and maintains compatibility with [Menhir's `ioption` behavior](https://github.com/let-def/menhir/blob/e8ba7bef219acd355798072c42abbd11335ecf09/src/standard.mly#L33-L41). + +https://github.com/ruby/lrama/pull/666 + +### Syntax Diagrams + +Lrama provides an API for generating HTML syntax diagrams. These visual diagrams are highly useful as grammar development tools and can also serve as a form of automatic self-documentation. + + + +If you use syntax diagrams, you add `--diagram` option. + +```console +$ exe/lrama --diagram sample.y +``` + +https://github.com/ruby/lrama/pull/523 + +### Support `--profile` option + +You can profile parser generation process without modification for Lrama source code. +Currently `--profile=call-stack` and `--profile=memory` are supported. + +```console +$ exe/lrama --profile=call-stack sample/calc.y +``` + +Then "tmp/stackprof-cpu-myapp.dump" is generated. + +https://github.com/ruby/lrama/pull/525 + +### Add support Start-Symbol: `%start` + +https://github.com/ruby/lrama/pull/576 + +## Lrama 0.7.0 (2025-01-21) + +### [EXPERIMENTAL] Support the generation of the IELR(1) parser described in this paper + +Support the generation of the IELR(1) parser described in this paper. +https://www.sciencedirect.com/science/article/pii/S0167642309001191 + +If you use IELR(1) parser, you can write the following directive in your grammar file. + +```yacc +%define lr.type ielr +``` + +But, currently IELR(1) parser is experimental feature. If you find any bugs, please report it to us. Thank you. + +### Support `-t` option as same as `--debug` option + +Support to `-t` option as same as `--debug` option. +These options align with Bison behavior. So same as `--debug` option. + +### Trace only explicit rules + +Support to trace only explicit rules. +If you use `--trace=rules` option, it shows include mid-rule actions. If you want to show only explicit rules, you can use `--trace=only-explicit-rules` option. + +Example: + +```yacc +%{ +%} +%union { + int i; +} +%token <i> number +%type <i> program +%% +program : number { printf("%d", $1); } number { $$ = $1 + $3; } + ; +%% +``` + +Result of `--trace=rules`: + +```console +$ exe/lrama --trace=rules sample.y +Grammar rules: +$accept -> program YYEOF +$@1 -> ε +program -> number $@1 number +``` + +Result of `--trace=only-explicit-rules`: + +```console +$ exe/lrama --trace=explicit-rules sample.y +Grammar rules: +$accept -> program YYEOF +program -> number number +``` + ## Lrama 0.6.11 (2024-12-23) ### Add support for %type declarations using %nterm in Nonterminal Symbols @@ -37,9 +432,9 @@ nterm.y:6:7: symbol EOI redeclared as a nonterminal ## Lrama 0.6.10 (2024-09-11) -### Aliased Named References for actions of RHS in parameterizing rules +### Aliased Named References for actions of RHS in Parameterizing rules -Allow to use aliased named references for actions of RHS in parameterizing rules. +Allow to use aliased named references for actions of RHS in Parameterizing rules. ```yacc %rule sum(X, Y): X[summand] '+' Y[addend] { $$ = $summand + $addend } @@ -49,9 +444,9 @@ Allow to use aliased named references for actions of RHS in parameterizing rules https://github.com/ruby/lrama/pull/410 -### Named References for actions of RHS in parameterizing rules caller side +### Named References for actions of RHS in Parameterizing rules caller side -Allow to use named references for actions of RHS in parameterizing rules caller side. +Allow to use named references for actions of RHS in Parameterizing rules caller side. ```yacc opt_nl: '\n'?[nl] <str> { $$ = $nl; } @@ -60,9 +455,9 @@ opt_nl: '\n'?[nl] <str> { $$ = $nl; } https://github.com/ruby/lrama/pull/414 -### Widen the definable position of parameterizing rules +### Widen the definable position of Parameterizing rules -Allow to define parameterizing rules in the middle of the grammar. +Allow to define Parameterizing rules in the middle of the grammar. ```yacc %rule defined_option(X): /* empty */ @@ -126,15 +521,15 @@ Change to `%locations` directive not set by default. https://github.com/ruby/lrama/pull/446 -### Diagnostics report for parameterizing rules redefine +### Diagnostics report for parameterized rules redefine -Support to warning redefined parameterizing rules. -Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterizing rules. +Support to warning redefined parameterized rules. +Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterized rules. ```console $ exe/lrama -W sample/calc.y -parameterizing rule redefined: redefined_method(X) -parameterizing rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) ``` https://github.com/ruby/lrama/pull/448 @@ -148,9 +543,9 @@ https://github.com/ruby/lrama/pull/457 ## Lrama 0.6.9 (2024-05-02) -### Callee side tag specification of parameterizing rules +### Callee side tag specification of Parameterizing rules -Allow to specify tag on callee side of parameterizing rules. +Allow to specify tag on callee side of Parameterizing rules. ```yacc %union { @@ -161,9 +556,9 @@ Allow to specify tag on callee side of parameterizing rules. ; ``` -### Named References for actions of RHS in parameterizing rules +### Named References for actions of RHS in Parameterizing rules -Allow to use named references for actions of RHS in parameterizing rules. +Allow to use named references for actions of RHS in Parameterizing rules. ```yacc %rule option(number): /* empty */ @@ -173,9 +568,9 @@ Allow to use named references for actions of RHS in parameterizing rules. ## Lrama 0.6.8 (2024-04-29) -### Nested parameterizing rules with tag +### Nested Parameterizing rules with tag -Allow to nested parameterizing rules with tag. +Allow to nested Parameterizing rules with tag. ```yacc %union { @@ -197,9 +592,9 @@ Allow to nested parameterizing rules with tag. ## Lrama 0.6.7 (2024-04-28) -### RHS of user defined parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. +### RHS of user defined Parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. -User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined parameterizing rules. +User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined Parameterizing rules. ``` %rule with_word_seps(X): /* empty */ @@ -259,7 +654,7 @@ expr : number { $$ = $1; } ### Typed Midrule Actions -User can specify the type of mid rule action by tag (`<bar>`) instead of specifying it with in an action. +User can specify the type of mid-rule action by tag (`<bar>`) instead of specifying it with in an action. ```yacc primary: k_case expr_value terms? @@ -334,7 +729,7 @@ https://github.com/ruby/lrama/pull/382 User can set codes for freeing semantic value resources by using `%destructor`. In general, these resources are freed by actions or after parsing. -However if syntax error happens in parsing, these codes may not be executed. +However, if syntax error happens in parsing, these codes may not be executed. Codes associated to `%destructor` are executed when semantic value is popped from the stack by an error. ```yacc @@ -372,7 +767,7 @@ Lrama introduces two features to support another semantic value stack by parser 1. Callback entry points User can emulate semantic value stack by these callbacks. -Lrama provides these five callbacks. Registered functions are called when each event happen. For example %after-shift function is called when shift happens on original semantic value stack. +Lrama provides these five callbacks. Registered functions are called when each event happens. For example %after-shift function is called when shift happens on original semantic value stack. * `%after-shift` function_name * `%before-reduce` function_name @@ -400,15 +795,15 @@ https://github.com/ruby/lrama/pull/367 ### %no-stdlib directive If `%no-stdlib` directive is set, Lrama doesn't load Lrama standard library for -parameterizing rules, stdlib.y. +parameterized rules, stdlib.y. https://github.com/ruby/lrama/pull/344 ## Lrama 0.6.1 (2024-01-13) -### Nested parameterizing rules +### Nested Parameterizing rules -Allow to pass an instantiated rule to other parameterizing rules. +Allow to pass an instantiated rule to other Parameterizing rules. ```yacc %rule constant(X) : X @@ -425,7 +820,7 @@ program : option(constant(number)) // Nested rule %% ``` -Allow to use nested parameterizing rules when define parameterizing rules. +Allow to use nested Parameterizing rules when define Parameterizing rules. ```yacc %rule option(x) : /* empty */ @@ -450,9 +845,9 @@ https://github.com/ruby/lrama/pull/337 ## Lrama 0.6.0 (2023-12-25) -### User defined parameterizing rules +### User defined Parameterizing rules -Allow to define parameterizing rule by `%rule` directive. +Allow to define Parameterizing rule by `%rule` directive. ```yacc %rule pair(X, Y): X Y { $$ = $1 + $2; } @@ -472,7 +867,7 @@ https://github.com/ruby/lrama/pull/285 ## Lrama 0.5.11 (2023-12-02) -### Type specification of parameterizing rules +### Type specification of Parameterizing rules Allow to specify type of rules by specifying tag, `<i>` in below example. Tag is post-modification style. @@ -496,13 +891,13 @@ https://github.com/ruby/lrama/pull/272 ### Parameterizing rules (option, nonempty_list, list) -Support function call style parameterizing rules for `option`, `nonempty_list` and `list`. +Support function call style Parameterizing rules for `option`, `nonempty_list` and `list`. https://github.com/ruby/lrama/pull/197 ### Parameterizing rules (separated_list) -Support `separated_list` and `separated_nonempty_list` parameterizing rules. +Support `separated_list` and `separated_nonempty_list` Parameterizing rules. ```text program: separated_list(',', number) @@ -558,7 +953,7 @@ https://github.com/ruby/lrama/pull/181 ### Racc parser -Replace Lrama's parser from hand written parser to LR parser generated by Racc. +Replace Lrama's parser from handwritten parser to LR parser generated by Racc. Lrama uses `--embedded` option to generate LR parser because Racc is changed from default gem to bundled gem by Ruby 3.3 (https://github.com/ruby/lrama/pull/132). https://github.com/ruby/lrama/pull/62 diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama index 1aece5d141..710ac0cb96 100755 --- a/tool/lrama/exe/lrama +++ b/tool/lrama/exe/lrama @@ -4,4 +4,4 @@ $LOAD_PATH << File.join(__dir__, "../lib") require "lrama" -Lrama::Command.new.run(ARGV.dup) +Lrama::Command.new(ARGV.dup).run diff --git a/tool/lrama/lib/lrama.rb b/tool/lrama/lib/lrama.rb index fe2e05807c..56ba0044d4 100644 --- a/tool/lrama/lib/lrama.rb +++ b/tool/lrama/lib/lrama.rb @@ -4,19 +4,19 @@ require_relative "lrama/bitmap" require_relative "lrama/command" require_relative "lrama/context" require_relative "lrama/counterexamples" -require_relative "lrama/diagnostics" +require_relative "lrama/diagram" require_relative "lrama/digraph" +require_relative "lrama/erb" require_relative "lrama/grammar" -require_relative "lrama/grammar_validator" require_relative "lrama/lexer" require_relative "lrama/logger" require_relative "lrama/option_parser" require_relative "lrama/options" require_relative "lrama/output" require_relative "lrama/parser" -require_relative "lrama/report" +require_relative "lrama/reporter" require_relative "lrama/state" require_relative "lrama/states" -require_relative "lrama/states_reporter" -require_relative "lrama/trace_reporter" +require_relative "lrama/tracer" require_relative "lrama/version" +require_relative "lrama/warnings" diff --git a/tool/lrama/lib/lrama/bitmap.rb b/tool/lrama/lib/lrama/bitmap.rb index b2de0248c8..88b255b012 100644 --- a/tool/lrama/lib/lrama/bitmap.rb +++ b/tool/lrama/lib/lrama/bitmap.rb @@ -1,7 +1,12 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama module Bitmap + # @rbs! + # type bitmap = Integer + + # @rbs (Array[Integer] ary) -> bitmap def self.from_array(ary) bit = 0 @@ -12,20 +17,31 @@ module Lrama bit end + # @rbs (Integer int) -> bitmap + def self.from_integer(int) + 1 << int + end + + # @rbs (bitmap int) -> Array[Integer] def self.to_array(int) a = [] #: Array[Integer] i = 0 - while int > 0 do - if int & 1 == 1 + len = int.bit_length + while i < len do + if int[i] == 1 a << i end i += 1 - int >>= 1 end a end + + # @rbs (bitmap int, Integer size) -> Array[bool] + def self.to_bool_array(int, size) + Array.new(size) { |i| int[i] == 1 } + end end end diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb index 0095c1a119..17aad1a1c1 100644 --- a/tool/lrama/lib/lrama/command.rb +++ b/tool/lrama/lib/lrama/command.rb @@ -5,63 +5,116 @@ module Lrama LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__))) STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y') - def run(argv) - begin - options = OptionParser.new.parse(argv) - rescue => e - message = e.message - message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty? - abort message - end - - Report::Duration.enable if options.trace_opts[:time] + def initialize(argv) + @logger = Lrama::Logger.new + @options = OptionParser.parse(argv) + @tracer = Tracer.new(STDERR, **@options.trace_opts) + @reporter = Reporter.new(**@options.report_opts) + @warnings = Warnings.new(@logger, @options.warnings) + rescue => e + abort format_error_message(e.message) + end - text = options.y.read - options.y.close if options.y != STDIN - begin - grammar = Lrama::Parser.new(text, options.grammar_file, options.debug).parse - unless grammar.no_stdlib - stdlib_grammar = Lrama::Parser.new(File.read(STDLIB_FILE_PATH), STDLIB_FILE_PATH, options.debug).parse - grammar.insert_before_parameterizing_rules(stdlib_grammar.parameterizing_rules) + def run + Lrama::Reporter::Profile::CallStack.report(@options.profile_opts[:call_stack]) do + Lrama::Reporter::Profile::Memory.report(@options.profile_opts[:memory]) do + execute_command_workflow end - grammar.prepare - grammar.validate! - rescue => e - raise e if options.debug - message = e.message - message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty? - abort message end - states = Lrama::States.new(grammar, trace_state: (options.trace_opts[:automaton] || options.trace_opts[:closure])) + end + + private + + def execute_command_workflow + @tracer.enable_duration + text = read_input + grammar = build_grammar(text) + states, context = compute_status(grammar) + render_reports(states) if @options.report_file + @tracer.trace(grammar) + render_diagram(grammar) + render_output(context, grammar) + states.validate!(@logger) + @warnings.warn(grammar, states) + end + + def read_input + text = @options.y.read + @options.y.close unless @options.y == STDIN + text + end + + def build_grammar(text) + grammar = + Lrama::Parser.new(text, @options.grammar_file, @options.debug, @options.locations, @options.define).parse + merge_stdlib(grammar) + prepare_grammar(grammar) + grammar + rescue => e + raise e if @options.debug + abort format_error_message(e.message) + end + + def format_error_message(message) + return message unless Exception.to_tty? + + message.gsub(/.+/, "\e[1m\\&\e[m") + end + + def merge_stdlib(grammar) + return if grammar.no_stdlib + + stdlib_text = File.read(STDLIB_FILE_PATH) + stdlib_grammar = Lrama::Parser.new( + stdlib_text, + STDLIB_FILE_PATH, + @options.debug, + @options.locations, + @options.define, + ).parse + + grammar.prepend_parameterized_rules(stdlib_grammar.parameterized_rules) + end + + def prepare_grammar(grammar) + grammar.prepare + grammar.validate! + end + + def compute_status(grammar) + states = Lrama::States.new(grammar, @tracer) states.compute - context = Lrama::Context.new(states) + states.compute_ielr if grammar.ielr_defined? + [states, Lrama::Context.new(states)] + end - if options.report_file - reporter = Lrama::StatesReporter.new(states) - File.open(options.report_file, "w+") do |f| - reporter.report(f, **options.report_opts) - end + def render_reports(states) + File.open(@options.report_file, "w+") do |f| + @reporter.report(f, states) end + end - reporter = Lrama::TraceReporter.new(grammar) - reporter.report(**options.trace_opts) + def render_diagram(grammar) + return unless @options.diagram - File.open(options.outfile, "w+") do |f| + File.open(@options.diagram_file, "w+") do |f| + Lrama::Diagram.render(out: f, grammar: grammar) + end + end + + def render_output(context, grammar) + File.open(@options.outfile, "w+") do |f| Lrama::Output.new( out: f, - output_file_path: options.outfile, - template_name: options.skeleton, - grammar_file_path: options.grammar_file, - header_file_path: options.header_file, + output_file_path: @options.outfile, + template_name: @options.skeleton, + grammar_file_path: @options.grammar_file, + header_file_path: @options.header_file, context: context, grammar: grammar, - error_recovery: options.error_recovery, + error_recovery: @options.error_recovery, ).render end - - logger = Lrama::Logger.new - exit false unless Lrama::GrammarValidator.new(grammar, states, logger).valid? - Lrama::Diagnostics.new(grammar, states, logger).run(options.diagnostic) end end end diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb index 9f406f8de0..eb068c1b9e 100644 --- a/tool/lrama/lib/lrama/context.rb +++ b/tool/lrama/lib/lrama/context.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -require_relative "report/duration" +require_relative "tracer/duration" module Lrama # This is passed to a template class Context - include Report::Duration + include Tracer::Duration ErrorActionNumber = -Float::INFINITY BaseMin = -Float::INFINITY @@ -231,8 +231,8 @@ module Lrama end # Shift is selected when S/R conflict exists. - state.selected_term_transitions.each do |shift, next_state| - actions[shift.next_sym.number] = next_state.id + state.selected_term_transitions.each do |shift| + actions[shift.next_sym.number] = shift.to_state.id end state.resolved_conflicts.select do |conflict| @@ -292,18 +292,18 @@ module Lrama # of a default nterm transition destination. @yydefgoto = Array.new(@states.nterms.count, 0) # Mapping from nterm to next_states - nterm_to_next_states = {} + nterm_to_to_states = {} @states.states.each do |state| - state.nterm_transitions.each do |shift, next_state| - key = shift.next_sym - nterm_to_next_states[key] ||= [] - nterm_to_next_states[key] << [state, next_state] # [from_state, to_state] + state.nterm_transitions.each do |goto| + key = goto.next_sym + nterm_to_to_states[key] ||= [] + nterm_to_to_states[key] << [state, goto.to_state] # [from_state, to_state] end end @states.nterms.each do |nterm| - if (states = nterm_to_next_states[nterm]) + if (states = nterm_to_to_states[nterm]) default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first default_goto = default_state.id not_default_gotos = [] @@ -417,27 +417,25 @@ module Lrama res = lowzero - froms_and_tos.first[0] + # Find the smallest `res` such that `@table[res + from]` is empty for all `from` in `froms_and_tos` while true do - ok = true + advanced = false - froms_and_tos.each do |from, to| - loc = res + from - - if @table[loc] - # If the cell of table is set, can not use the cell. - ok = false - break - end + while used_res[res] + res += 1 + advanced = true end - if ok && used_res[res] - ok = false + froms_and_tos.each do |from, to| + while @table[res + from] + res += 1 + advanced = true + end end - if ok + unless advanced + # no advance means that the current `res` satisfies the condition break - else - res += 1 end end diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb index ee2b5d5959..60d830d048 100644 --- a/tool/lrama/lib/lrama/counterexamples.rb +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -1,35 +1,64 @@ +# rbs_inline: enabled # frozen_string_literal: true require "set" +require "timeout" require_relative "counterexamples/derivation" require_relative "counterexamples/example" +require_relative "counterexamples/node" require_relative "counterexamples/path" -require_relative "counterexamples/production_path" -require_relative "counterexamples/start_path" require_relative "counterexamples/state_item" -require_relative "counterexamples/transition_path" require_relative "counterexamples/triple" module Lrama # See: https://www.cs.cornell.edu/andru/papers/cupex/cupex.pdf # 4. Constructing Nonunifying Counterexamples class Counterexamples - attr_reader :transitions, :productions - + PathSearchTimeLimit = 10 # 10 sec + CumulativeTimeLimit = 120 # 120 sec + + # @rbs! + # @states: States + # @iterate_count: Integer + # @total_duration: Float + # @exceed_cumulative_time_limit: bool + # @state_items: Hash[[State, State::Item], StateItem] + # @triples: Hash[Integer, Triple] + # @transitions: Hash[[StateItem, Grammar::Symbol], StateItem] + # @reverse_transitions: Hash[[StateItem, Grammar::Symbol], Set[StateItem]] + # @productions: Hash[StateItem, Set[StateItem]] + # @reverse_productions: Hash[[State, Grammar::Symbol], Set[StateItem]] # Grammar::Symbol is nterm + # @state_item_shift: Integer + + attr_reader :transitions #: Hash[[StateItem, Grammar::Symbol], StateItem] + attr_reader :productions #: Hash[StateItem, Set[StateItem]] + + # @rbs (States states) -> void def initialize(states) @states = states + @iterate_count = 0 + @total_duration = 0 + @exceed_cumulative_time_limit = false + @triples = {} + setup_state_items setup_transitions setup_productions end + # @rbs () -> "#<Counterexamples>" def to_s "#<Counterexamples>" end alias :inspect :to_s + # @rbs (State conflict_state) -> Array[Example] def compute(conflict_state) conflict_state.conflicts.flat_map do |conflict| + # Check cumulative time limit for not each path search method call but each conflict + # to avoid one of example's path to be nil. + next if @exceed_cumulative_time_limit + case conflict.type when :shift_reduce # @type var conflict: State::ShiftReduceConflict @@ -38,22 +67,50 @@ module Lrama # @type var conflict: State::ReduceReduceConflict reduce_reduce_examples(conflict_state, conflict) end + rescue Timeout::Error => e + STDERR.puts "Counterexamples calculation for state #{conflict_state.id} #{e.message} with #{@iterate_count} iteration" + increment_total_duration(PathSearchTimeLimit) + nil end.compact end private + # @rbs (State state, State::Item item) -> StateItem + def get_state_item(state, item) + @state_items[[state, item]] + end + + # For optimization, create all StateItem in advance + # and use them by fetching an instance from `@state_items`. + # Do not create new StateItem instance in the shortest path search process + # to avoid miss hash lookup. + # + # @rbs () -> void + def setup_state_items + @state_items = {} + count = 0 + + @states.states.each do |state| + state.items.each do |item| + @state_items[[state, item]] = StateItem.new(count, state, item) + count += 1 + end + end + + @state_item_shift = Math.log(count, 2).ceil + end + + # @rbs () -> void def setup_transitions - # Hash [StateItem, Symbol] => StateItem @transitions = {} - # Hash [StateItem, Symbol] => Set(StateItem) @reverse_transitions = {} @states.states.each do |src_state| trans = {} #: Hash[Grammar::Symbol, State] - src_state.transitions.each do |shift, next_state| - trans[shift.next_sym] = next_state + src_state.transitions.each do |transition| + trans[transition.next_sym] = transition.to_state end src_state.items.each do |src_item| @@ -63,8 +120,8 @@ module Lrama dest_state.kernels.each do |dest_item| next unless (src_item.rule == dest_item.rule) && (src_item.position + 1 == dest_item.position) - src_state_item = StateItem.new(src_state, src_item) - dest_state_item = StateItem.new(dest_state, dest_item) + src_state_item = get_state_item(src_state, src_item) + dest_state_item = get_state_item(dest_state, dest_item) @transitions[[src_state_item, sym]] = dest_state_item @@ -77,21 +134,20 @@ module Lrama end end + # @rbs () -> void def setup_productions - # Hash [StateItem] => Set(Item) @productions = {} - # Hash [State, Symbol] => Set(Item). Symbol is nterm @reverse_productions = {} @states.states.each do |state| - # LHS => Set(Item) - h = {} #: Hash[Grammar::Symbol, Set[States::Item]] + # Grammar::Symbol is LHS + h = {} #: Hash[Grammar::Symbol, Set[StateItem]] state.closure.each do |item| sym = item.lhs h[sym] ||= Set.new - h[sym] << item + h[sym] << get_state_item(state, item) end state.items.each do |item| @@ -99,101 +155,118 @@ module Lrama next if item.next_sym.term? sym = item.next_sym - state_item = StateItem.new(state, item) - # @type var key: [State, Grammar::Symbol] - key = [state, sym] - + state_item = get_state_item(state, item) @productions[state_item] = h[sym] + # @type var key: [State, Grammar::Symbol] + key = [state, sym] @reverse_productions[key] ||= Set.new - @reverse_productions[key] << item + @reverse_productions[key] << state_item end end end + # For optimization, use same Triple if it's already created. + # Do not create new Triple instance anywhere else + # to avoid miss hash lookup. + # + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> Triple + def get_triple(state_item, precise_lookahead_set) + key = (precise_lookahead_set << @state_item_shift) | state_item.id + @triples[key] ||= Triple.new(state_item, precise_lookahead_set) + end + + # @rbs (State conflict_state, State::ShiftReduceConflict conflict) -> Example def shift_reduce_example(conflict_state, conflict) conflict_symbol = conflict.symbols.first - # @type var shift_conflict_item: ::Lrama::States::Item + # @type var shift_conflict_item: ::Lrama::State::Item shift_conflict_item = conflict_state.items.find { |item| item.next_sym == conflict_symbol } - path2 = shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) - path1 = find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) + end + path1 = with_timeout("#find_shift_conflict_shortest_path:") do + find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + end Example.new(path1, path2, conflict, conflict_symbol, self) end + # @rbs (State conflict_state, State::ReduceReduceConflict conflict) -> Example def reduce_reduce_examples(conflict_state, conflict) conflict_symbol = conflict.symbols.first - path1 = shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) - path2 = shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + path1 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) + end + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + end Example.new(path1, path2, conflict, conflict_symbol, self) end - def find_shift_conflict_shortest_path(reduce_path, conflict_state, conflict_item) - state_items = find_shift_conflict_shortest_state_items(reduce_path, conflict_state, conflict_item) - build_paths_from_state_items(state_items) - end + # @rbs (Array[StateItem]? reduce_state_items, State conflict_state, State::Item conflict_item) -> Array[StateItem] + def find_shift_conflict_shortest_path(reduce_state_items, conflict_state, conflict_item) + time1 = Time.now.to_f + @iterate_count = 0 - def find_shift_conflict_shortest_state_items(reduce_path, conflict_state, conflict_item) - target_state_item = StateItem.new(conflict_state, conflict_item) + target_state_item = get_state_item(conflict_state, conflict_item) result = [target_state_item] - reversed_reduce_path = reduce_path.to_a.reverse + reversed_state_items = reduce_state_items.to_a.reverse # Index for state_item i = 0 - while (path = reversed_reduce_path[i]) + while (state_item = reversed_state_items[i]) # Index for prev_state_item j = i + 1 _j = j - while (prev_path = reversed_reduce_path[j]) - if prev_path.production? + while (prev_state_item = reversed_state_items[j]) + if prev_state_item.type == :production j += 1 else break end end - state_item = path.to - prev_state_item = prev_path&.to - if target_state_item == state_item || target_state_item.item.start_item? result.concat( - reversed_reduce_path[_j..-1] #: Array[StartPath|TransitionPath|ProductionPath] - .map(&:to)) + reversed_state_items[_j..-1] #: Array[StateItem] + ) break end - if target_state_item.item.beginning_of_rule? - queue = [] #: Array[Array[StateItem]] - queue << [target_state_item] + if target_state_item.type == :production + queue = [] #: Array[Node[StateItem]] + queue << Node.new(target_state_item, nil) # Find reverse production while (sis = queue.shift) - si = sis.last + @iterate_count += 1 + si = sis.elem # Reach to start state if si.item.start_item? - sis.shift - result.concat(sis) + a = Node.to_a(sis).reverse + a.shift + result.concat(a) target_state_item = si break end - if si.item.beginning_of_rule? + if si.type == :production # @type var key: [State, Grammar::Symbol] key = [si.state, si.item.lhs] - @reverse_productions[key].each do |item| - state_item = StateItem.new(si.state, item) - queue << (sis + [state_item]) + @reverse_productions[key].each do |state_item| + queue << Node.new(state_item, sis) end else # @type var key: [StateItem, Grammar::Symbol] key = [si, si.item.previous_sym] @reverse_transitions[key].each do |prev_target_state_item| next if prev_target_state_item.state != prev_state_item&.state - sis.shift - result.concat(sis) + a = Node.to_a(sis).reverse + a.shift + result.concat(a) result << prev_target_state_item target_state_item = prev_target_state_item i = j @@ -216,68 +289,106 @@ module Lrama end end + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "find_shift_conflict_shortest_path #{@iterate_count} iteration", duration) + end + result.reverse end - def build_paths_from_state_items(state_items) - state_items.zip([nil] + state_items).map do |si, prev_si| - case - when prev_si.nil? - StartPath.new(si) - when si.item.beginning_of_rule? - ProductionPath.new(prev_si, si) - else - TransitionPath.new(prev_si, si) + # @rbs (StateItem target) -> Set[StateItem] + def reachable_state_items(target) + result = Set.new + queue = [target] + + while (state_item = queue.shift) + next if result.include?(state_item) + result << state_item + + @reverse_transitions[[state_item, state_item.item.previous_sym]]&.each do |prev_state_item| + queue << prev_state_item + end + + if state_item.item.beginning_of_rule? + @reverse_productions[[state_item.state, state_item.item.lhs]]&.each do |si| + queue << si + end end end + + result end + # @rbs (State conflict_state, State::Item conflict_reduce_item, Grammar::Symbol conflict_term) -> ::Array[StateItem]? def shortest_path(conflict_state, conflict_reduce_item, conflict_term) - # queue: is an array of [Triple, [Path]] - queue = [] #: Array[[Triple, Array[StartPath|TransitionPath|ProductionPath]]] + time1 = Time.now.to_f + @iterate_count = 0 + + queue = [] #: Array[[Triple, Path]] visited = {} #: Hash[Triple, true] start_state = @states.states.first #: Lrama::State + conflict_term_bit = Bitmap::from_integer(conflict_term.number) raise "BUG: Start state should be just one kernel." if start_state.kernels.count != 1 + reachable = reachable_state_items(get_state_item(conflict_state, conflict_reduce_item)) + start = get_triple(get_state_item(start_state, start_state.kernels.first), Bitmap::from_integer(@states.eof_symbol.number)) - start = Triple.new(start_state, start_state.kernels.first, Set.new([@states.eof_symbol])) + queue << [start, Path.new(start.state_item, nil)] - queue << [start, [StartPath.new(start.state_item)]] + while (triple, path = queue.shift) + @iterate_count += 1 - while true - triple, paths = queue.shift + # Found + if (triple.state == conflict_state) && (triple.item == conflict_reduce_item) && (triple.l & conflict_term_bit != 0) + state_items = [path.state_item] - next if visited[triple] - visited[triple] = true + while (path = path.parent) + state_items << path.state_item + end - # Found - if triple.state == conflict_state && triple.item == conflict_reduce_item && triple.l.include?(conflict_term) - return paths + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "shortest_path #{@iterate_count} iteration", duration) + end + + return state_items.reverse end # transition - triple.state.transitions.each do |shift, next_state| - next unless triple.item.next_sym && triple.item.next_sym == shift.next_sym - next_state.kernels.each do |kernel| - next if kernel.rule != triple.item.rule - t = Triple.new(next_state, kernel, triple.l) - queue << [t, paths + [TransitionPath.new(triple.state_item, t.state_item)]] + next_state_item = @transitions[[triple.state_item, triple.item.next_sym]] + if next_state_item && reachable.include?(next_state_item) + # @type var t: Triple + t = get_triple(next_state_item, triple.l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] end end # production step - triple.state.closure.each do |item| - next unless triple.item.next_sym && triple.item.next_sym == item.lhs + @productions[triple.state_item]&.each do |si| + next unless reachable.include?(si) + l = follow_l(triple.item, triple.l) - t = Triple.new(triple.state, item, l) - queue << [t, paths + [ProductionPath.new(triple.state_item, t.state_item)]] + # @type var t: Triple + t = get_triple(si, l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end end - - break if queue.empty? end return nil end + # @rbs (State::Item item, Bitmap::bitmap current_l) -> Bitmap::bitmap def follow_l(item, current_l) # 1. follow_L (A -> X1 ... Xn-1 • Xn) = L # 2. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = {Xk+2} if Xk+2 is a terminal @@ -287,11 +398,28 @@ module Lrama when item.number_of_rest_symbols == 1 current_l when item.next_next_sym.term? - Set.new([item.next_next_sym]) + item.next_next_sym.number_bitmap when !item.next_next_sym.nullable - item.next_next_sym.first_set + item.next_next_sym.first_set_bitmap else - item.next_next_sym.first_set + follow_l(item.new_by_next_position, current_l) + item.next_next_sym.first_set_bitmap | follow_l(item.new_by_next_position, current_l) + end + end + + # @rbs [T] (String message) { -> T } -> T + def with_timeout(message) + Timeout.timeout(PathSearchTimeLimit, Timeout::Error, message + " timeout of #{PathSearchTimeLimit} sec exceeded") do + yield + end + end + + # @rbs (Float|Integer duration) -> void + def increment_total_duration(duration) + @total_duration += duration + + if !@exceed_cumulative_time_limit && @total_duration > CumulativeTimeLimit + @exceed_cumulative_time_limit = true + STDERR.puts "CumulativeTimeLimit #{CumulativeTimeLimit} sec exceeded then skip following Counterexamples calculation" end end end diff --git a/tool/lrama/lib/lrama/counterexamples/derivation.rb b/tool/lrama/lib/lrama/counterexamples/derivation.rb index 368d7f1032..a2b74767a9 100644 --- a/tool/lrama/lib/lrama/counterexamples/derivation.rb +++ b/tool/lrama/lib/lrama/counterexamples/derivation.rb @@ -1,34 +1,44 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples class Derivation - attr_reader :item, :left, :right - attr_writer :right + # @rbs! + # @item: State::Item + # @left: Derivation? - def initialize(item, left, right = nil) + attr_reader :item #: State::Item + attr_reader :left #: Derivation? + attr_accessor :right #: Derivation? + + # @rbs (State::Item item, Derivation? left) -> void + def initialize(item, left) @item = item @left = left - @right = right end + # @rbs () -> ::String def to_s "#<Derivation(#{item.display_name})>" end alias :inspect :to_s + # @rbs () -> Array[String] def render_strings_for_report result = [] #: Array[String] _render_for_report(self, 0, result, 0) result.map(&:rstrip) end + # @rbs () -> String def render_for_report render_strings_for_report.join("\n") end private + # @rbs (Derivation derivation, Integer offset, Array[String] strings, Integer index) -> Integer def _render_for_report(derivation, offset, strings, index) item = derivation.item if strings[index] diff --git a/tool/lrama/lib/lrama/counterexamples/example.rb b/tool/lrama/lib/lrama/counterexamples/example.rb index bb08428fcd..c007f45af4 100644 --- a/tool/lrama/lib/lrama/counterexamples/example.rb +++ b/tool/lrama/lib/lrama/counterexamples/example.rb @@ -1,12 +1,31 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples class Example - attr_reader :path1, :path2, :conflict, :conflict_symbol + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @path1: ::Array[StateItem] + # @path2: ::Array[StateItem] + # @conflict: State::conflict + # @conflict_symbol: Grammar::Symbol + # @counterexamples: Counterexamples + # @derivations1: Derivation + # @derivations2: Derivation + + attr_reader :path1 #: ::Array[StateItem] + attr_reader :path2 #: ::Array[StateItem] + attr_reader :conflict #: State::conflict + attr_reader :conflict_symbol #: Grammar::Symbol # path1 is shift conflict when S/R conflict # path2 is always reduce conflict + # + # @rbs (Array[StateItem]? path1, Array[StateItem]? path2, State::conflict conflict, Grammar::Symbol conflict_symbol, Counterexamples counterexamples) -> void def initialize(path1, path2, conflict, conflict_symbol, counterexamples) @path1 = path1 @path2 = path2 @@ -15,69 +34,75 @@ module Lrama @counterexamples = counterexamples end + # @rbs () -> (:shift_reduce | :reduce_reduce) def type @conflict.type end + # @rbs () -> State::Item def path1_item - @path1.last.to.item + @path1.last.item end + # @rbs () -> State::Item def path2_item - @path2.last.to.item + @path2.last.item end + # @rbs () -> Derivation def derivations1 @derivations1 ||= _derivations(path1) end + # @rbs () -> Derivation def derivations2 @derivations2 ||= _derivations(path2) end private - def _derivations(paths) + # @rbs (Array[StateItem] state_items) -> Derivation + def _derivations(state_items) derivation = nil #: Derivation current = :production - last_path = paths.last #: Path - lookahead_sym = last_path.to.item.end_of_rule? ? @conflict_symbol : nil + last_state_item = state_items.last #: StateItem + lookahead_sym = last_state_item.item.end_of_rule? ? @conflict_symbol : nil - paths.reverse_each do |path| - item = path.to.item + state_items.reverse_each do |si| + item = si.item case current when :production - case path - when StartPath + case si.type + when :start derivation = Derivation.new(item, derivation) current = :start - when TransitionPath + when :transition derivation = Derivation.new(item, derivation) current = :transition - when ProductionPath + when :production derivation = Derivation.new(item, derivation) current = :production else - raise "Unexpected. #{path}" + raise "Unexpected. #{si}" end if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym) - state_item = @counterexamples.transitions[[path.to, item.next_sym]] - derivation2 = find_derivation_for_symbol(state_item, lookahead_sym) + si2 = @counterexamples.transitions[[si, item.next_sym]] + derivation2 = find_derivation_for_symbol(si2, lookahead_sym) derivation.right = derivation2 # steep:ignore lookahead_sym = nil end when :transition - case path - when StartPath + case si.type + when :start derivation = Derivation.new(item, derivation) current = :start - when TransitionPath + when :transition # ignore current = :transition - when ProductionPath + when :production # ignore current = :production end @@ -91,6 +116,7 @@ module Lrama derivation end + # @rbs (StateItem state_item, Grammar::Symbol sym) -> Derivation? def find_derivation_for_symbol(state_item, sym) queue = [] #: Array[Array[StateItem]] queue << [state_item] @@ -110,9 +136,8 @@ module Lrama end if next_sym.nterm? && next_sym.first_set.include?(sym) - @counterexamples.productions[si].each do |next_item| - next if next_item.empty_rule? - next_si = StateItem.new(si.state, next_item) + @counterexamples.productions[si].each do |next_si| + next if next_si.item.empty_rule? next if sis.include?(next_si) queue << (sis + [next_si]) end diff --git a/tool/lrama/lib/lrama/counterexamples/node.rb b/tool/lrama/lib/lrama/counterexamples/node.rb new file mode 100644 index 0000000000..9214a0e7f1 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/node.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + # @rbs generic E < Object -- Type of an element + class Node + attr_reader :elem #: E + attr_reader :next_node #: Node[E]? + + # @rbs [E < Object] (Node[E] node) -> Array[E] + def self.to_a(node) + a = [] # steep:ignore UnannotatedEmptyCollection + + while (node) + a << node.elem + node = node.next_node + end + + a + end + + # @rbs (E elem, Node[E]? next_node) -> void + def initialize(elem, next_node) + @elem = elem + @next_node = next_node + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/path.rb b/tool/lrama/lib/lrama/counterexamples/path.rb index 0a5823dd21..6b1325f73b 100644 --- a/tool/lrama/lib/lrama/counterexamples/path.rb +++ b/tool/lrama/lib/lrama/counterexamples/path.rb @@ -1,29 +1,27 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples class Path - def initialize(from_state_item, to_state_item) - @from_state_item = from_state_item - @to_state_item = to_state_item - end + # @rbs! + # @state_item: StateItem + # @parent: Path? - def from - @from_state_item - end + attr_reader :state_item #: StateItem + attr_reader :parent #: Path? - def to - @to_state_item + # @rbs (StateItem state_item, Path? parent) -> void + def initialize(state_item, parent) + @state_item = state_item + @parent = parent end + # @rbs () -> ::String def to_s - "#<Path(#{type})>" + "#<Path>" end alias :inspect :to_s - - def type - raise NotImplementedError - end end end end diff --git a/tool/lrama/lib/lrama/counterexamples/production_path.rb b/tool/lrama/lib/lrama/counterexamples/production_path.rb deleted file mode 100644 index 0a230c7fce..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/production_path.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Counterexamples - class ProductionPath < Path - def type - :production - end - - def transition? - false - end - - def production? - true - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/start_path.rb b/tool/lrama/lib/lrama/counterexamples/start_path.rb deleted file mode 100644 index c0351c8248..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/start_path.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Counterexamples - class StartPath < Path - def initialize(to_state_item) - super nil, to_state_item - end - - def type - :start - end - - def transition? - false - end - - def production? - false - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/state_item.rb b/tool/lrama/lib/lrama/counterexamples/state_item.rb index c919818324..8c2481d793 100644 --- a/tool/lrama/lib/lrama/counterexamples/state_item.rb +++ b/tool/lrama/lib/lrama/counterexamples/state_item.rb @@ -1,8 +1,31 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples - class StateItem < Struct.new(:state, :item) + class StateItem + attr_reader :id #: Integer + attr_reader :state #: State + attr_reader :item #: State::Item + + # @rbs (Integer id, State state, State::Item item) -> void + def initialize(id, state, item) + @id = id + @state = state + @item = item + end + + # @rbs () -> (:start | :transition | :production) + def type + case + when item.start_item? + :start + when item.beginning_of_rule? + :production + else + :transition + end + end end end end diff --git a/tool/lrama/lib/lrama/counterexamples/transition_path.rb b/tool/lrama/lib/lrama/counterexamples/transition_path.rb deleted file mode 100644 index 47bfbc4f98..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/transition_path.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Counterexamples - class TransitionPath < Path - def type - :transition - end - - def transition? - true - end - - def production? - false - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/triple.rb b/tool/lrama/lib/lrama/counterexamples/triple.rb index 64014ee223..98fe051f53 100644 --- a/tool/lrama/lib/lrama/counterexamples/triple.rb +++ b/tool/lrama/lib/lrama/counterexamples/triple.rb @@ -1,21 +1,39 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Counterexamples - # s: state - # itm: item within s - # l: precise lookahead set - class Triple < Struct.new(:s, :itm, :l) - alias :state :s - alias :item :itm - alias :precise_lookahead_set :l + class Triple + attr_reader :precise_lookahead_set #: Bitmap::bitmap + alias :l :precise_lookahead_set + + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> void + def initialize(state_item, precise_lookahead_set) + @state_item = state_item + @precise_lookahead_set = precise_lookahead_set + end + + # @rbs () -> State + def state + @state_item.state + end + alias :s :state + + # @rbs () -> State::Item + def item + @state_item.item + end + alias :itm :item + + # @rbs () -> StateItem def state_item - StateItem.new(state, item) + @state_item end + # @rbs () -> ::String def inspect - "#{state.inspect}. #{item.display_name}. #{l.map(&:id).map(&:s_value)}" + "#{state.inspect}. #{item.display_name}. #{l.to_s(2)}" end alias :to_s :inspect end diff --git a/tool/lrama/lib/lrama/diagnostics.rb b/tool/lrama/lib/lrama/diagnostics.rb deleted file mode 100644 index e9da398c89..0000000000 --- a/tool/lrama/lib/lrama/diagnostics.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Diagnostics - def initialize(grammar, states, logger) - @grammar = grammar - @states = states - @logger = logger - end - - def run(diagnostic) - if diagnostic - diagnose_conflict - diagnose_parameterizing_redefined - end - end - - private - - def diagnose_conflict - if @states.sr_conflicts_count != 0 - @logger.warn("shift/reduce conflicts: #{@states.sr_conflicts_count} found") - end - - if @states.rr_conflicts_count != 0 - @logger.warn("reduce/reduce conflicts: #{@states.rr_conflicts_count} found") - end - end - - def diagnose_parameterizing_redefined - @grammar.parameterizing_rule_resolver.redefined_rules.each do |rule| - @logger.warn("parameterizing rule redefined: #{rule}") - end - end - end -end diff --git a/tool/lrama/lib/lrama/diagram.rb b/tool/lrama/lib/lrama/diagram.rb new file mode 100644 index 0000000000..985808933f --- /dev/null +++ b/tool/lrama/lib/lrama/diagram.rb @@ -0,0 +1,77 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Diagram + class << self + # @rbs (IO out, Grammar grammar, String template_name) -> void + def render(out:, grammar:, template_name: 'diagram/diagram.html') + return unless require_railroad_diagrams + new(out: out, grammar: grammar, template_name: template_name).render + end + + # @rbs () -> bool + def require_railroad_diagrams + require "railroad_diagrams" + true + rescue LoadError + warn "railroad_diagrams is not installed. Please run `bundle install`." + false + end + end + + # @rbs (IO out, Grammar grammar, String template_name) -> void + def initialize(out:, grammar:, template_name: 'diagram/diagram.html') + @grammar = grammar + @out = out + @template_name = template_name + end + + # @rbs () -> void + def render + RailroadDiagrams::TextDiagram.set_formatting(RailroadDiagrams::TextDiagram::PARTS_UNICODE) + @out << ERB.render(template_file, output: self) + end + + # @rbs () -> string + def default_style + RailroadDiagrams::Style::default_style + end + + # @rbs () -> string + def diagrams + result = +'' + @grammar.unique_rule_s_values.each do |s_value| + diagrams = + @grammar.select_rules_by_s_value(s_value).map { |r| r.to_diagrams } + add_diagram( + s_value, + RailroadDiagrams::Diagram.new( + RailroadDiagrams::Choice.new(0, *diagrams), + ), + result + ) + end + result + end + + private + + # @rbs () -> string + def template_dir + File.expand_path('../../template', __dir__) + end + + # @rbs () -> string + def template_file + File.join(template_dir, @template_name) + end + + # @rbs (String name, RailroadDiagrams::Diagram diagram, String result) -> void + def add_diagram(name, diagram, result) + result << "\n<h2 class=\"diagram-header\">#{RailroadDiagrams.escape_html(name)}</h2>" + diagram.write_svg(result.method(:<<)) + result << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/digraph.rb b/tool/lrama/lib/lrama/digraph.rb index d2bb88101a..52865f52dd 100644 --- a/tool/lrama/lib/lrama/digraph.rb +++ b/tool/lrama/lib/lrama/digraph.rb @@ -1,23 +1,73 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama - # Algorithm Digraph of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # Digraph Algorithm of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # + # Digraph is an algorithm for graph data structure. + # The algorithm efficiently traverses SCC (Strongly Connected Component) of graph + # and merges nodes attributes within the same SCC. + # + # `compute_read_sets` and `compute_follow_sets` have the same structure. + # Graph of gotos and attributes of gotos are given then compute propagated attributes for each node. + # + # In the case of `compute_read_sets`: + # + # * Set of gotos is nodes of graph + # * `reads_relation` is edges of graph + # * `direct_read_sets` is nodes attributes + # + # In the case of `compute_follow_sets`: + # + # * Set of gotos is nodes of graph + # * `includes_relation` is edges of graph + # * `read_sets` is nodes attributes + # + # + # @rbs generic X < Object -- Type of a node + # @rbs generic Y < _Or -- Type of attribute sets assigned to a node which should support merge operation (#| method) class Digraph + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # interface _Or + # def |: (self) -> self + # end + # @sets: Array[X] + # @relation: Hash[X, Array[X]] + # @base_function: Hash[X, Y] + # @stack: Array[X] + # @h: Hash[X, (Integer|Float)?] + # @result: Hash[X, Y] + + # @rbs sets: Array[X] -- Nodes of graph + # @rbs relation: Hash[X, Array[X]] -- Edges of graph + # @rbs base_function: Hash[X, Y] -- Attributes of nodes + # @rbs return: void def initialize(sets, relation, base_function) + # X in the paper @sets = sets + # R in the paper @relation = relation + # F' in the paper @base_function = base_function + # S in the paper @stack = [] + # N in the paper @h = Hash.new(0) + # F in the paper @result = {} end + # @rbs () -> Hash[X, Y] def compute @sets.each do |x| next if @h[x] != 0 @@ -29,6 +79,7 @@ module Lrama private + # @rbs (X x) -> void def traverse(x) @stack.push(x) d = @stack.count diff --git a/tool/lrama/lib/lrama/erb.rb b/tool/lrama/lib/lrama/erb.rb new file mode 100644 index 0000000000..8f8be54811 --- /dev/null +++ b/tool/lrama/lib/lrama/erb.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "erb" + +module Lrama + class ERB + # @rbs (String file, **untyped kwargs) -> String + def self.render(file, **kwargs) + new(file).render(**kwargs) + end + + # @rbs (String file) -> void + def initialize(file) + input = File.read(file) + if ::ERB.instance_method(:initialize).parameters.last.first == :key + @erb = ::ERB.new(input, trim_mode: '-') + else + @erb = ::ERB.new(input, nil, '-') # steep:ignore UnexpectedPositionalArgument + end + @erb.filename = file + end + + # @rbs (**untyped kwargs) -> String + def render(**kwargs) + @erb.result_with_hash(kwargs) + end + end +end diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb index 52625627bc..95a80bb01c 100644 --- a/tool/lrama/lib/lrama/grammar.rb +++ b/tool/lrama/lib/lrama/grammar.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true require "forwardable" @@ -7,7 +8,8 @@ require_relative "grammar/code" require_relative "grammar/counter" require_relative "grammar/destructor" require_relative "grammar/error_token" -require_relative "grammar/parameterizing_rule" +require_relative "grammar/inline" +require_relative "grammar/parameterized" require_relative "grammar/percent_code" require_relative "grammar/precedence" require_relative "grammar/printer" @@ -23,19 +25,89 @@ require_relative "lexer" module Lrama # Grammar is the result of parsing an input grammar file class Grammar + # @rbs! + # + # interface _DelegatedMethods + # def rules: () -> Array[Rule] + # def accept_symbol: () -> Grammar::Symbol + # def eof_symbol: () -> Grammar::Symbol + # def undef_symbol: () -> Grammar::Symbol + # def precedences: () -> Array[Precedence] + # + # # delegate to @symbols_resolver + # def symbols: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def ielr_defined?: () -> bool + # end + # + # include Symbols::Resolver::_DelegatedMethods + # + # @rule_counter: Counter + # @percent_codes: Array[PercentCode] + # @printers: Array[Printer] + # @destructors: Array[Destructor] + # @error_tokens: Array[ErrorToken] + # @symbols_resolver: Symbols::Resolver + # @types: Array[Type] + # @rule_builders: Array[RuleBuilder] + # @rules: Array[Rule] + # @sym_to_rules: Hash[Integer, Array[Rule]] + # @parameterized_resolver: Parameterized::Resolver + # @empty_symbol: Grammar::Symbol + # @eof_symbol: Grammar::Symbol + # @error_symbol: Grammar::Symbol + # @undef_symbol: Grammar::Symbol + # @accept_symbol: Grammar::Symbol + # @aux: Auxiliary + # @no_stdlib: bool + # @locations: bool + # @define: Hash[String, String] + # @required: bool + # @union: Union + # @precedences: Array[Precedence] + # @start_nterm: Lrama::Lexer::Token::Base? + extend Forwardable - attr_reader :percent_codes, :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux, :parameterizing_rule_resolver - attr_accessor :union, :expect, :printers, :error_tokens, :lex_param, :parse_param, :initial_action, - :after_shift, :before_reduce, :after_reduce, :after_shift_error_token, :after_pop_stack, - :symbols_resolver, :types, :rules, :rule_builders, :sym_to_rules, :no_stdlib, :locations + attr_reader :percent_codes #: Array[PercentCode] + attr_reader :eof_symbol #: Grammar::Symbol + attr_reader :error_symbol #: Grammar::Symbol + attr_reader :undef_symbol #: Grammar::Symbol + attr_reader :accept_symbol #: Grammar::Symbol + attr_reader :aux #: Auxiliary + attr_reader :parameterized_resolver #: Parameterized::Resolver + attr_reader :precedences #: Array[Precedence] + attr_accessor :union #: Union + attr_accessor :expect #: Integer + attr_accessor :printers #: Array[Printer] + attr_accessor :error_tokens #: Array[ErrorToken] + attr_accessor :lex_param #: String + attr_accessor :parse_param #: String + attr_accessor :initial_action #: Grammar::Code::InitialActionCode + attr_accessor :after_shift #: Lexer::Token::Base + attr_accessor :before_reduce #: Lexer::Token::Base + attr_accessor :after_reduce #: Lexer::Token::Base + attr_accessor :after_shift_error_token #: Lexer::Token::Base + attr_accessor :after_pop_stack #: Lexer::Token::Base + attr_accessor :symbols_resolver #: Symbols::Resolver + attr_accessor :types #: Array[Type] + attr_accessor :rules #: Array[Rule] + attr_accessor :rule_builders #: Array[RuleBuilder] + attr_accessor :sym_to_rules #: Hash[Integer, Array[Rule]] + attr_accessor :no_stdlib #: bool + attr_accessor :locations #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :required #: bool def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value, :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type, :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number! - def initialize(rule_counter) + # @rbs (Counter rule_counter, bool locations, Hash[String, String] define) -> void + def initialize(rule_counter, locations, define = {}) @rule_counter = rule_counter # Code defined by "%code" @@ -48,7 +120,7 @@ module Lrama @rule_builders = [] @rules = [] @sym_to_rules = {} - @parameterizing_rule_resolver = ParameterizingRule::Resolver.new + @parameterized_resolver = Parameterized::Resolver.new @empty_symbol = nil @eof_symbol = nil @error_symbol = nil @@ -56,92 +128,131 @@ module Lrama @accept_symbol = nil @aux = Auxiliary.new @no_stdlib = false - @locations = false + @locations = locations + @define = define + @required = false + @precedences = [] + @start_nterm = nil append_special_symbols end + # @rbs (Counter rule_counter, Counter midrule_action_counter) -> RuleBuilder def create_rule_builder(rule_counter, midrule_action_counter) - RuleBuilder.new(rule_counter, midrule_action_counter, @parameterizing_rule_resolver) + RuleBuilder.new(rule_counter, midrule_action_counter, @parameterized_resolver) end + # @rbs (id: Lexer::Token::Base, code: Lexer::Token::UserCode) -> Array[PercentCode] def add_percent_code(id:, code:) @percent_codes << PercentCode.new(id.s_value, code.s_value) end + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Destructor] def add_destructor(ident_or_tags:, token_code:, lineno:) @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) end + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Printer] def add_printer(ident_or_tags:, token_code:, lineno:) @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) end + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[ErrorToken] def add_error_token(ident_or_tags:, token_code:, lineno:) @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) end + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> Array[Type] def add_type(id:, tag:) @types << Type.new(id: id, tag: tag) end - def add_nonassoc(sym, precedence) - set_precedence(sym, Precedence.new(type: :nonassoc, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_nonassoc(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :nonassoc, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_left(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :left, precedence: precedence, lineno: lineno)) end - def add_left(sym, precedence) - set_precedence(sym, Precedence.new(type: :left, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_right(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :right, precedence: precedence, lineno: lineno)) end - def add_right(sym, precedence) - set_precedence(sym, Precedence.new(type: :right, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_precedence(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :precedence, precedence: precedence, lineno: lineno)) end - def add_precedence(sym, precedence) - set_precedence(sym, Precedence.new(type: :precedence, precedence: precedence)) + # @rbs (Lrama::Lexer::Token::Base id) -> Lrama::Lexer::Token::Base + def set_start_nterm(id) + # When multiple `%start` directives are defined, Bison does not generate an error, + # whereas Lrama does generate an error. + # Related Bison's specification are + # refs: https://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html + if @start_nterm.nil? + @start_nterm = id + else + start = @start_nterm #: Lrama::Lexer::Token::Base + raise "Start non-terminal is already set to #{start.s_value} (line: #{start.first_line}). Cannot set to #{id.s_value} (line: #{id.first_line})." + end end + # @rbs (Grammar::Symbol sym, Precedence precedence) -> (Precedence | bot) def set_precedence(sym, precedence) - raise "" if sym.nterm? + @precedences << precedence sym.precedence = precedence end + # @rbs (Grammar::Code::NoReferenceCode code, Integer lineno) -> Union def set_union(code, lineno) @union = Union.new(code: code, lineno: lineno) end + # @rbs (RuleBuilder builder) -> Array[RuleBuilder] def add_rule_builder(builder) @rule_builders << builder end - def add_parameterizing_rule(rule) - @parameterizing_rule_resolver.add_parameterizing_rule(rule) + # @rbs (Parameterized::Rule rule) -> Array[Parameterized::Rule] + def add_parameterized_rule(rule) + @parameterized_resolver.add_rule(rule) end - def parameterizing_rules - @parameterizing_rule_resolver.rules + # @rbs () -> Array[Parameterized::Rule] + def parameterized_rules + @parameterized_resolver.rules end - def insert_before_parameterizing_rules(rules) - @parameterizing_rule_resolver.rules = rules + @parameterizing_rule_resolver.rules + # @rbs (Array[Parameterized::Rule] rules) -> Array[Parameterized::Rule] + def prepend_parameterized_rules(rules) + @parameterized_resolver.rules = rules + @parameterized_resolver.rules end + # @rbs (Integer prologue_first_lineno) -> Integer def prologue_first_lineno=(prologue_first_lineno) @aux.prologue_first_lineno = prologue_first_lineno end + # @rbs (String prologue) -> String def prologue=(prologue) @aux.prologue = prologue end + # @rbs (Integer epilogue_first_lineno) -> Integer def epilogue_first_lineno=(epilogue_first_lineno) @aux.epilogue_first_lineno = epilogue_first_lineno end + # @rbs (String epilogue) -> String def epilogue=(epilogue) @aux.epilogue = epilogue end + # @rbs () -> void def prepare resolve_inline_rules normalize_rules @@ -150,6 +261,7 @@ module Lrama fill_default_precedence fill_symbols fill_sym_to_rules + sort_precedence compute_nullable compute_first_set set_locations @@ -158,21 +270,51 @@ module Lrama # TODO: More validation methods # # * Validation for no_declared_type_reference + # + # @rbs () -> void def validate! @symbols_resolver.validate! + validate_no_precedence_for_nterm! validate_rule_lhs_is_nterm! + validate_duplicated_precedence! end + # @rbs (Grammar::Symbol sym) -> Array[Rule] def find_rules_by_symbol!(sym) find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found") end + # @rbs (Grammar::Symbol sym) -> Array[Rule]? def find_rules_by_symbol(sym) @sym_to_rules[sym.number] end + # @rbs (String s_value) -> Array[Rule] + def select_rules_by_s_value(s_value) + @rules.select {|rule| rule.lhs.id.s_value == s_value } + end + + # @rbs () -> Array[String] + def unique_rule_s_values + @rules.map {|rule| rule.lhs.id.s_value }.uniq + end + + # @rbs () -> bool + def ielr_defined? + @define.key?('lr.type') && @define['lr.type'] == 'ielr' + end + private + # @rbs () -> void + def sort_precedence + @precedences.sort_by! do |prec| + prec.symbol.number + end + @precedences.freeze + end + + # @rbs () -> Array[Grammar::Symbol] def compute_nullable @rules.each do |rule| case @@ -222,6 +364,7 @@ module Lrama end end + # @rbs () -> Array[Grammar::Symbol] def compute_first_set terms.each do |term| term.first_set = Set.new([term]).freeze @@ -257,12 +400,14 @@ module Lrama end end + # @rbs () -> Array[RuleBuilder] def setup_rules @rule_builders.each do |builder| builder.setup_rules end end + # @rbs () -> Grammar::Symbol def append_special_symbols # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) @@ -293,11 +438,12 @@ module Lrama @accept_symbol = term end + # @rbs () -> void def resolve_inline_rules - while @rule_builders.any? {|r| r.has_inline_rules? } do + while @rule_builders.any?(&:has_inline_rules?) do @rule_builders = @rule_builders.flat_map do |builder| if builder.has_inline_rules? - builder.resolve_inline_rules + Inline::Resolver.new(builder).resolve else builder end @@ -305,14 +451,10 @@ module Lrama end end + # @rbs () -> void def normalize_rules - # Add $accept rule to the top of rules - rule_builder = @rule_builders.first # : RuleBuilder - lineno = rule_builder ? rule_builder.line : 0 - @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [rule_builder.lhs, @eof_symbol.id], token_code: nil, lineno: lineno) - + add_accept_rule setup_rules - @rule_builders.each do |builder| builder.rules.each do |rule| add_nterm(id: rule._lhs, tag: rule.lhs_tag) @@ -320,23 +462,42 @@ module Lrama end end - @rules.sort_by!(&:id) + nterms.freeze + @rules.sort_by!(&:id).freeze + end + + # Add $accept rule to the top of rules + def add_accept_rule + if @start_nterm + start = @start_nterm #: Lrama::Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [start, @eof_symbol.id], token_code: nil, lineno: start.line) + else + rule_builder = @rule_builders.first #: RuleBuilder + lineno = rule_builder ? rule_builder.line : 0 + lhs = rule_builder.lhs #: Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + end end # Collect symbols from rules + # + # @rbs () -> void def collect_symbols @rules.flat_map(&:_rhs).each do |s| case s when Lrama::Lexer::Token::Char add_term(id: s) - when Lrama::Lexer::Token + when Lrama::Lexer::Token::Base # skip else raise "Unknown class: #{s}" end end + + terms.freeze end + # @rbs () -> void def set_lhs_and_rhs @rules.each do |rule| rule.lhs = token_to_symbol(rule._lhs) if rule._lhs @@ -350,6 +511,8 @@ module Lrama # Rule inherits precedence from the last term in RHS. # # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html + # + # @rbs () -> void def fill_default_precedence @rules.each do |rule| # Explicitly specified precedence has the highest priority @@ -364,6 +527,7 @@ module Lrama end end + # @rbs () -> Array[Grammar::Symbol] def fill_symbols fill_symbol_number fill_nterm_type(@types) @@ -373,6 +537,7 @@ module Lrama sort_by_number! end + # @rbs () -> Array[Rule] def fill_sym_to_rules @rules.each do |rule| key = rule.lhs.number @@ -381,13 +546,48 @@ module Lrama end end + # @rbs () -> void + def validate_no_precedence_for_nterm! + errors = [] #: Array[String] + + nterms.each do |nterm| + next if nterm.precedence.nil? + + errors << "[BUG] Precedence #{nterm.name} (line: #{nterm.precedence.lineno}) is defined for nonterminal symbol (line: #{nterm.id.first_line}). Precedence can be defined for only terminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void def validate_rule_lhs_is_nterm! errors = [] #: Array[String] rules.each do |rule| next if rule.lhs.nterm? - errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is term. It should be nterm." + errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is terminal symbol. It should be nonterminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # # @rbs () -> void + def validate_duplicated_precedence! + errors = [] #: Array[String] + seen = {} #: Hash[String, Precedence] + + precedences.each do |prec| + s_value = prec.s_value + if first = seen[s_value] + errors << "%#{prec.type} redeclaration for #{s_value} (line: #{prec.lineno}) previous declaration was %#{first.type} (line: #{first.lineno})" + else + seen[s_value] = prec + end end return if errors.empty? @@ -395,6 +595,7 @@ module Lrama raise errors.join("\n") end + # @rbs () -> void def set_locations @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? } end diff --git a/tool/lrama/lib/lrama/grammar/auxiliary.rb b/tool/lrama/lib/lrama/grammar/auxiliary.rb index 2bacee6f1a..76cfb74d4d 100644 --- a/tool/lrama/lib/lrama/grammar/auxiliary.rb +++ b/tool/lrama/lib/lrama/grammar/auxiliary.rb @@ -1,9 +1,14 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar # Grammar file information not used by States but by Output - class Auxiliary < Struct.new(:prologue_first_lineno, :prologue, :epilogue_first_lineno, :epilogue, keyword_init: true) + class Auxiliary + attr_accessor :prologue_first_lineno #: Integer? + attr_accessor :prologue #: String? + attr_accessor :epilogue_first_lineno #: Integer? + attr_accessor :epilogue #: String? end end end diff --git a/tool/lrama/lib/lrama/grammar/binding.rb b/tool/lrama/lib/lrama/grammar/binding.rb index 5e6e7c594b..5940d153a9 100644 --- a/tool/lrama/lib/lrama/grammar/binding.rb +++ b/tool/lrama/lib/lrama/grammar/binding.rb @@ -1,34 +1,78 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Binding - attr_reader :actual_args, :count + # @rbs @actual_args: Array[Lexer::Token::Base] + # @rbs @param_to_arg: Hash[String, Lexer::Token::Base] - def initialize(parameterizing_rule, actual_args) - @parameters = parameterizing_rule.parameters + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> void + def initialize(params, actual_args) @actual_args = actual_args - @parameter_to_arg = @parameters.zip(actual_args).map do |param, arg| - [param.s_value, arg] - end.to_h + @param_to_arg = build_param_to_arg(params, @actual_args) + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def resolve_symbol(sym) + return create_instantiate_rule(sym) if sym.is_a?(Lexer::Token::InstantiateRule) + find_arg_for_param(sym) + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def concatenated_args_str(token) + "#{token.rule_name}_#{format_args(token)}" + end + + private + + # @rbs (Lexer::Token::InstantiateRule sym) -> Lexer::Token::InstantiateRule + def create_instantiate_rule(sym) + Lrama::Lexer::Token::InstantiateRule.new( + s_value: sym.s_value, + location: sym.location, + args: resolve_args(sym.args), + lhs_tag: sym.lhs_tag + ) + end + + # @rbs (Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def resolve_args(args) + args.map { |arg| resolve_symbol(arg) } end - def resolve_symbol(symbol) - if symbol.is_a?(Lexer::Token::InstantiateRule) - resolved_args = symbol.args.map { |arg| resolve_symbol(arg) } - Lrama::Lexer::Token::InstantiateRule.new(s_value: symbol.s_value, location: symbol.location, args: resolved_args, lhs_tag: symbol.lhs_tag) + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def find_arg_for_param(sym) + if (arg = @param_to_arg[sym.s_value]&.dup) + arg.alias_name = sym.alias_name + arg else - parameter_to_arg(symbol) || symbol + sym end end - private + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> Hash[String, Lexer::Token::Base?] + def build_param_to_arg(params, actual_args) + params.zip(actual_args).map do |param, arg| + [param.s_value, arg] + end.to_h + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def format_args(token) + token_to_args_s_values(token).join('_') + end - def parameter_to_arg(symbol) - if (arg = @parameter_to_arg[symbol.s_value].dup) - arg.alias_name = symbol.alias_name + # @rbs (Lexer::Token::InstantiateRule token) -> Array[String] + def token_to_args_s_values(token) + token.args.flat_map do |arg| + resolved = resolve_symbol(arg) + if resolved.is_a?(Lexer::Token::InstantiateRule) + [resolved.s_value] + resolved.args.map(&:s_value) + else + [resolved.s_value] + end end - arg end end end diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb index b6c1cc49e7..f1b860eeba 100644 --- a/tool/lrama/lib/lrama/grammar/code.rb +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true require "forwardable" @@ -10,17 +11,28 @@ require_relative "code/rule_action" module Lrama class Grammar class Code + # @rbs! + # + # # delegated + # def s_value: -> String + # def line: -> Integer + # def column: -> Integer + # def references: -> Array[Lrama::Grammar::Reference] + extend Forwardable def_delegators "token_code", :s_value, :line, :column, :references - attr_reader :type, :token_code + attr_reader :type #: ::Symbol + attr_reader :token_code #: Lexer::Token::UserCode + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode) -> void def initialize(type:, token_code:) @type = type @token_code = token_code end + # @rbs (Code other) -> bool def ==(other) self.class == other.class && self.type == other.type && @@ -28,6 +40,8 @@ module Lrama end # $$, $n, @$, @n are translated to C code + # + # @rbs () -> String def translated_code t_code = s_value.dup @@ -45,6 +59,7 @@ module Lrama private + # @rbs (Lrama::Grammar::Reference ref) -> bot def reference_to_c(ref) raise NotImplementedError.new("#reference_to_c is not implemented") end diff --git a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb index 794017257c..d71b62e513 100644 --- a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb @@ -1,9 +1,18 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Code class DestructorCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void def initialize(type:, token_code:, tag:) super(type: type, token_code: token_code) @tag = tag @@ -17,6 +26,8 @@ module Lrama # * ($1) error # * (@1) error # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ diff --git a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb index 02f2badc9e..cb36041524 100644 --- a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama @@ -12,6 +13,8 @@ module Lrama # * ($1) error # * (@1) error # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ diff --git a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb index ab12f32e29..1d39919979 100644 --- a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama @@ -12,6 +13,8 @@ module Lrama # * ($1) error # * (@1) error # * ($:1) error + # + # @rbs (Reference ref) -> bot def reference_to_c(ref) case when ref.type == :dollar # $$, $n diff --git a/tool/lrama/lib/lrama/grammar/code/printer_code.rb b/tool/lrama/lib/lrama/grammar/code/printer_code.rb index c0b8d24306..c6e25d5235 100644 --- a/tool/lrama/lib/lrama/grammar/code/printer_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/printer_code.rb @@ -1,9 +1,18 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Code class PrinterCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void def initialize(type:, token_code:, tag:) super(type: type, token_code: token_code) @tag = tag @@ -17,6 +26,8 @@ module Lrama # * ($1) error # * (@1) error # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ diff --git a/tool/lrama/lib/lrama/grammar/code/rule_action.rb b/tool/lrama/lib/lrama/grammar/code/rule_action.rb index 363ecdf25d..e71e93e5a5 100644 --- a/tool/lrama/lib/lrama/grammar/code/rule_action.rb +++ b/tool/lrama/lib/lrama/grammar/code/rule_action.rb @@ -1,9 +1,18 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Code class RuleAction < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @rule: Rule + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, rule: Rule) -> void def initialize(type:, token_code:, rule:) super(type: type, token_code: token_code) @rule = rule @@ -38,6 +47,8 @@ module Lrama # "Position in grammar" $1 # "Index for yyvsp" 0 # "$:n" $:1 + # + # @rbs (Reference ref) -> String def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ @@ -66,6 +77,7 @@ module Lrama end end + # @rbs () -> Integer def position_in_rhs # If rule is not derived rule, User Code is only action at # the end of rule RHS. In such case, the action is located on @@ -74,15 +86,20 @@ module Lrama end # If this is midrule action, RHS is an RHS of the original rule. + # + # @rbs () -> Array[Grammar::Symbol] def rhs (@rule.original_rule || @rule).rhs end # Unlike `rhs`, LHS is always an LHS of the rule. + # + # @rbs () -> Grammar::Symbol def lhs @rule.lhs end + # @rbs (Reference ref) -> bot def raise_tag_not_found_error(ref) raise "Tag is not specified for '$#{ref.value}' in '#{@rule.display_name}'" end diff --git a/tool/lrama/lib/lrama/grammar/counter.rb b/tool/lrama/lib/lrama/grammar/counter.rb index dc91b87b71..ced934309d 100644 --- a/tool/lrama/lib/lrama/grammar/counter.rb +++ b/tool/lrama/lib/lrama/grammar/counter.rb @@ -1,12 +1,22 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Counter + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @number: Integer + + # @rbs (Integer number) -> void def initialize(number) @number = number end + # @rbs () -> Integer def increment n = @number @number += 1 diff --git a/tool/lrama/lib/lrama/grammar/destructor.rb b/tool/lrama/lib/lrama/grammar/destructor.rb index a2b6fde0ed..0ce8611e77 100644 --- a/tool/lrama/lib/lrama/grammar/destructor.rb +++ b/tool/lrama/lib/lrama/grammar/destructor.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class Destructor < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + class Destructor + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident|Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String def translated_code(tag) Code::DestructorCode.new(type: :destructor, token_code: token_code, tag: tag).translated_code end diff --git a/tool/lrama/lib/lrama/grammar/error_token.rb b/tool/lrama/lib/lrama/grammar/error_token.rb index 50eaafeebc..9d9ed54ae2 100644 --- a/tool/lrama/lib/lrama/grammar/error_token.rb +++ b/tool/lrama/lib/lrama/grammar/error_token.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class ErrorToken < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + class ErrorToken + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident | Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String def translated_code(tag) Code::PrinterCode.new(type: :error_token, token_code: token_code, tag: tag).translated_code end diff --git a/tool/lrama/lib/lrama/grammar/inline.rb b/tool/lrama/lib/lrama/grammar/inline.rb new file mode 100644 index 0000000000..c02ab6002b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative 'inline/resolver' diff --git a/tool/lrama/lib/lrama/grammar/inline/resolver.rb b/tool/lrama/lib/lrama/grammar/inline/resolver.rb new file mode 100644 index 0000000000..aca689ccfb --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline/resolver.rb @@ -0,0 +1,80 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Inline + class Resolver + # @rbs (Lrama::Grammar::RuleBuilder rule_builder) -> void + def initialize(rule_builder) + @rule_builder = rule_builder + end + + # @rbs () -> Array[Lrama::Grammar::RuleBuilder] + def resolve + resolved_builders = [] #: Array[Lrama::Grammar::RuleBuilder] + @rule_builder.rhs.each_with_index do |token, i| + if (rule = @rule_builder.parameterized_resolver.find_inline(token)) + rule.rhs.each do |rhs| + builder = build_rule(rhs, token, i, rule) + resolved_builders << builder + end + break + end + end + resolved_builders + end + + private + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Lrama::Lexer::Token token, Integer index, Lrama::Grammar::Parameterized::Rule rule) -> Lrama::Grammar::RuleBuilder + def build_rule(rhs, token, index, rule) + builder = RuleBuilder.new( + @rule_builder.rule_counter, + @rule_builder.midrule_action_counter, + @rule_builder.parameterized_resolver, + lhs_tag: @rule_builder.lhs_tag + ) + resolve_rhs(builder, rhs, index, token, rule) + builder.lhs = @rule_builder.lhs + builder.line = @rule_builder.line + builder.precedence_sym = @rule_builder.precedence_sym + builder.user_code = replace_user_code(rhs, index) + builder + end + + # @rbs (Lrama::Grammar::RuleBuilder builder, Lrama::Grammar::Parameterized::Rhs rhs, Integer index, Lrama::Lexer::Token token, Lrama::Grammar::Parameterized::Rule rule) -> void + def resolve_rhs(builder, rhs, index, token, rule) + @rule_builder.rhs.each_with_index do |tok, i| + if i == index + rhs.symbols.each do |sym| + if token.is_a?(Lexer::Token::InstantiateRule) + bindings = Binding.new(rule.parameters, token.args) + builder.add_rhs(bindings.resolve_symbol(sym)) + else + builder.add_rhs(sym) + end + end + else + builder.add_rhs(tok) + end + end + end + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Integer index) -> Lrama::Lexer::Token::UserCode + def replace_user_code(rhs, index) + user_code = @rule_builder.user_code + return user_code if rhs.user_code.nil? || user_code.nil? + + code = user_code.s_value.gsub(/\$#{index + 1}/, rhs.user_code.s_value) + user_code.references.each do |ref| + next if ref.index.nil? || ref.index <= index # nil は $$ の場合 + code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (rhs.symbols.count - 1)}") + code = code.gsub(/@#{ref.index}/, "@#{ref.index + (rhs.symbols.count - 1)}") + end + Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized.rb b/tool/lrama/lib/lrama/grammar/parameterized.rb new file mode 100644 index 0000000000..48db3433f3 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'parameterized/resolver' +require_relative 'parameterized/rhs' +require_relative 'parameterized/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb index 06f2f1cef7..558f308190 100644 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb +++ b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb @@ -1,40 +1,49 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class ParameterizingRule + class Parameterized class Resolver - attr_accessor :rules, :created_lhs_list + attr_accessor :rules #: Array[Rule] + attr_accessor :created_lhs_list #: Array[Lexer::Token::Base] + # @rbs () -> void def initialize @rules = [] @created_lhs_list = [] end - def add_parameterizing_rule(rule) + # @rbs (Rule rule) -> Array[Rule] + def add_rule(rule) @rules << rule end + # @rbs (Lexer::Token::InstantiateRule token) -> Rule? def find_rule(token) select_rules(@rules, token).last end + # @rbs (Lexer::Token::Base token) -> Rule? def find_inline(token) - @rules.reverse.find { |rule| rule.name == token.s_value && rule.is_inline } + @rules.reverse.find { |rule| rule.name == token.s_value && rule.inline? } end + # @rbs (String lhs_s_value) -> Lexer::Token::Base? def created_lhs(lhs_s_value) @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } end + # @rbs () -> Array[Rule] def redefined_rules @rules.select { |rule| @rules.count { |r| r.name == rule.name && r.required_parameters_count == rule.required_parameters_count } > 1 } end private + # @rbs (Array[Rule] rules, Lexer::Token::InstantiateRule token) -> Array[Rule] def select_rules(rules, token) - rules = select_not_inline_rules(rules) + rules = reject_inline_rules(rules) rules = select_rules_by_name(rules, token.rule_name) rules = rules.select { |rule| rule.required_parameters_count == token.args_count } if rules.empty? @@ -44,14 +53,16 @@ module Lrama end end - def select_not_inline_rules(rules) - rules.select { |rule| !rule.is_inline } + # @rbs (Array[Rule] rules) -> Array[Rule] + def reject_inline_rules(rules) + rules.reject(&:inline?) end + # @rbs (Array[Rule] rules, String rule_name) -> Array[Rule] def select_rules_by_name(rules, rule_name) rules = rules.select { |rule| rule.name == rule_name } if rules.empty? - raise "Parameterizing rule does not exist. `#{rule_name}`" + raise "Parameterized rule does not exist. `#{rule_name}`" else rules end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb index f60781c053..663de49100 100644 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb +++ b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb @@ -1,17 +1,22 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class ParameterizingRule + class Parameterized class Rhs - attr_accessor :symbols, :user_code, :precedence_sym + attr_accessor :symbols #: Array[Lexer::Token::Base] + attr_accessor :user_code #: Lexer::Token::UserCode? + attr_accessor :precedence_sym #: Grammar::Symbol? + # @rbs () -> void def initialize @symbols = [] @user_code = nil @precedence_sym = nil end + # @rbs (Grammar::Binding bindings) -> Lexer::Token::UserCode? def resolve_user_code(bindings) return unless user_code diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rule.rb b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb new file mode 100644 index 0000000000..7048be3cff --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb @@ -0,0 +1,36 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rule + attr_reader :name #: String + attr_reader :parameters #: Array[Lexer::Token::Base] + attr_reader :rhs #: Array[Rhs] + attr_reader :required_parameters_count #: Integer + attr_reader :tag #: Lexer::Token::Tag? + + # @rbs (String name, Array[Lexer::Token::Base] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void + def initialize(name, parameters, rhs, tag: nil, is_inline: false) + @name = name + @parameters = parameters + @rhs = rhs + @tag = tag + @is_inline = is_inline + @required_parameters_count = parameters.count + end + + # @rbs () -> String + def to_s + "#{@name}(#{@parameters.map(&:s_value).join(', ')})" + end + + # @rbs () -> bool + def inline? + @is_inline + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb deleted file mode 100644 index ddc1a467ce..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -require_relative 'parameterizing_rule/resolver' -require_relative 'parameterizing_rule/rhs' -require_relative 'parameterizing_rule/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb deleted file mode 100644 index cc200d2fb6..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Grammar - class ParameterizingRule - class Rule - attr_reader :name, :parameters, :rhs_list, :required_parameters_count, :tag, :is_inline - - def initialize(name, parameters, rhs_list, tag: nil, is_inline: false) - @name = name - @parameters = parameters - @rhs_list = rhs_list - @tag = tag - @is_inline = is_inline - @required_parameters_count = parameters.count - end - - def to_s - "#{@name}(#{@parameters.map(&:s_value).join(', ')})" - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/percent_code.rb b/tool/lrama/lib/lrama/grammar/percent_code.rb index 416a2d2753..9afb903056 100644 --- a/tool/lrama/lib/lrama/grammar/percent_code.rb +++ b/tool/lrama/lib/lrama/grammar/percent_code.rb @@ -1,10 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class PercentCode - attr_reader :name, :code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @name: String + # @code: String + attr_reader :name #: String + attr_reader :code #: String + + # @rbs (String name, String code) -> void def initialize(name, code) @name = name @code = code diff --git a/tool/lrama/lib/lrama/grammar/precedence.rb b/tool/lrama/lib/lrama/grammar/precedence.rb index 13cf960c32..b4c6403372 100644 --- a/tool/lrama/lib/lrama/grammar/precedence.rb +++ b/tool/lrama/lib/lrama/grammar/precedence.rb @@ -1,13 +1,55 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class Precedence < Struct.new(:type, :precedence, keyword_init: true) + class Precedence < Struct.new(:type, :symbol, :precedence, :s_value, :lineno, keyword_init: true) include Comparable + # @rbs! + # type type_enum = :left | :right | :nonassoc | :precedence + # + # attr_accessor type: type_enum + # attr_accessor symbol: Grammar::Symbol + # attr_accessor precedence: Integer + # attr_accessor s_value: String + # attr_accessor lineno: Integer + # + # def initialize: (?type: type_enum, ?symbol: Grammar::Symbol, ?precedence: Integer, ?s_value: ::String, ?lineno: Integer) -> void + attr_reader :used_by_lalr #: Array[State::ResolvedConflict] + attr_reader :used_by_ielr #: Array[State::ResolvedConflict] + + # @rbs (Precedence other) -> Integer def <=>(other) self.precedence <=> other.precedence end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_lalr(resolved_conflict) + @used_by_lalr ||= [] #: Array[State::ResolvedConflict] + @used_by_lalr << resolved_conflict + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_ielr(resolved_conflict) + @used_by_ielr ||= [] #: Array[State::ResolvedConflict] + @used_by_ielr << resolved_conflict + end + + # @rbs () -> bool + def used_by? + used_by_lalr? || used_by_ielr? + end + + # @rbs () -> bool + def used_by_lalr? + !@used_by_lalr.nil? && !@used_by_lalr.empty? + end + + # @rbs () -> bool + def used_by_ielr? + !@used_by_ielr.nil? && !@used_by_ielr.empty? + end end end end diff --git a/tool/lrama/lib/lrama/grammar/printer.rb b/tool/lrama/lib/lrama/grammar/printer.rb index b78459e819..490fe701db 100644 --- a/tool/lrama/lib/lrama/grammar/printer.rb +++ b/tool/lrama/lib/lrama/grammar/printer.rb @@ -1,8 +1,17 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Printer < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + # @rbs! + # attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag] + # attr_accessor token_code: Lexer::Token::UserCode + # attr_accessor lineno: Integer + # + # def initialize: (?ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], ?token_code: Lexer::Token::UserCode, ?lineno: Integer) -> void + + # @rbs (Lexer::Token::Tag tag) -> String def translated_code(tag) Code::PrinterCode.new(type: :printer, token_code: token_code, tag: tag).translated_code end diff --git a/tool/lrama/lib/lrama/grammar/reference.rb b/tool/lrama/lib/lrama/grammar/reference.rb index b044516bdb..7e3badfecc 100644 --- a/tool/lrama/lib/lrama/grammar/reference.rb +++ b/tool/lrama/lib/lrama/grammar/reference.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama @@ -8,6 +9,18 @@ module Lrama # index: Integer # ex_tag: "$<tag>1" (Optional) class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + # @rbs! + # attr_accessor type: ::Symbol + # attr_accessor name: String + # attr_accessor number: Integer + # attr_accessor index: Integer + # attr_accessor ex_tag: Lexer::Token::Base? + # attr_accessor first_column: Integer + # attr_accessor last_column: Integer + # + # def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token::Base?, first_column: Integer, last_column: Integer) -> void + + # @rbs () -> (String|Integer) def value name || number end diff --git a/tool/lrama/lib/lrama/grammar/rule.rb b/tool/lrama/lib/lrama/grammar/rule.rb index 1f55bf8bfb..d00d6a8883 100644 --- a/tool/lrama/lib/lrama/grammar/rule.rb +++ b/tool/lrama/lib/lrama/grammar/rule.rb @@ -1,11 +1,38 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar # _rhs holds original RHS element. Use rhs to refer to Symbol. class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true) - attr_accessor :original_rule + # @rbs! + # + # interface _DelegatedMethods + # def lhs: -> Grammar::Symbol + # def rhs: -> Array[Grammar::Symbol] + # end + # + # attr_accessor id: Integer + # attr_accessor _lhs: Lexer::Token::Base + # attr_accessor lhs: Grammar::Symbol + # attr_accessor lhs_tag: Lexer::Token::Tag? + # attr_accessor _rhs: Array[Lexer::Token::Base] + # attr_accessor rhs: Array[Grammar::Symbol] + # attr_accessor token_code: Lexer::Token::UserCode? + # attr_accessor position_in_original_rule_rhs: Integer + # attr_accessor nullable: bool + # attr_accessor precedence_sym: Grammar::Symbol? + # attr_accessor lineno: Integer? + # + # def initialize: ( + # ?id: Integer, ?_lhs: Lexer::Token::Base?, ?lhs: Lexer::Token::Base, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token::Base], ?rhs: Array[Grammar::Symbol], + # ?token_code: Lexer::Token::UserCode?, ?position_in_original_rule_rhs: Integer?, ?nullable: bool, + # ?precedence_sym: Grammar::Symbol?, ?lineno: Integer? + # ) -> void + attr_accessor :original_rule #: Rule + + # @rbs (Rule other) -> bool def ==(other) self.class == other.class && self.lhs == other.lhs && @@ -18,14 +45,35 @@ module Lrama self.lineno == other.lineno end + # @rbs () -> String def display_name l = lhs.id.s_value r = empty_rule? ? "ε" : rhs.map {|r| r.id.s_value }.join(" ") + "#{l} -> #{r}" + end + + # @rbs () -> String + def display_name_without_action + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map do |r| + r.id.s_value if r.first_set.any? + end.compact.join(" ") "#{l} -> #{r}" end + # @rbs () -> (RailroadDiagrams::Skip | RailroadDiagrams::Sequence) + def to_diagrams + if rhs.empty? + RailroadDiagrams::Skip.new + else + RailroadDiagrams::Sequence.new(*rhs_to_diagram) + end + end + # Used by #user_actions + # + # @rbs () -> String def as_comment l = lhs.id.s_value r = empty_rule? ? "%empty" : rhs.map(&:display_name).join(" ") @@ -33,35 +81,55 @@ module Lrama "#{l}: #{r}" end + # @rbs () -> String def with_actions "#{display_name} {#{token_code&.s_value}}" end # opt_nl: ε <-- empty_rule # | '\n' <-- not empty_rule + # + # @rbs () -> bool def empty_rule? rhs.empty? end + # @rbs () -> Precedence? def precedence precedence_sym&.precedence end + # @rbs () -> bool def initial_rule? id == 0 end + # @rbs () -> String? def translated_code return nil unless token_code Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self).translated_code end + # @rbs () -> bool def contains_at_reference? return false unless token_code token_code.references.any? {|r| r.type == :at } end + + private + + # @rbs () -> Array[(RailroadDiagrams::Terminal | RailroadDiagrams::NonTerminal)] + def rhs_to_diagram + rhs.map do |r| + if r.term + RailroadDiagrams::Terminal.new(r.id.s_value) + else + RailroadDiagrams::NonTerminal.new(r.id.s_value) + end + end + end end end end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb index 06097eb71c..34fdca6c86 100644 --- a/tool/lrama/lib/lrama/grammar/rule_builder.rb +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -1,15 +1,38 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class RuleBuilder - attr_accessor :lhs, :line - attr_reader :lhs_tag, :rhs, :user_code, :precedence_sym - - def initialize(rule_counter, midrule_action_counter, parameterizing_rule_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @position_in_original_rule_rhs: Integer? + # @skip_preprocess_references: bool + # @rules: Array[Rule] + # @rule_builders_for_parameterized: Array[RuleBuilder] + # @rule_builders_for_derived_rules: Array[RuleBuilder] + # @parameterized_rules: Array[Rule] + # @midrule_action_rules: Array[Rule] + # @replaced_rhs: Array[Lexer::Token::Base]? + + attr_accessor :lhs #: Lexer::Token::Base? + attr_accessor :line #: Integer? + attr_reader :rule_counter #: Counter + attr_reader :midrule_action_counter #: Counter + attr_reader :parameterized_resolver #: Grammar::Parameterized::Resolver + attr_reader :lhs_tag #: Lexer::Token::Tag? + attr_reader :rhs #: Array[Lexer::Token::Base] + attr_reader :user_code #: Lexer::Token::UserCode? + attr_reader :precedence_sym #: Grammar::Symbol? + + # @rbs (Counter rule_counter, Counter midrule_action_counter, Grammar::Parameterized::Resolver parameterized_resolver, ?Integer position_in_original_rule_rhs, ?lhs_tag: Lexer::Token::Tag?, ?skip_preprocess_references: bool) -> void + def initialize(rule_counter, midrule_action_counter, parameterized_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) @rule_counter = rule_counter @midrule_action_counter = midrule_action_counter - @parameterizing_rule_resolver = parameterizing_rule_resolver + @parameterized_resolver = parameterized_resolver @position_in_original_rule_rhs = position_in_original_rule_rhs @skip_preprocess_references = skip_preprocess_references @@ -20,12 +43,13 @@ module Lrama @precedence_sym = nil @line = nil @rules = [] - @rule_builders_for_parameterizing_rules = [] + @rule_builders_for_parameterized = [] @rule_builders_for_derived_rules = [] - @parameterizing_rules = [] + @parameterized_rules = [] @midrule_action_rules = [] end + # @rbs (Lexer::Token::Base rhs) -> void def add_rhs(rhs) @line ||= rhs.line @@ -34,6 +58,7 @@ module Lrama @rhs << rhs end + # @rbs (Lexer::Token::UserCode? user_code) -> void def user_code=(user_code) @line ||= user_code&.line @@ -42,72 +67,59 @@ module Lrama @user_code = user_code end + # @rbs (Grammar::Symbol? precedence_sym) -> void def precedence_sym=(precedence_sym) flush_user_code @precedence_sym = precedence_sym end + # @rbs () -> void def complete_input freeze_rhs end + # @rbs () -> void def setup_rules preprocess_references unless @skip_preprocess_references process_rhs + resolve_inline_rules build_rules end + # @rbs () -> Array[Grammar::Rule] def rules - @parameterizing_rules + @midrule_action_rules + @rules + @parameterized_rules + @midrule_action_rules + @rules end + # @rbs () -> bool def has_inline_rules? - rhs.any? { |token| @parameterizing_rule_resolver.find_inline(token) } - end - - def resolve_inline_rules - resolved_builders = [] #: Array[RuleBuilder] - rhs.each_with_index do |token, i| - if (inline_rule = @parameterizing_rule_resolver.find_inline(token)) - inline_rule.rhs_list.each do |inline_rhs| - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, lhs_tag: lhs_tag) - if token.is_a?(Lexer::Token::InstantiateRule) - resolve_inline_rhs(rule_builder, inline_rhs, i, Binding.new(inline_rule, token.args)) - else - resolve_inline_rhs(rule_builder, inline_rhs, i) - end - rule_builder.lhs = lhs - rule_builder.line = line - rule_builder.precedence_sym = precedence_sym - rule_builder.user_code = replace_inline_user_code(inline_rhs, i) - resolved_builders << rule_builder - end - break - end - end - resolved_builders + rhs.any? { |token| @parameterized_resolver.find_inline(token) } end private + # @rbs () -> void def freeze_rhs @rhs.freeze end + # @rbs () -> void def preprocess_references numberize_references end + # @rbs () -> void def build_rules - tokens = @replaced_rhs + tokens = @replaced_rhs #: Array[Lexer::Token::Base] + return if tokens.any? { |t| @parameterized_resolver.find_inline(t) } rule = Rule.new( id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code, position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line ) @rules = [rule] - @parameterizing_rules = @rule_builders_for_parameterizing_rules.map do |rule_builder| + @parameterized_rules = @rule_builders_for_parameterized.map do |rule_builder| rule_builder.rules end.flatten @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder| @@ -120,31 +132,33 @@ module Lrama # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on. # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`. + # + # @rbs () -> void def process_rhs return if @replaced_rhs - @replaced_rhs = [] + replaced_rhs = [] #: Array[Lexer::Token::Base] rhs.each_with_index do |token, i| case token when Lrama::Lexer::Token::Char - @replaced_rhs << token + replaced_rhs << token when Lrama::Lexer::Token::Ident - @replaced_rhs << token + replaced_rhs << token when Lrama::Lexer::Token::InstantiateRule - parameterizing_rule = @parameterizing_rule_resolver.find_rule(token) - raise "Unexpected token. #{token}" unless parameterizing_rule + parameterized_rule = @parameterized_resolver.find_rule(token) + raise "Unexpected token. #{token}" unless parameterized_rule - bindings = Binding.new(parameterizing_rule, token.args) - lhs_s_value = lhs_s_value(token, bindings) - if (created_lhs = @parameterizing_rule_resolver.created_lhs(lhs_s_value)) - @replaced_rhs << created_lhs + bindings = Binding.new(parameterized_rule.parameters, token.args) + lhs_s_value = bindings.concatenated_args_str(token) + if (created_lhs = @parameterized_resolver.created_lhs(lhs_s_value)) + replaced_rhs << created_lhs else lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) - @replaced_rhs << lhs_token - @parameterizing_rule_resolver.created_lhs_list << lhs_token - parameterizing_rule.rhs_list.each do |r| - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, lhs_tag: token.lhs_tag || parameterizing_rule.tag) + replaced_rhs << lhs_token + @parameterized_resolver.created_lhs_list << lhs_token + parameterized_rule.rhs.each do |r| + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, lhs_tag: token.lhs_tag || parameterized_rule.tag) rule_builder.lhs = lhs_token r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } rule_builder.line = line @@ -152,63 +166,48 @@ module Lrama rule_builder.user_code = r.resolve_user_code(bindings) rule_builder.complete_input rule_builder.setup_rules - @rule_builders_for_parameterizing_rules << rule_builder + @rule_builders_for_parameterized << rule_builder end end when Lrama::Lexer::Token::UserCode prefix = token.referred ? "@" : "$@" tag = token.tag || lhs_tag new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) - @replaced_rhs << new_token + replaced_rhs << new_token - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, i, lhs_tag: tag, skip_preprocess_references: true) + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, i, lhs_tag: tag, skip_preprocess_references: true) rule_builder.lhs = new_token rule_builder.user_code = token rule_builder.complete_input rule_builder.setup_rules @rule_builders_for_derived_rules << rule_builder + when Lrama::Lexer::Token::Empty + # Noop else raise "Unexpected token. #{token}" end end - end - def lhs_s_value(token, bindings) - s_values = token.args.map do |arg| - resolved = bindings.resolve_symbol(arg) - if resolved.is_a?(Lexer::Token::InstantiateRule) - [resolved.s_value, resolved.args.map(&:s_value)] - else - resolved.s_value - end - end - "#{token.rule_name}_#{s_values.join('_')}" + @replaced_rhs = replaced_rhs end - def resolve_inline_rhs(rule_builder, inline_rhs, index, bindings = nil) - rhs.each_with_index do |token, i| - if index == i - inline_rhs.symbols.each { |sym| rule_builder.add_rhs(bindings.nil? ? sym : bindings.resolve_symbol(sym)) } - else - rule_builder.add_rhs(token) + # @rbs () -> void + def resolve_inline_rules + while @rule_builders_for_parameterized.any?(&:has_inline_rules?) do + @rule_builders_for_parameterized = @rule_builders_for_parameterized.flat_map do |rule_builder| + if rule_builder.has_inline_rules? + inlined_builders = Inline::Resolver.new(rule_builder).resolve + inlined_builders.each { |builder| builder.setup_rules } + inlined_builders + else + rule_builder + end end end end - def replace_inline_user_code(inline_rhs, index) - return user_code if inline_rhs.user_code.nil? - return user_code if user_code.nil? - - code = user_code.s_value.gsub(/\$#{index + 1}/, inline_rhs.user_code.s_value) - user_code.references.each do |ref| - next if ref.index.nil? || ref.index <= index # nil is a case for `$$` - code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (inline_rhs.symbols.count-1)}") - code = code.gsub(/@#{ref.index}/, "@#{ref.index + (inline_rhs.symbols.count-1)}") - end - Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) - end - + # @rbs () -> void def numberize_references # Bison n'th component is 1-origin (rhs + [user_code]).compact.each.with_index(1) do |token, i| @@ -221,7 +220,10 @@ module Lrama if ref_name == '$' ref.name = '$' else - candidates = ([lhs] + rhs).each_with_index.select {|token, _i| token.referred_by?(ref_name) } + candidates = ([lhs] + rhs).each_with_index.select do |token, _i| + # @type var token: Lexer::Token::Base + token.referred_by?(ref_name) + end if candidates.size >= 2 token.invalid_ref(ref, "Referring symbol `#{ref_name}` is duplicated.") @@ -256,6 +258,7 @@ module Lrama end end + # @rbs () -> void def flush_user_code if (c = @user_code) @rhs << c diff --git a/tool/lrama/lib/lrama/grammar/stdlib.y b/tool/lrama/lib/lrama/grammar/stdlib.y index d6e89c908c..dd397c9e08 100644 --- a/tool/lrama/lib/lrama/grammar/stdlib.y +++ b/tool/lrama/lib/lrama/grammar/stdlib.y @@ -3,26 +3,43 @@ stdlib.y This is lrama's standard library. It provides a number of - parameterizing rule definitions, such as options and lists, + parameterized rule definitions, such as options and lists, that should be useful in a number of situations. **********************************************************************/ +%% + // ------------------------------------------------------------------- // Options /* - * program: option(number) + * program: option(X) + * + * => + * + * program: option_X + * option_X: %empty + * option_X: X + */ +%rule option(X) + : /* empty */ + | X + ; + + +/* + * program: ioption(X) * * => * - * program: option_number - * option_number: %empty - * option_number: number + * program: %empty + * program: X */ -%rule option(X): /* empty */ - | X - ; +%rule %inline ioption(X) + : /* empty */ + | X + ; // ------------------------------------------------------------------- // Sequences @@ -35,8 +52,9 @@ * program: preceded_opening_X * preceded_opening_X: opening X */ -%rule preceded(opening, X): opening X { $$ = $2; } - ; +%rule preceded(opening, X) + : opening X { $$ = $2; } + ; /* * program: terminated(X, closing) @@ -46,8 +64,9 @@ * program: terminated_X_closing * terminated_X_closing: X closing */ -%rule terminated(X, closing): X closing { $$ = $1; } - ; +%rule terminated(X, closing) + : X closing { $$ = $1; } + ; /* * program: delimited(opening, X, closing) @@ -57,66 +76,67 @@ * program: delimited_opening_X_closing * delimited_opening_X_closing: opening X closing */ -%rule delimited(opening, X, closing): opening X closing { $$ = $2; } - ; +%rule delimited(opening, X, closing) + : opening X closing { $$ = $2; } + ; // ------------------------------------------------------------------- // Lists /* - * program: list(number) + * program: list(X) * * => * - * program: list_number - * list_number: %empty - * list_number: list_number number + * program: list_X + * list_X: %empty + * list_X: list_X X */ -%rule list(X): /* empty */ - | list(X) X - ; +%rule list(X) + : /* empty */ + | list(X) X + ; /* - * program: nonempty_list(number) + * program: nonempty_list(X) * * => * - * program: nonempty_list_number - * nonempty_list_number: number - * nonempty_list_number: nonempty_list_number number + * program: nonempty_list_X + * nonempty_list_X: X + * nonempty_list_X: nonempty_list_X X */ -%rule nonempty_list(X): X - | nonempty_list(X) X - ; +%rule nonempty_list(X) + : X + | nonempty_list(X) X + ; /* - * program: separated_nonempty_list(comma, number) + * program: separated_nonempty_list(separator, X) * * => * - * program: separated_nonempty_list_comma_number - * separated_nonempty_list_comma_number: number - * separated_nonempty_list_comma_number: separated_nonempty_list_comma_number comma number + * program: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separated_nonempty_list_separator_X separator X */ -%rule separated_nonempty_list(separator, X): X - | separated_nonempty_list(separator, X) separator X - ; +%rule separated_nonempty_list(separator, X) + : X + | separated_nonempty_list(separator, X) separator X + ; /* - * program: separated_list(comma, number) + * program: separated_list(separator, X) * * => * - * program: separated_list_comma_number - * separated_list_comma_number: option_separated_nonempty_list_comma_number - * option_separated_nonempty_list_comma_number: %empty - * option_separated_nonempty_list_comma_number: separated_nonempty_list_comma_number - * separated_nonempty_list_comma_number: number - * separated_nonempty_list_comma_number: comma separated_nonempty_list_comma_number number + * program: separated_list_separator_X + * separated_list_separator_X: option_separated_nonempty_list_separator_X + * option_separated_nonempty_list_separator_X: %empty + * option_separated_nonempty_list_separator_X: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separator separated_nonempty_list_separator_X X */ -%rule separated_list(separator, X): option(separated_nonempty_list(separator, X)) - ; - -%% - -%union{}; +%rule separated_list(separator, X) + : option(separated_nonempty_list(separator, X)) + ; diff --git a/tool/lrama/lib/lrama/grammar/symbol.rb b/tool/lrama/lib/lrama/grammar/symbol.rb index f9dffcad6c..07aee0c0a2 100644 --- a/tool/lrama/lib/lrama/grammar/symbol.rb +++ b/tool/lrama/lib/lrama/grammar/symbol.rb @@ -1,19 +1,35 @@ +# rbs_inline: enabled # frozen_string_literal: true # Symbol is both of nterm and term # `number` is both for nterm and term # `token_id` is tokentype for term, internal sequence number for nterm # -# TODO: Add validation for ASCII code range for Token::Char module Lrama class Grammar class Symbol - attr_accessor :id, :alias_name, :tag, :number, :token_id, :nullable, :precedence, - :printer, :destructor, :error_token, :first_set, :first_set_bitmap - attr_reader :term - attr_writer :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + attr_accessor :id #: Lexer::Token::Base + attr_accessor :alias_name #: String? + attr_reader :number #: Integer + attr_accessor :number_bitmap #: Bitmap::bitmap + attr_accessor :tag #: Lexer::Token::Tag? + attr_accessor :token_id #: Integer + attr_accessor :nullable #: bool + attr_accessor :precedence #: Precedence? + attr_accessor :printer #: Printer? + attr_accessor :destructor #: Destructor? + attr_accessor :error_token #: ErrorToken + attr_accessor :first_set #: Set[Grammar::Symbol] + attr_accessor :first_set_bitmap #: Bitmap::bitmap + attr_reader :term #: bool + attr_writer :eof_symbol #: bool + attr_writer :error_symbol #: bool + attr_writer :undef_symbol #: bool + attr_writer :accept_symbol #: bool + # @rbs (id: Lexer::Token::Base, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token::Tag?, + # ?token_id: Integer?, ?nullable: bool?, ?precedence: Precedence?, ?printer: Printer?) -> void def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil) @id = id @alias_name = alias_name @@ -27,77 +43,105 @@ module Lrama @destructor = destructor end + # @rbs (Integer) -> void + def number=(number) + @number = number + @number_bitmap = Bitmap::from_integer(number) + end + + # @rbs () -> bool def term? term end + # @rbs () -> bool def nterm? !term end + # @rbs () -> bool def eof_symbol? !!@eof_symbol end + # @rbs () -> bool def error_symbol? !!@error_symbol end + # @rbs () -> bool def undef_symbol? !!@undef_symbol end + # @rbs () -> bool def accept_symbol? !!@accept_symbol end + # @rbs () -> bool + def midrule? + return false if term? + + name.include?("$") || name.include?("@") + end + + # @rbs () -> String + def name + id.s_value + end + + # @rbs () -> String def display_name - alias_name || id.s_value + alias_name || name end # name for yysymbol_kind_t # # See: b4_symbol_kind_base # @type var name: String + # @rbs () -> String def enum_name case when accept_symbol? - name = "YYACCEPT" + res = "YYACCEPT" when eof_symbol? - name = "YYEOF" + res = "YYEOF" when term? && id.is_a?(Lrama::Lexer::Token::Char) - name = number.to_s + display_name + res = number.to_s + display_name when term? && id.is_a?(Lrama::Lexer::Token::Ident) - name = id.s_value - when nterm? && (id.s_value.include?("$") || id.s_value.include?("@")) - name = number.to_s + id.s_value + res = name + when midrule? + res = number.to_s + name when nterm? - name = id.s_value + res = name else raise "Unexpected #{self}" end - "YYSYMBOL_" + name.gsub(/\W+/, "_") + "YYSYMBOL_" + res.gsub(/\W+/, "_") end # comment for yysymbol_kind_t + # + # @rbs () -> String? def comment case when accept_symbol? # YYSYMBOL_YYACCEPT - id.s_value + name when eof_symbol? # YYEOF alias_name when (term? && 0 < token_id && token_id < 128) # YYSYMBOL_3_backslash_, YYSYMBOL_14_ - alias_name || id.s_value - when id.s_value.include?("$") || id.s_value.include?("@") + display_name + when midrule? # YYSYMBOL_21_1 - id.s_value + name else # YYSYMBOL_keyword_class, YYSYMBOL_strings_1 - alias_name || id.s_value + display_name end end end diff --git a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb index 52f4ff90bd..085a835d28 100644 --- a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb +++ b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb @@ -1,24 +1,54 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Symbols class Resolver - attr_reader :terms, :nterms - + # @rbs! + # + # interface _DelegatedMethods + # def symbols: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def add_nterm: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + # def add_term: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + # def find_symbol_by_number!: (Integer number) -> Grammar::Symbol + # def find_symbol_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol + # def token_to_symbol: (Lexer::Token::Base token) -> Grammar::Symbol + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def fill_nterm_type: (Array[Grammar::Type] types) -> void + # def fill_symbol_number: () -> void + # def fill_printer: (Array[Grammar::Printer] printers) -> void + # def fill_destructor: (Array[Destructor] destructors) -> (Destructor | bot) + # def fill_error_token: (Array[Grammar::ErrorToken] error_tokens) -> void + # def sort_by_number!: () -> Array[Grammar::Symbol] + # end + # + # @symbols: Array[Grammar::Symbol]? + # @number: Integer + # @used_numbers: Hash[Integer, bool] + + attr_reader :terms #: Array[Grammar::Symbol] + attr_reader :nterms #: Array[Grammar::Symbol] + + # @rbs () -> void def initialize @terms = [] @nterms = [] end + # @rbs () -> Array[Grammar::Symbol] def symbols @symbols ||= (@terms + @nterms) end + # @rbs () -> Array[Grammar::Symbol] def sort_by_number! symbols.sort_by!(&:number) end + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) if token_id && (sym = find_symbol_by_token_id(token_id)) if replace @@ -43,6 +73,7 @@ module Lrama term end + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol def add_nterm(id:, alias_name: nil, tag: nil) if (sym = find_symbol_by_id(id)) return sym @@ -57,32 +88,39 @@ module Lrama nterm end + # @rbs (::String s_value) -> Grammar::Symbol? def find_term_by_s_value(s_value) terms.find { |s| s.id.s_value == s_value } end + # @rbs (::String s_value) -> Grammar::Symbol? def find_symbol_by_s_value(s_value) symbols.find { |s| s.id.s_value == s_value } end + # @rbs (::String s_value) -> Grammar::Symbol def find_symbol_by_s_value!(s_value) find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`") end + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol? def find_symbol_by_id(id) symbols.find do |s| s.id == id || s.alias_name == id.s_value end end + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol def find_symbol_by_id!(id) find_symbol_by_id(id) || (raise "Symbol not found. #{id}") end + # @rbs (Integer token_id) -> Grammar::Symbol? def find_symbol_by_token_id(token_id) symbols.find {|s| s.token_id == token_id } end + # @rbs (Integer number) -> Grammar::Symbol def find_symbol_by_number!(number) sym = symbols[number] @@ -92,6 +130,7 @@ module Lrama sym end + # @rbs () -> void def fill_symbol_number # YYEMPTY = -2 # YYEOF = 0 @@ -102,6 +141,7 @@ module Lrama fill_nterms_number end + # @rbs (Array[Grammar::Type] types) -> void def fill_nterm_type(types) types.each do |type| nterm = find_nterm_by_id!(type.id) @@ -109,6 +149,7 @@ module Lrama end end + # @rbs (Array[Grammar::Printer] printers) -> void def fill_printer(printers) symbols.each do |sym| printers.each do |printer| @@ -126,6 +167,7 @@ module Lrama end end + # @rbs (Array[Destructor] destructors) -> (Array[Grammar::Symbol] | bot) def fill_destructor(destructors) symbols.each do |sym| destructors.each do |destructor| @@ -143,6 +185,7 @@ module Lrama end end + # @rbs (Array[Grammar::ErrorToken] error_tokens) -> void def fill_error_token(error_tokens) symbols.each do |sym| error_tokens.each do |token| @@ -160,28 +203,33 @@ module Lrama end end + # @rbs (Lexer::Token::Base token) -> Grammar::Symbol def token_to_symbol(token) case token - when Lrama::Lexer::Token + when Lrama::Lexer::Token::Base find_symbol_by_id!(token) else raise "Unknown class: #{token}" end end + # @rbs () -> void def validate! validate_number_uniqueness! validate_alias_name_uniqueness! + validate_symbols! end private + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol def find_nterm_by_id!(id) @nterms.find do |s| s.id == id end || (raise "Symbol not found. #{id}") end + # @rbs () -> void def fill_terms_number # Character literal in grammar file has # token id corresponding to ASCII code by default, @@ -245,6 +293,7 @@ module Lrama end end + # @rbs () -> void def fill_nterms_number token_id = 0 @@ -266,6 +315,7 @@ module Lrama end end + # @rbs () -> Hash[Integer, bool] def used_numbers return @used_numbers if defined?(@used_numbers) @@ -276,6 +326,7 @@ module Lrama @used_numbers end + # @rbs () -> void def validate_number_uniqueness! invalid = symbols.group_by(&:number).select do |number, syms| syms.count > 1 @@ -286,6 +337,7 @@ module Lrama raise "Symbol number is duplicated. #{invalid}" end + # @rbs () -> void def validate_alias_name_uniqueness! invalid = symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| syms.count > 1 @@ -295,6 +347,15 @@ module Lrama raise "Symbol alias name is duplicated. #{invalid}" end + + # @rbs () -> void + def validate_symbols! + symbols.each { |sym| sym.id.validate } + errors = symbols.map { |sym| sym.id.errors }.flatten.compact + return if errors.empty? + + raise errors.join("\n") + end end end end diff --git a/tool/lrama/lib/lrama/grammar/type.rb b/tool/lrama/lib/lrama/grammar/type.rb index 65537288b3..c631769447 100644 --- a/tool/lrama/lib/lrama/grammar/type.rb +++ b/tool/lrama/lib/lrama/grammar/type.rb @@ -1,15 +1,27 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar class Type - attr_reader :id, :tag + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @id: Lexer::Token::Base + # @tag: Lexer::Token::Tag + attr_reader :id #: Lexer::Token::Base + attr_reader :tag #: Lexer::Token::Tag + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> void def initialize(id:, tag:) @id = id @tag = tag end + # @rbs (Grammar::Type other) -> bool def ==(other) self.class == other.class && self.id == other.id && diff --git a/tool/lrama/lib/lrama/grammar/union.rb b/tool/lrama/lib/lrama/grammar/union.rb index 5f1bee0069..774cc66fc6 100644 --- a/tool/lrama/lib/lrama/grammar/union.rb +++ b/tool/lrama/lib/lrama/grammar/union.rb @@ -1,8 +1,19 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Grammar - class Union < Struct.new(:code, :lineno, keyword_init: true) + class Union + attr_reader :code #: Grammar::Code::NoReferenceCode + attr_reader :lineno #: Integer + + # @rbs (code: Grammar::Code::NoReferenceCode, lineno: Integer) -> void + def initialize(code:, lineno:) + @code = code + @lineno = lineno + end + + # @rbs () -> String def braces_less_code # Braces is already removed by lexer code.s_value diff --git a/tool/lrama/lib/lrama/grammar_validator.rb b/tool/lrama/lib/lrama/grammar_validator.rb deleted file mode 100644 index 7790499589..0000000000 --- a/tool/lrama/lib/lrama/grammar_validator.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class GrammarValidator - def initialize(grammar, states, logger) - @grammar = grammar - @states = states - @logger = logger - end - - def valid? - conflicts_within_threshold? - end - - private - - def conflicts_within_threshold? - return true unless @grammar.expect - - [sr_conflicts_within_threshold(@grammar.expect), rr_conflicts_within_threshold(0)].all? - end - - def sr_conflicts_within_threshold(expected) - return true if expected == @states.sr_conflicts_count - - @logger.error("shift/reduce conflicts: #{@states.sr_conflicts_count} found, #{expected} expected") - false - end - - def rr_conflicts_within_threshold(expected) - return true if expected == @states.rr_conflicts_count - - @logger.error("reduce/reduce conflicts: #{@states.rr_conflicts_count} found, #{expected} expected") - false - end - end -end diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb index 8146628656..ce98b505a7 100644 --- a/tool/lrama/lib/lrama/lexer.rb +++ b/tool/lrama/lib/lrama/lexer.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true require "strscan" @@ -8,10 +9,26 @@ require_relative "lexer/token" module Lrama class Lexer - attr_reader :head_line, :head_column, :line - attr_accessor :status, :end_symbol - - SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze + # @rbs! + # + # type token = lexer_token | c_token + # + # type lexer_token = [String, Token::Token] | + # [::Symbol, Token::Tag] | + # [::Symbol, Token::Char] | + # [::Symbol, Token::Str] | + # [::Symbol, Token::Int] | + # [::Symbol, Token::Ident] + # + # type c_token = [:C_DECLARATION, Token::UserCode] + + attr_reader :head_line #: Integer + attr_reader :head_column #: Integer + attr_reader :line #: Integer + attr_accessor :status #: :initial | :c_declaration + attr_accessor :end_symbol #: String? + + SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze #: Array[String] PERCENT_TOKENS = %w( %union %token @@ -42,8 +59,11 @@ module Lrama %no-stdlib %inline %locations - ).freeze + %categories + %start + ).freeze #: Array[String] + # @rbs (GrammarFile grammar_file) -> void def initialize(grammar_file) @grammar_file = grammar_file @scanner = StringScanner.new(grammar_file.text) @@ -53,6 +73,7 @@ module Lrama @end_symbol = nil end + # @rbs () -> token? def next_token case @status when :initial @@ -62,10 +83,12 @@ module Lrama end end + # @rbs () -> Integer def column @scanner.pos - @head end + # @rbs () -> Location def location Location.new( grammar_file: @grammar_file, @@ -74,13 +97,14 @@ module Lrama ) end + # @rbs () -> lexer_token? def lex_token until @scanner.eos? do case when @scanner.scan(/\n/) newline when @scanner.scan(/\s+/) - # noop + @scanner.matched.count("\n").times { newline } when @scanner.scan(/\/\*/) lex_comment when @scanner.scan(/\/\/.*(?<newline>\n)?/) @@ -96,11 +120,11 @@ module Lrama when @scanner.eos? return when @scanner.scan(/#{SYMBOLS.join('|')}/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/[\?\+\*]/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/<\w+>/) return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/'.'/) @@ -108,9 +132,9 @@ module Lrama when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/".*?"/) - return [:STRING, %Q(#{@scanner.matched})] + return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] when @scanner.scan(/\d+/) - return [:INTEGER, Integer(@scanner.matched)] + return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)] when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location) type = @@ -121,69 +145,72 @@ module Lrama end return [type, token] else - raise ParseError, "Unexpected token: #{@scanner.peek(10).chomp}." + raise ParseError, location.generate_error_message("Unexpected token") # steep:ignore UnknownConstant end end + # @rbs () -> c_token def lex_c_code nested = 0 - code = '' + code = +'' reset_first_position until @scanner.eos? do case when @scanner.scan(/{/) - code += @scanner.matched + code << @scanner.matched nested += 1 when @scanner.scan(/}/) if nested == 0 && @end_symbol == '}' @scanner.unscan return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] else - code += @scanner.matched + code << @scanner.matched nested -= 1 end when @scanner.check(/#{@end_symbol}/) return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] when @scanner.scan(/\n/) - code += @scanner.matched + code << @scanner.matched newline when @scanner.scan(/".*?"/) - code += %Q(#{@scanner.matched}) + code << %Q(#{@scanner.matched}) @line += @scanner.matched.count("\n") when @scanner.scan(/'.*?'/) - code += %Q(#{@scanner.matched}) + code << %Q(#{@scanner.matched}) when @scanner.scan(/[^\"'\{\}\n]+/) - code += @scanner.matched - when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) - code += @scanner.matched + code << @scanner.matched + when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) # steep:ignore + code << @scanner.matched else - code += @scanner.getch + code << @scanner.getch end end - raise ParseError, "Unexpected code: #{code}." + raise ParseError, location.generate_error_message("Unexpected code: #{code}") # steep:ignore UnknownConstant end private + # @rbs () -> void def lex_comment until @scanner.eos? do case - when @scanner.scan(/\n/) - newline - when @scanner.scan(/\*\//) + when @scanner.scan_until(/[\s\S]*?\*\//) + @scanner.matched.count("\n").times { newline } return - else - @scanner.getch + when @scanner.scan_until(/\n/) + newline end end end + # @rbs () -> void def reset_first_position @head_line = line @head_column = column end + # @rbs () -> void def newline @line += 1 @head = @scanner.pos diff --git a/tool/lrama/lib/lrama/lexer/grammar_file.rb b/tool/lrama/lib/lrama/lexer/grammar_file.rb index 45c3122975..37e82ff18d 100644 --- a/tool/lrama/lib/lrama/lexer/grammar_file.rb +++ b/tool/lrama/lib/lrama/lexer/grammar_file.rb @@ -1,30 +1,37 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Lexer class GrammarFile class Text < String + # @rbs () -> String def inspect length <= 50 ? super : "#{self[0..47]}...".inspect end end - attr_reader :path, :text + attr_reader :path #: String + attr_reader :text #: String + # @rbs (String path, String text) -> void def initialize(path, text) @path = path @text = Text.new(text).freeze end + # @rbs () -> String def inspect "<#{self.class}: @path=#{path}, @text=#{text.inspect}>" end + # @rbs (GrammarFile other) -> bool def ==(other) self.class == other.class && self.path == other.path end + # @rbs () -> Array[String] def lines @lines ||= text.split("\n") end diff --git a/tool/lrama/lib/lrama/lexer/location.rb b/tool/lrama/lib/lrama/lexer/location.rb index bf8f4f7e3e..4465576d53 100644 --- a/tool/lrama/lib/lrama/lexer/location.rb +++ b/tool/lrama/lib/lrama/lexer/location.rb @@ -1,10 +1,16 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Lexer class Location - attr_reader :grammar_file, :first_line, :first_column, :last_line, :last_column + attr_reader :grammar_file #: GrammarFile + attr_reader :first_line #: Integer + attr_reader :first_column #: Integer + attr_reader :last_line #: Integer + attr_reader :last_column #: Integer + # @rbs (grammar_file: GrammarFile, first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:) @grammar_file = grammar_file @first_line = first_line @@ -13,6 +19,7 @@ module Lrama @last_column = last_column end + # @rbs (Location other) -> bool def ==(other) self.class == other.class && self.grammar_file == other.grammar_file && @@ -22,6 +29,7 @@ module Lrama self.last_column == other.last_column end + # @rbs (Integer left, Integer right) -> Location def partial_location(left, right) offset = -first_column new_first_line = -1 @@ -52,42 +60,67 @@ module Lrama ) end + # @rbs () -> String def to_s "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})" end + # @rbs (String error_message) -> String def generate_error_message(error_message) <<~ERROR.chomp #{path}:#{first_line}:#{first_column}: #{error_message} - #{line_with_carets} + #{error_with_carets} ERROR end - def line_with_carets + # @rbs () -> String + def error_with_carets <<~TEXT - #{text} - #{carets} + #{formatted_first_lineno} | #{text} + #{line_number_padding} | #{carets_line} TEXT end private + # @rbs () -> String def path grammar_file.path end - def blanks - (text[0...first_column] or raise "#{first_column} is invalid").gsub(/[^\t]/, ' ') + # @rbs () -> String + def carets_line + leading_whitespace + highlight_marker end - def carets - blanks + '^' * (last_column - first_column) + # @rbs () -> String + def leading_whitespace + (text[0...first_column] or raise "Invalid first_column: #{first_column}") + .gsub(/[^\t]/, ' ') end + # @rbs () -> String + def highlight_marker + length = last_column - first_column + '^' + '~' * [0, length - 1].max + end + + # @rbs () -> String + def formatted_first_lineno + first_line.to_s.rjust(4) + end + + # @rbs () -> String + def line_number_padding + ' ' * formatted_first_lineno.length + end + + # @rbs () -> String def text @text ||= _text.join("\n") end + # @rbs () -> Array[String] def _text @_text ||=begin range = (first_line - 1)...last_line diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb index 45a097f682..37f77aa069 100644 --- a/tool/lrama/lib/lrama/lexer/token.rb +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -1,58 +1,20 @@ +# rbs_inline: enabled # frozen_string_literal: true +require_relative 'token/base' require_relative 'token/char' +require_relative 'token/empty' require_relative 'token/ident' require_relative 'token/instantiate_rule' +require_relative 'token/int' +require_relative 'token/str' require_relative 'token/tag' +require_relative 'token/token' require_relative 'token/user_code' module Lrama class Lexer - class Token - attr_reader :s_value, :location - attr_accessor :alias_name, :referred - - def initialize(s_value:, alias_name: nil, location: nil) - s_value.freeze - @s_value = s_value - @alias_name = alias_name - @location = location - end - - def to_s - "value: `#{s_value}`, location: #{location}" - end - - def referred_by?(string) - [self.s_value, self.alias_name].compact.include?(string) - end - - def ==(other) - self.class == other.class && self.s_value == other.s_value - end - - def first_line - location.first_line - end - alias :line :first_line - - def first_column - location.first_column - end - alias :column :first_column - - def last_line - location.last_line - end - - def last_column - location.last_column - end - - def invalid_ref(ref, message) - location = self.location.partial_location(ref.first_column, ref.last_column) - raise location.generate_error_message(message) - end + module Token end end end diff --git a/tool/lrama/lib/lrama/lexer/token/base.rb b/tool/lrama/lib/lrama/lexer/token/base.rb new file mode 100644 index 0000000000..3df93bbc73 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/base.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Base + attr_reader :s_value #: String + attr_reader :location #: Location + attr_accessor :alias_name #: String + attr_accessor :referred #: bool + attr_reader :errors #: Array[String] + + # @rbs (s_value: String, ?alias_name: String, ?location: Location) -> void + def initialize(s_value:, alias_name: nil, location: nil) + s_value.freeze + @s_value = s_value + @alias_name = alias_name + @location = location + @errors = [] + end + + # @rbs () -> String + def to_s + "value: `#{s_value}`, location: #{location}" + end + + # @rbs (String string) -> bool + def referred_by?(string) + [self.s_value, self.alias_name].compact.include?(string) + end + + # @rbs (Lexer::Token::Base other) -> bool + def ==(other) + self.class == other.class && self.s_value == other.s_value + end + + # @rbs () -> Integer + def first_line + location.first_line + end + alias :line :first_line + + # @rbs () -> Integer + def first_column + location.first_column + end + alias :column :first_column + + # @rbs () -> Integer + def last_line + location.last_line + end + + # @rbs () -> Integer + def last_column + location.last_column + end + + # @rbs (Lrama::Grammar::Reference ref, String message) -> bot + def invalid_ref(ref, message) + location = self.location.partial_location(ref.first_column, ref.last_column) + raise location.generate_error_message(message) + end + + # @rbs () -> bool + def validate + true + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/char.rb b/tool/lrama/lib/lrama/lexer/token/char.rb index 9e21952c42..f4ef7c9fbc 100644 --- a/tool/lrama/lib/lrama/lexer/token/char.rb +++ b/tool/lrama/lib/lrama/lexer/token/char.rb @@ -1,9 +1,23 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Lexer - class Token - class Char < Token + module Token + class Char < Base + # @rbs () -> void + def validate + validate_ascii_code_range + end + + private + + # @rbs () -> void + def validate_ascii_code_range + unless s_value.ascii_only? + errors << "Invalid character: `#{s_value}`. Only ASCII characters are allowed." + end + end end end end diff --git a/tool/lrama/lib/lrama/lexer/token/empty.rb b/tool/lrama/lib/lrama/lexer/token/empty.rb new file mode 100644 index 0000000000..375e256493 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/empty.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Empty < Base + def initialize(location: nil) + super(s_value: '%empty', location: location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/ident.rb b/tool/lrama/lib/lrama/lexer/token/ident.rb index 84835c00bc..4880be9073 100644 --- a/tool/lrama/lib/lrama/lexer/token/ident.rb +++ b/tool/lrama/lib/lrama/lexer/token/ident.rb @@ -1,9 +1,10 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Lexer - class Token - class Ident < Token + module Token + class Ident < Base end end end diff --git a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb index db7e611c5f..7051ba75a4 100644 --- a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb +++ b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb @@ -1,21 +1,26 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Lexer - class Token - class InstantiateRule < Token - attr_reader :args, :lhs_tag + module Token + class InstantiateRule < Base + attr_reader :args #: Array[Lexer::Token::Base] + attr_reader :lhs_tag #: Lexer::Token::Tag? + # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token::Base], ?lhs_tag: Lexer::Token::Tag?) -> void def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil) super s_value: s_value, alias_name: alias_name, location: location @args = args @lhs_tag = lhs_tag end + # @rbs () -> String def rule_name s_value end + # @rbs () -> Integer def args_count args.count end diff --git a/tool/lrama/lib/lrama/lexer/token/int.rb b/tool/lrama/lib/lrama/lexer/token/int.rb new file mode 100644 index 0000000000..7daf48d4d3 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/int.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Int < Base + # @rbs! + # def initialize: (s_value: Integer, ?alias_name: String, ?location: Location) -> void + # def s_value: () -> Integer + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/str.rb b/tool/lrama/lib/lrama/lexer/token/str.rb new file mode 100644 index 0000000000..cf9de6cf0f --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/str.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Str < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/tag.rb b/tool/lrama/lib/lrama/lexer/token/tag.rb index 52dcb50ce7..68c6268219 100644 --- a/tool/lrama/lib/lrama/lexer/token/tag.rb +++ b/tool/lrama/lib/lrama/lexer/token/tag.rb @@ -1,11 +1,13 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Lexer - class Token - class Tag < Token - # Omit "<>" + module Token + class Tag < Base + # @rbs () -> String def member + # Omit "<>" s_value[1..-2] or raise "Unexpected Tag format (#{s_value})" end end diff --git a/tool/lrama/lib/lrama/lexer/token/token.rb b/tool/lrama/lib/lrama/lexer/token/token.rb new file mode 100644 index 0000000000..935797efc6 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/token.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Token < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb index c606200d7a..166f04954a 100644 --- a/tool/lrama/lib/lrama/lexer/token/user_code.rb +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -1,19 +1,22 @@ +# rbs_inline: enabled # frozen_string_literal: true require "strscan" module Lrama class Lexer - class Token - class UserCode < Token - attr_accessor :tag + module Token + class UserCode < Base + attr_accessor :tag #: Lexer::Token::Tag + # @rbs () -> Array[Lrama::Grammar::Reference] def references @references ||= _references end private + # @rbs () -> Array[Lrama::Grammar::Reference] def _references scanner = StringScanner.new(s_value) references = [] #: Array[Grammar::Reference] @@ -32,45 +35,72 @@ module Lrama references end + # @rbs (StringScanner scanner) -> Lrama::Grammar::Reference? def scan_reference(scanner) start = scanner.pos - case - # $ references - # It need to wrap an identifier with brackets to use ".-" for identifiers - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$ - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1 - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[2]), index: Integer(scanner[2]), ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets) - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos) - - # @ references - # It need to wrap an identifier with brackets to use ".-" for identifiers - when scanner.scan(/@\$/) # @$ - return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) - when scanner.scan(/@(\d+)/) # @1 - return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[1]), index: Integer(scanner[1]), first_column: start, last_column: scanner.pos) - when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets) - return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos) - when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @[expr.right], @[expr-right] (named reference with brackets) - return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos) + if scanner.scan(/ + # $ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + \$(<[a-zA-Z0-9_]+>)?(?: + (\$) # $$, $<long>$ + | (\d+) # $1, $2, $<long>1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $foo, $expr, $<long>program (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + ) + | + # @ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + @(?: + (\$) # @$ + | (\d+) # @1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # @foo, @expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # @[expr.right], @[expr-right] (named reference with brackets) + ) + | + # $: references + \$: + (?: + (\$) # $:$ + | (\d+) # $:1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $:foo, $:expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $:[expr.right], $:[expr-right] (named reference with brackets) + ) + /x) + case + # $ references + when scanner[2] # $$, $<long>$ + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[3] # $1, $2, $<long>1 + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[3]), index: Integer(scanner[3]), ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[4] # $foo, $expr, $<long>program (named reference without brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[4], ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[5] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[5], ex_tag: tag, first_column: start, last_column: scanner.pos) - # $: references - when scanner.scan(/\$:\$/) # $:$ - return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) - when scanner.scan(/\$:(\d+)/) # $:1 - return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[1]), first_column: start, last_column: scanner.pos) - when scanner.scan(/\$:([a-zA-Z_][a-zA-Z0-9_]*)/) # $:foo, $:expr (named reference without brackets) - return Lrama::Grammar::Reference.new(type: :index, name: scanner[1], first_column: start, last_column: scanner.pos) - when scanner.scan(/\$:\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $:[expr.right], $:[expr-right] (named reference with brackets) - return Lrama::Grammar::Reference.new(type: :index, name: scanner[1], first_column: start, last_column: scanner.pos) + # @ references + when scanner[6] # @$ + return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) + when scanner[7] # @1 + return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[7]), index: Integer(scanner[7]), first_column: start, last_column: scanner.pos) + when scanner[8] # @foo, @expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[8], first_column: start, last_column: scanner.pos) + when scanner[9] # @[expr.right], @[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[9], first_column: start, last_column: scanner.pos) + # $: references + when scanner[10] # $:$ + return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) + when scanner[11] # $:1 + return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[11]), index: Integer(scanner[11]), first_column: start, last_column: scanner.pos) + when scanner[12] # $:foo, $:expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[12], first_column: start, last_column: scanner.pos) + when scanner[13] # $:[expr.right], $:[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[13], first_column: start, last_column: scanner.pos) + end end end end diff --git a/tool/lrama/lib/lrama/logger.rb b/tool/lrama/lib/lrama/logger.rb index e98eef0fa5..291eea5296 100644 --- a/tool/lrama/lib/lrama/logger.rb +++ b/tool/lrama/lib/lrama/logger.rb @@ -1,17 +1,31 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class Logger + # @rbs (IO out) -> void def initialize(out = STDERR) @out = out end - def warn(message) + # @rbs () -> void + def line_break + @out << "\n" + end + + # @rbs (String message) -> void + def trace(message) @out << message << "\n" end + # @rbs (String message) -> void + def warn(message) + @out << 'warning: ' << message << "\n" + end + + # @rbs (String message) -> void def error(message) - @out << message << "\n" + @out << 'error: ' << message << "\n" end end end diff --git a/tool/lrama/lib/lrama/option_parser.rb b/tool/lrama/lib/lrama/option_parser.rb index 0727d1b37f..5a15d59c7b 100644 --- a/tool/lrama/lib/lrama/option_parser.rb +++ b/tool/lrama/lib/lrama/option_parser.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true require 'optparse' @@ -5,17 +6,32 @@ require 'optparse' module Lrama # Handle option parsing for the command line interface. class OptionParser + # @rbs! + # @options: Lrama::Options + # @trace: Array[String] + # @report: Array[String] + # @profile: Array[String] + + # @rbs (Array[String]) -> Lrama::Options + def self.parse(argv) + new.parse(argv) + end + + # @rbs () -> void def initialize @options = Options.new @trace = [] @report = [] + @profile = [] end + # @rbs (Array[String]) -> Lrama::Options def parse(argv) parse_by_option_parser(argv) @options.trace_opts = validate_trace(@trace) @options.report_opts = validate_report(@report) + @options.profile_opts = validate_profile(@profile) @options.grammar_file = argv.shift unless @options.grammar_file @@ -46,6 +62,7 @@ module Lrama private + # @rbs (Array[String]) -> void def parse_by_option_parser(argv) ::OptionParser.new do |o| o.banner = <<~BANNER @@ -59,8 +76,15 @@ module Lrama o.separator '' o.separator 'Tuning the Parser:' o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v } - o.on('-t', 'reserved, do nothing') { } - o.on('--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.on('-t', '--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.separator " same as '-Dparse.trace'" + o.on('--locations', 'enable location support') {|v| @options.locations = true } + o.on('-D', '--define=NAME[=VALUE]', Array, "similar to '%define NAME VALUE'") do |v| + @options.define = v.each_with_object({}) do |item, hash| # steep:ignore UnannotatedEmptyCollection + key, value = item.split('=', 2) + hash[key] = value + end + end o.separator '' o.separator 'Output:' o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v } @@ -86,14 +110,24 @@ module Lrama o.on_tail ' automaton display states' o.on_tail ' closure display states' o.on_tail ' rules display grammar rules' + o.on_tail ' only-explicit-rules display only explicit grammar rules' o.on_tail ' actions display grammar rules with actions' o.on_tail ' time display generation time' o.on_tail ' all include all the above traces' o.on_tail ' none disable all traces' + o.on('--diagram=[FILE]', 'generate a diagram of the rules') do |v| + @options.diagram = true + @options.diagram_file = v if v + end + o.on('--profile=PROFILES', Array, 'profiles parser generation parts') {|v| @profile = v } + o.on_tail '' + o.on_tail 'PROFILES is a list of comma-separated words that can include:' + o.on_tail ' call-stack use sampling call-stack profiler (stackprof gem)' + o.on_tail ' memory use memory profiler (memory_profiler gem)' o.on('-v', '--verbose', "same as '--report=state'") {|_v| @report << 'states' } o.separator '' o.separator 'Diagnostics:' - o.on('-W', '--warnings', 'report the warnings') {|v| @options.diagnostic = true } + o.on('-W', '--warnings', 'report the warnings') {|v| @options.warnings = true } o.separator '' o.separator 'Error Recovery:' o.on('-e', 'enable error recovery') {|v| @options.error_recovery = true } @@ -106,9 +140,10 @@ module Lrama end end - ALIASED_REPORTS = { cex: :counterexamples }.freeze - VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze + ALIASED_REPORTS = { cex: :counterexamples }.freeze #: Hash[Symbol, Symbol] + VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze #: Array[Symbol] + # @rbs (Array[String]) -> Hash[Symbol, bool] def validate_report(report) h = { grammar: true } return h if report.empty? @@ -130,34 +165,55 @@ module Lrama return h end + # @rbs (String) -> Symbol def aliased_report_option(opt) (ALIASED_REPORTS[opt.to_sym] || opt).to_sym end VALID_TRACES = %w[ locations scan parse automaton bitsets closure - grammar rules actions resource sets muscles - tools m4-early m4 skeleton time ielr cex - ].freeze + grammar rules only-explicit-rules actions resource + sets muscles tools m4-early m4 skeleton time ielr cex + ].freeze #: Array[String] NOT_SUPPORTED_TRACES = %w[ locations scan parse bitsets grammar resource sets muscles tools m4-early m4 skeleton ielr cex - ].freeze + ].freeze #: Array[String] + SUPPORTED_TRACES = VALID_TRACES - NOT_SUPPORTED_TRACES #: Array[String] + # @rbs (Array[String]) -> Hash[Symbol, bool] def validate_trace(trace) - h = {} + h = {} #: Hash[Symbol, bool] return h if trace.empty? || trace == ['none'] - supported = VALID_TRACES - NOT_SUPPORTED_TRACES + all_traces = SUPPORTED_TRACES - %w[only-explicit-rules] if trace == ['all'] - supported.each { |t| h[t.to_sym] = true } + all_traces.each { |t| h[t.gsub(/-/, '_').to_sym] = true } return h end trace.each do |t| - if supported.include?(t) - h[t.to_sym] = true + if SUPPORTED_TRACES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid trace option \"#{t}\".\nValid options are [#{SUPPORTED_TRACES.join(", ")}]." + end + end + + return h + end + + VALID_PROFILES = %w[call-stack memory].freeze #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_profile(profile) + h = {} #: Hash[Symbol, bool] + return h if profile.empty? + + profile.each do |t| + if VALID_PROFILES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true else - raise "Invalid trace option \"#{t}\"." + raise "Invalid profile option \"#{t}\".\nValid options are [#{VALID_PROFILES.join(", ")}]." end end diff --git a/tool/lrama/lib/lrama/options.rb b/tool/lrama/lib/lrama/options.rb index ccd7680348..87aec62448 100644 --- a/tool/lrama/lib/lrama/options.rb +++ b/tool/lrama/lib/lrama/options.rb @@ -1,27 +1,46 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama # Command line options. class Options - attr_accessor :skeleton, :header, :header_file, - :report_file, :outfile, - :error_recovery, :grammar_file, - :trace_opts, :report_opts, - :diagnostic, :y, :debug + attr_accessor :skeleton #: String + attr_accessor :locations #: bool + attr_accessor :header #: bool + attr_accessor :header_file #: String? + attr_accessor :report_file #: String? + attr_accessor :outfile #: String + attr_accessor :error_recovery #: bool + attr_accessor :grammar_file #: String + attr_accessor :trace_opts #: Hash[Symbol, bool]? + attr_accessor :report_opts #: Hash[Symbol, bool]? + attr_accessor :warnings #: bool + attr_accessor :y #: IO + attr_accessor :debug #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :diagram #: bool + attr_accessor :diagram_file #: String + attr_accessor :profile_opts #: Hash[Symbol, bool]? + # @rbs () -> void def initialize @skeleton = "bison/yacc.c" + @locations = false + @define = {} @header = false @header_file = nil @report_file = nil @outfile = "y.tab.c" @error_recovery = false - @grammar_file = nil + @grammar_file = '' @trace_opts = nil @report_opts = nil - @diagnostic = false + @warnings = false @y = STDIN @debug = false + @diagram = false + @diagram_file = "diagram.html" + @profile_opts = nil end end end diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb index 3c7316ac6d..d527be8bd4 100644 --- a/tool/lrama/lib/lrama/output.rb +++ b/tool/lrama/lib/lrama/output.rb @@ -1,13 +1,12 @@ # frozen_string_literal: true -require "erb" require "forwardable" -require_relative "report/duration" +require_relative "tracer/duration" module Lrama class Output extend Forwardable - include Report::Duration + include Tracer::Duration attr_reader :grammar_file_path, :context, :grammar, :error_recovery, :include_header @@ -43,7 +42,7 @@ module Lrama end def render_partial(file) - render_template(partial_file(file)) + ERB.render(partial_file(file), context: @context, output: self) end def render @@ -405,16 +404,10 @@ module Lrama private def eval_template(file, path) - tmp = render_template(file) + tmp = ERB.render(file, context: @context, output: self) replace_special_variables(tmp, path) end - def render_template(file) - erb = self.class.erb(File.read(file)) - erb.filename = file - erb.result_with_hash(context: @context, output: self) - end - def template_file File.join(template_dir, @template_name) end diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb index 237a34d6a8..20c3ad347f 100644 --- a/tool/lrama/lib/lrama/parser.rb +++ b/tool/lrama/lib/lrama/parser.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true # # DO NOT MODIFY!!!! # This file is automatically generated by Racc 1.8.1 @@ -654,21 +655,25 @@ end module Lrama class Parser < Racc::Parser -module_eval(<<'...end parser.y/module_eval...', 'parser.y', 428) +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 504) -include Lrama::Report::Duration +include Lrama::Tracer::Duration -def initialize(text, path, debug = false) +def initialize(text, path, debug = false, locations = false, define = {}) + @path = path @grammar_file = Lrama::Lexer::GrammarFile.new(path, text) - @yydebug = debug + @yydebug = debug || define.key?('parse.trace') @rule_counter = Lrama::Grammar::Counter.new(0) @midrule_action_counter = Lrama::Grammar::Counter.new(1) + @locations = locations + @define = define end def parse - report_duration(:parse) do + message = "parse '#{File.basename(@path)}'" + report_duration(message) do @lexer = Lrama::Lexer.new(@grammar_file) - @grammar = Lrama::Grammar.new(@rule_counter) + @grammar = Lrama::Grammar.new(@rule_counter, @locations, @define) @precedence_number = 0 reset_precs do_parse @@ -681,7 +686,14 @@ def next_token end def on_error(error_token_id, error_value, value_stack) - if error_value.is_a?(Lrama::Lexer::Token) + case error_value + when Lrama::Lexer::Token::Int + location = error_value.location + value = "#{error_value.s_value}" + when Lrama::Lexer::Token::Token + location = error_value.location + value = "\"#{error_value.s_value}\"" + when Lrama::Lexer::Token::Base location = error_value.location value = "'#{error_value.s_value}'" else @@ -695,7 +707,7 @@ def on_error(error_token_id, error_value, value_stack) end def on_action_error(error_message, error_value) - if error_value.is_a?(Lrama::Lexer::Token) + if error_value.is_a?(Lrama::Lexer::Token::Base) location = error_value.location else location = @lexer.location @@ -707,10 +719,15 @@ end private def reset_precs - @prec_seen = false + @opening_prec_seen = false + @trailing_prec_seen = false @code_after_prec = false end +def prec_seen? + @opening_prec_seen || @trailing_prec_seen +end + def begin_c_declaration(end_symbol) @lexer.status = :c_declaration @lexer.end_symbol = end_symbol @@ -728,306 +745,322 @@ end ##### State transition tables begin ### racc_action_table = [ - 89, 49, 90, 167, 49, 101, 173, 49, 101, 167, - 49, 101, 173, 6, 101, 80, 49, 49, 48, 48, - 41, 76, 76, 49, 49, 48, 48, 42, 76, 76, - 49, 49, 48, 48, 101, 96, 113, 49, 87, 48, - 150, 101, 96, 151, 45, 171, 169, 170, 151, 176, - 170, 91, 169, 170, 81, 176, 170, 20, 24, 25, - 26, 27, 28, 29, 30, 31, 87, 32, 33, 34, - 35, 36, 37, 38, 39, 49, 4, 48, 5, 101, - 96, 181, 182, 183, 128, 20, 24, 25, 26, 27, - 28, 29, 30, 31, 46, 32, 33, 34, 35, 36, - 37, 38, 39, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 53, 20, 24, 25, 26, 27, 28, 29, - 30, 31, 53, 32, 33, 34, 35, 36, 37, 38, - 39, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 44, 20, 24, 25, 26, 27, 28, 29, 30, 31, - 53, 32, 33, 34, 35, 36, 37, 38, 39, 49, - 4, 48, 5, 101, 96, 49, 49, 48, 48, 101, - 101, 49, 49, 48, 48, 101, 101, 49, 49, 48, - 197, 101, 101, 49, 49, 197, 48, 101, 101, 49, - 49, 197, 48, 101, 181, 182, 183, 128, 204, 210, - 217, 205, 205, 205, 49, 49, 48, 48, 49, 49, - 48, 48, 49, 49, 48, 48, 181, 182, 183, 116, - 117, 56, 53, 53, 53, 53, 53, 62, 63, 64, - 65, 66, 68, 68, 68, 82, 53, 53, 104, 108, - 108, 115, 122, 123, 125, 128, 129, 133, 139, 140, - 141, 142, 144, 145, 101, 154, 139, 157, 154, 161, - 162, 68, 164, 165, 172, 177, 154, 184, 128, 188, - 154, 190, 128, 154, 199, 154, 128, 68, 165, 206, - 165, 68, 68, 215, 128, 68 ] + 98, 98, 99, 99, 87, 53, 53, 52, 178, 110, + 110, 97, 53, 53, 184, 178, 110, 110, 53, 181, + 184, 162, 110, 6, 163, 181, 181, 53, 53, 52, + 52, 181, 79, 79, 53, 53, 52, 52, 43, 79, + 79, 53, 4, 52, 5, 110, 88, 94, 182, 125, + 126, 163, 100, 100, 180, 193, 194, 195, 137, 185, + 188, 180, 4, 44, 5, 185, 188, 94, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 46, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 47, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 47, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 50, + 57, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 57, + 60, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 53, 53, 52, + 52, 110, 105, 53, 53, 52, 52, 110, 105, 53, + 53, 52, 52, 110, 105, 53, 53, 52, 52, 110, + 105, 53, 53, 52, 52, 110, 110, 53, 53, 52, + 209, 110, 110, 53, 53, 209, 52, 110, 110, 53, + 53, 209, 52, 110, 193, 194, 195, 137, 216, 222, + 229, 217, 217, 217, 53, 53, 52, 52, 193, 194, + 195, 57, 57, 57, 57, 66, 67, 68, 69, 70, + 72, 72, 72, 86, 89, 47, 57, 57, 113, 117, + 117, 79, 123, 124, 131, 47, 133, 137, 139, 143, + 149, 150, 151, 152, 133, 155, 156, 157, 110, 166, + 149, 169, 172, 173, 72, 175, 176, 183, 189, 166, + 196, 137, 200, 202, 137, 166, 211, 166, 137, 72, + 176, 218, 176, 72, 72, 227, 137, 72 ] racc_action_check = [ - 47, 153, 47, 153, 159, 153, 159, 178, 159, 178, - 189, 178, 189, 1, 189, 39, 35, 36, 35, 36, - 5, 35, 36, 37, 38, 37, 38, 6, 37, 38, - 59, 74, 59, 74, 59, 59, 74, 60, 45, 60, - 138, 60, 60, 138, 9, 156, 153, 153, 156, 159, - 159, 47, 178, 178, 39, 189, 189, 45, 45, 45, - 45, 45, 45, 45, 45, 45, 83, 45, 45, 45, - 45, 45, 45, 45, 45, 61, 0, 61, 0, 61, - 61, 166, 166, 166, 166, 83, 83, 83, 83, 83, - 83, 83, 83, 83, 11, 83, 83, 83, 83, 83, - 83, 83, 83, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 13, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 14, 3, 3, 3, 3, 3, 3, 3, - 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 15, 8, 8, 8, 8, 8, 8, 8, 8, 97, - 2, 97, 2, 97, 97, 71, 108, 71, 108, 71, - 108, 109, 169, 109, 169, 109, 169, 176, 184, 176, - 184, 176, 184, 190, 205, 190, 205, 190, 205, 206, - 12, 206, 12, 206, 174, 174, 174, 174, 196, 201, - 214, 196, 201, 214, 69, 76, 69, 76, 104, 105, - 104, 105, 111, 113, 111, 113, 198, 198, 198, 81, - 81, 16, 17, 20, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 40, 51, 56, 67, 70, - 72, 80, 84, 85, 86, 87, 93, 107, 115, 116, - 117, 118, 127, 128, 134, 140, 141, 143, 144, 145, - 146, 150, 151, 152, 158, 163, 165, 167, 168, 171, - 172, 173, 175, 177, 187, 188, 192, 193, 195, 197, - 200, 202, 204, 209, 210, 216 ] + 51, 97, 51, 97, 41, 75, 165, 75, 165, 75, + 165, 51, 171, 190, 171, 190, 171, 190, 201, 165, + 201, 148, 201, 1, 148, 171, 190, 36, 37, 36, + 37, 201, 36, 37, 38, 39, 38, 39, 5, 38, + 39, 117, 0, 117, 0, 117, 41, 46, 168, 88, + 88, 168, 51, 97, 165, 177, 177, 177, 177, 171, + 171, 190, 2, 6, 2, 201, 201, 90, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 9, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 10, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 11, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 3, 3, 12, + 14, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 15, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 8, 8, 16, + 17, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 18, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 63, 13, 63, + 13, 63, 63, 64, 73, 64, 73, 64, 64, 65, + 78, 65, 78, 65, 65, 106, 79, 106, 79, 106, + 106, 118, 180, 118, 180, 118, 180, 188, 196, 188, + 196, 188, 196, 202, 217, 202, 217, 202, 217, 218, + 113, 218, 113, 218, 186, 186, 186, 186, 208, 213, + 226, 208, 213, 226, 114, 123, 114, 123, 210, 210, + 210, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 40, 42, 47, 55, 60, 71, 74, + 76, 80, 81, 87, 91, 92, 93, 94, 102, 116, + 124, 125, 126, 127, 133, 136, 137, 138, 144, 150, + 151, 153, 156, 158, 162, 163, 164, 170, 174, 176, + 178, 179, 182, 184, 187, 189, 199, 200, 204, 205, + 207, 209, 212, 214, 216, 221, 222, 228 ] racc_action_pointer = [ - 66, 13, 150, 90, nil, 13, 27, nil, 118, 35, - nil, 88, 187, 63, 73, 101, 216, 173, nil, nil, - 174, nil, nil, nil, 175, 176, 177, 222, 223, 224, - 225, 226, 224, 225, 226, 13, 14, 20, 21, 10, - 233, nil, nil, nil, nil, 34, nil, -5, nil, nil, - nil, 187, nil, nil, nil, nil, 188, nil, nil, 27, - 34, 72, nil, nil, nil, nil, nil, 230, nil, 201, - 231, 162, 232, nil, 28, nil, 202, nil, nil, nil, - 200, 215, nil, 62, 233, 221, 222, 191, nil, nil, - nil, nil, nil, 244, nil, nil, nil, 156, nil, nil, - nil, nil, nil, nil, 205, 206, nil, 241, 163, 168, - nil, 209, nil, 210, nil, 243, 206, 209, 240, nil, - nil, nil, nil, nil, nil, nil, nil, 209, 248, nil, - nil, nil, nil, nil, 247, nil, nil, nil, -2, nil, - 208, 251, nil, 255, 211, 204, 210, nil, nil, nil, - 253, 257, 217, -2, nil, nil, 3, nil, 218, 1, - nil, nil, nil, 222, nil, 219, 30, 226, 214, 169, - nil, 226, 223, 230, 143, 218, 174, 226, 4, nil, - nil, nil, nil, nil, 175, nil, nil, 272, 228, 7, - 180, nil, 222, 269, nil, 232, 156, 238, 165, nil, - 234, 157, 273, nil, 274, 181, 186, nil, nil, 233, - 230, nil, nil, nil, 158, nil, 277, nil, nil ] + 32, 23, 52, 93, nil, 31, 63, nil, 123, 68, + 74, 84, 103, 165, 94, 111, 123, 135, 141, nil, + nil, nil, nil, nil, 215, 216, 217, 218, 230, 231, + 232, 233, 234, 232, 233, 234, 24, 25, 31, 32, + 238, -1, 242, nil, nil, nil, 43, 232, nil, nil, + nil, -5, nil, nil, nil, 230, nil, nil, nil, nil, + 231, nil, nil, 164, 170, 176, nil, nil, nil, nil, + nil, 240, nil, 171, 241, 2, 242, nil, 177, 183, + 243, 244, nil, nil, nil, nil, nil, 209, 45, nil, + 63, 245, 242, 243, 202, nil, nil, -4, nil, nil, + nil, nil, 256, nil, nil, nil, 182, nil, nil, nil, + nil, nil, nil, 207, 221, nil, 253, 38, 188, nil, + nil, nil, nil, 222, 255, 215, 218, 252, nil, nil, + nil, nil, nil, 251, nil, nil, 219, 261, 250, nil, + nil, nil, nil, nil, 261, nil, nil, nil, -24, nil, + 219, 265, nil, 269, nil, nil, 216, nil, 256, nil, + nil, nil, 266, 270, 227, 3, nil, nil, 3, nil, + 228, 9, nil, nil, 232, nil, 229, 3, 236, 226, + 189, nil, 236, nil, 239, nil, 162, 229, 194, 235, + 10, nil, nil, nil, nil, nil, 195, nil, nil, 284, + 237, 15, 200, nil, 233, 281, nil, 241, 173, 247, + 176, nil, 243, 174, 285, nil, 286, 201, 206, nil, + nil, 278, 241, nil, nil, nil, 175, nil, 289, nil, + nil ] racc_action_default = [ - -1, -128, -1, -3, -10, -128, -128, -2, -3, -128, - -16, -128, -128, -128, -128, -128, -128, -128, -24, -25, - -128, -32, -33, -34, -128, -128, -128, -128, -128, -128, - -128, -128, -50, -50, -50, -128, -128, -128, -128, -128, - -128, -13, 219, -4, -26, -128, -17, -123, -93, -94, - -122, -14, -19, -85, -20, -21, -128, -23, -31, -128, - -128, -128, -38, -39, -40, -41, -42, -43, -51, -128, - -44, -128, -45, -46, -88, -90, -128, -47, -48, -49, - -128, -128, -11, -5, -7, -95, -128, -68, -18, -124, - -125, -126, -15, -128, -22, -27, -28, -29, -35, -83, - -84, -127, -36, -37, -128, -52, -54, -56, -128, -79, - -81, -88, -89, -128, -91, -128, -128, -128, -128, -6, - -8, -9, -120, -96, -97, -98, -69, -128, -128, -86, - -30, -55, -53, -57, -76, -82, -80, -92, -128, -62, - -66, -128, -12, -128, -66, -128, -128, -58, -77, -78, - -50, -128, -60, -64, -67, -70, -128, -121, -99, -100, - -102, -119, -87, -128, -63, -66, -68, -93, -68, -128, - -116, -128, -66, -93, -68, -68, -128, -66, -65, -71, - -72, -108, -109, -110, -128, -74, -75, -128, -66, -101, - -128, -103, -68, -50, -107, -59, -128, -93, -111, -117, - -61, -128, -50, -106, -50, -128, -128, -112, -113, -128, - -68, -104, -73, -114, -128, -118, -50, -115, -105 ] + -1, -136, -1, -3, -10, -136, -136, -2, -3, -136, + -14, -14, -136, -136, -136, -136, -136, -136, -136, -28, + -29, -34, -35, -36, -136, -136, -136, -136, -136, -136, + -136, -136, -136, -54, -54, -54, -136, -136, -136, -136, + -136, -136, -136, -13, 231, -4, -136, -14, -16, -17, + -20, -131, -100, -101, -130, -18, -23, -89, -24, -25, + -136, -27, -37, -136, -136, -136, -41, -42, -43, -44, + -45, -46, -55, -136, -47, -136, -48, -49, -92, -136, + -95, -97, -98, -50, -51, -52, -53, -136, -136, -11, + -5, -7, -14, -136, -72, -15, -21, -131, -132, -133, + -134, -19, -136, -26, -30, -31, -32, -38, -87, -88, + -135, -39, -40, -136, -56, -58, -60, -136, -83, -85, + -93, -94, -96, -136, -136, -136, -136, -136, -6, -8, + -9, -128, -104, -102, -105, -73, -136, -136, -136, -90, + -33, -59, -57, -61, -80, -86, -84, -99, -136, -66, + -70, -136, -12, -136, -103, -109, -136, -22, -136, -62, + -81, -82, -54, -136, -64, -68, -71, -74, -136, -129, + -106, -107, -127, -91, -136, -67, -70, -72, -100, -72, + -136, -124, -136, -109, -100, -110, -72, -72, -136, -70, + -69, -75, -76, -116, -117, -118, -136, -78, -79, -136, + -70, -108, -136, -111, -72, -54, -115, -63, -136, -100, + -119, -125, -65, -136, -54, -114, -54, -136, -136, -120, + -121, -136, -72, -112, -77, -122, -136, -126, -54, -123, + -113 ] racc_goto_table = [ - 69, 109, 50, 152, 57, 127, 84, 58, 112, 160, - 114, 59, 60, 61, 86, 52, 54, 55, 98, 102, - 103, 159, 106, 110, 175, 74, 74, 74, 74, 138, - 9, 1, 3, 180, 7, 43, 120, 160, 109, 109, - 195, 192, 121, 94, 119, 112, 40, 137, 118, 189, - 47, 200, 86, 92, 175, 156, 130, 131, 132, 107, - 135, 136, 88, 196, 111, 207, 111, 70, 72, 201, - 73, 77, 78, 79, 67, 147, 134, 178, 148, 149, - 93, 146, 124, 166, 179, 214, 185, 158, 208, 174, - 187, 209, 191, 193, 107, 107, 143, nil, nil, 186, - nil, 111, nil, 111, nil, nil, 194, nil, 166, nil, - 202, nil, nil, nil, 198, nil, nil, nil, 163, 174, - 198, nil, nil, nil, nil, nil, nil, nil, 216, nil, - nil, nil, nil, nil, nil, 213, 198, nil, nil, nil, + 73, 118, 136, 54, 48, 49, 164, 96, 91, 120, + 121, 93, 187, 148, 107, 111, 112, 119, 134, 171, + 56, 58, 59, 3, 61, 7, 78, 78, 78, 78, + 62, 63, 64, 65, 115, 74, 76, 192, 1, 129, + 168, 95, 187, 118, 118, 207, 204, 201, 77, 83, + 84, 85, 128, 138, 147, 93, 212, 140, 154, 145, + 146, 101, 130, 116, 42, 127, 103, 208, 78, 78, + 219, 9, 51, 213, 141, 142, 45, 71, 159, 144, + 190, 160, 161, 102, 158, 191, 132, 197, 122, 226, + 170, 177, 220, 199, 203, 205, 221, 186, 153, nil, + nil, nil, nil, 116, 116, nil, 198, nil, nil, nil, + nil, nil, 214, 78, 206, nil, 177, nil, nil, nil, + nil, nil, 210, nil, nil, nil, nil, 186, 210, 174, + 228, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 225, 210, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 203, nil, nil, nil, nil, nil, nil, nil, nil, - 211, nil, 212, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, 218 ] + nil, nil, 215, nil, nil, nil, nil, nil, nil, nil, + nil, 223, nil, 224, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 230 ] racc_goto_check = [ - 27, 20, 29, 33, 15, 40, 8, 15, 46, 39, - 46, 15, 15, 15, 12, 16, 16, 16, 22, 22, - 22, 50, 28, 43, 38, 29, 29, 29, 29, 32, - 7, 1, 6, 36, 6, 7, 5, 39, 20, 20, - 33, 36, 9, 15, 8, 46, 10, 46, 11, 50, - 13, 33, 12, 16, 38, 32, 22, 28, 28, 29, - 43, 43, 14, 37, 29, 36, 29, 24, 24, 37, - 25, 25, 25, 25, 23, 30, 31, 34, 41, 42, - 44, 45, 48, 20, 40, 37, 40, 49, 51, 20, - 52, 53, 40, 40, 29, 29, 54, nil, nil, 20, - nil, 29, nil, 29, nil, nil, 20, nil, 20, nil, - 40, nil, nil, nil, 20, nil, nil, nil, 27, 20, - 20, nil, nil, nil, nil, nil, nil, nil, 40, nil, - nil, nil, nil, nil, nil, 20, 20, nil, nil, nil, + 29, 22, 42, 31, 14, 14, 35, 16, 8, 48, + 48, 13, 40, 34, 24, 24, 24, 45, 52, 54, + 18, 18, 18, 6, 17, 6, 31, 31, 31, 31, + 17, 17, 17, 17, 30, 26, 26, 38, 1, 5, + 34, 14, 40, 22, 22, 35, 38, 54, 27, 27, + 27, 27, 8, 16, 48, 13, 35, 24, 52, 45, + 45, 18, 9, 31, 10, 11, 17, 39, 31, 31, + 38, 7, 15, 39, 30, 30, 7, 25, 32, 33, + 36, 43, 44, 46, 47, 42, 14, 42, 50, 39, + 53, 22, 55, 56, 42, 42, 57, 22, 58, nil, + nil, nil, nil, 31, 31, nil, 22, nil, nil, nil, + nil, nil, 42, 31, 22, nil, 22, nil, nil, nil, + nil, nil, 22, nil, nil, nil, nil, 22, 22, 29, + 42, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 22, 22, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 27, nil, nil, nil, nil, nil, nil, nil, nil, - 27, nil, 27, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, 27 ] + nil, nil, 29, nil, nil, nil, nil, nil, nil, nil, + nil, 29, nil, 29, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 29 ] racc_goto_pointer = [ - nil, 31, nil, nil, nil, -48, 32, 27, -39, -42, - 42, -34, -31, 38, 15, -13, 2, nil, nil, nil, - -70, nil, -41, 42, 34, 35, nil, -32, -47, -10, - -59, -31, -86, -137, -88, nil, -133, -121, -135, -135, - -82, -56, -55, -48, 27, -48, -66, nil, -3, -57, - -123, -110, -80, -108, -26 ] + nil, 38, nil, nil, nil, -52, 23, 68, -38, -29, + 60, -24, nil, -35, -6, 59, -44, 6, 6, nil, + nil, nil, -74, nil, -49, 44, 1, 12, nil, -33, + -39, -10, -66, -37, -111, -144, -96, nil, -140, -129, + -159, nil, -92, -63, -62, -58, 26, -55, -69, nil, + 8, nil, -75, -65, -136, -118, -88, -115, -33 ] racc_goto_default = [ - nil, nil, 2, 8, 83, nil, nil, nil, nil, nil, - nil, nil, 10, nil, nil, 51, nil, 21, 22, 23, - 95, 97, nil, nil, nil, nil, 105, 71, nil, 99, - nil, nil, nil, nil, 153, 126, nil, nil, 168, 155, - nil, 100, nil, nil, nil, nil, 75, 85, nil, nil, - nil, nil, nil, nil, nil ] + nil, nil, 2, 8, 90, nil, nil, nil, nil, nil, + nil, nil, 10, 11, nil, nil, nil, 55, nil, 21, + 22, 23, 104, 106, nil, nil, nil, nil, 114, 75, + nil, 108, nil, nil, nil, nil, 165, 135, nil, nil, + 179, 167, nil, 109, nil, nil, nil, nil, 81, 80, + 82, 92, nil, nil, nil, nil, nil, nil, nil ] racc_reduce_table = [ 0, 0, :racc_error, - 0, 63, :_reduce_1, - 2, 63, :_reduce_2, - 0, 64, :_reduce_3, - 2, 64, :_reduce_4, - 1, 65, :_reduce_5, - 2, 65, :_reduce_6, - 0, 66, :_reduce_none, - 1, 66, :_reduce_none, - 5, 58, :_reduce_none, - 0, 67, :_reduce_10, - 0, 68, :_reduce_11, - 5, 59, :_reduce_12, - 2, 59, :_reduce_none, - 1, 73, :_reduce_14, - 2, 73, :_reduce_15, - 1, 60, :_reduce_none, - 2, 60, :_reduce_17, - 3, 60, :_reduce_none, - 2, 60, :_reduce_none, - 2, 60, :_reduce_20, - 2, 60, :_reduce_21, - 3, 60, :_reduce_22, - 2, 60, :_reduce_23, - 1, 60, :_reduce_24, - 1, 60, :_reduce_25, - 2, 60, :_reduce_none, - 1, 78, :_reduce_27, - 1, 78, :_reduce_28, - 1, 79, :_reduce_29, - 2, 79, :_reduce_30, - 2, 69, :_reduce_31, - 1, 69, :_reduce_none, - 1, 69, :_reduce_none, - 1, 69, :_reduce_none, - 3, 69, :_reduce_35, - 3, 69, :_reduce_36, - 3, 69, :_reduce_37, - 2, 69, :_reduce_38, - 2, 69, :_reduce_39, - 2, 69, :_reduce_40, - 2, 69, :_reduce_41, - 2, 69, :_reduce_42, - 2, 74, :_reduce_none, - 2, 74, :_reduce_44, - 2, 74, :_reduce_45, - 2, 74, :_reduce_46, - 2, 74, :_reduce_47, - 2, 74, :_reduce_48, - 2, 74, :_reduce_49, - 0, 84, :_reduce_none, - 1, 84, :_reduce_none, - 1, 85, :_reduce_52, - 2, 85, :_reduce_53, - 2, 80, :_reduce_54, - 3, 80, :_reduce_55, - 0, 88, :_reduce_none, - 1, 88, :_reduce_none, - 3, 83, :_reduce_58, - 8, 75, :_reduce_59, - 5, 76, :_reduce_60, - 8, 76, :_reduce_61, - 1, 89, :_reduce_62, - 3, 89, :_reduce_63, - 1, 90, :_reduce_64, - 3, 90, :_reduce_65, - 0, 96, :_reduce_none, - 1, 96, :_reduce_none, - 0, 97, :_reduce_none, - 1, 97, :_reduce_none, - 1, 91, :_reduce_70, - 3, 91, :_reduce_71, - 3, 91, :_reduce_72, - 6, 91, :_reduce_73, - 3, 91, :_reduce_74, - 3, 91, :_reduce_75, - 0, 99, :_reduce_none, - 1, 99, :_reduce_none, - 1, 87, :_reduce_78, - 1, 100, :_reduce_79, - 2, 100, :_reduce_80, - 2, 81, :_reduce_81, - 3, 81, :_reduce_82, - 1, 77, :_reduce_none, - 1, 77, :_reduce_none, - 0, 101, :_reduce_85, - 0, 102, :_reduce_86, - 5, 72, :_reduce_87, - 1, 103, :_reduce_88, - 2, 103, :_reduce_89, - 1, 82, :_reduce_90, - 2, 82, :_reduce_91, - 3, 82, :_reduce_92, - 1, 86, :_reduce_93, - 1, 86, :_reduce_94, - 0, 105, :_reduce_none, - 1, 105, :_reduce_none, + 0, 64, :_reduce_1, + 2, 64, :_reduce_2, + 0, 65, :_reduce_3, + 2, 65, :_reduce_4, + 1, 66, :_reduce_5, + 2, 66, :_reduce_6, + 0, 67, :_reduce_none, + 1, 67, :_reduce_none, + 5, 59, :_reduce_none, + 0, 68, :_reduce_10, + 0, 69, :_reduce_11, + 5, 60, :_reduce_12, + 2, 60, :_reduce_13, + 0, 72, :_reduce_14, + 2, 72, :_reduce_15, 2, 61, :_reduce_none, 2, 61, :_reduce_none, - 4, 104, :_reduce_99, - 1, 106, :_reduce_100, - 3, 106, :_reduce_101, - 1, 107, :_reduce_102, - 3, 107, :_reduce_103, - 5, 107, :_reduce_104, - 7, 107, :_reduce_105, - 4, 107, :_reduce_106, - 3, 107, :_reduce_107, - 1, 93, :_reduce_108, - 1, 93, :_reduce_109, - 1, 93, :_reduce_110, - 0, 108, :_reduce_none, - 1, 108, :_reduce_none, - 2, 94, :_reduce_113, - 3, 94, :_reduce_114, - 4, 94, :_reduce_115, - 0, 109, :_reduce_116, - 0, 110, :_reduce_117, - 5, 95, :_reduce_118, - 3, 92, :_reduce_119, - 0, 111, :_reduce_120, - 3, 62, :_reduce_121, - 1, 70, :_reduce_none, - 0, 71, :_reduce_none, + 1, 76, :_reduce_18, + 2, 76, :_reduce_19, + 2, 70, :_reduce_20, + 3, 70, :_reduce_21, + 5, 70, :_reduce_22, + 2, 70, :_reduce_none, + 2, 70, :_reduce_24, + 2, 70, :_reduce_25, + 3, 70, :_reduce_26, + 2, 70, :_reduce_27, + 1, 70, :_reduce_28, + 1, 70, :_reduce_29, + 1, 81, :_reduce_30, + 1, 81, :_reduce_31, + 1, 82, :_reduce_32, + 2, 82, :_reduce_33, 1, 71, :_reduce_none, 1, 71, :_reduce_none, 1, 71, :_reduce_none, - 1, 98, :_reduce_127 ] - -racc_reduce_n = 128 - -racc_shift_n = 219 + 2, 71, :_reduce_37, + 3, 71, :_reduce_38, + 3, 71, :_reduce_39, + 3, 71, :_reduce_40, + 2, 71, :_reduce_41, + 2, 71, :_reduce_42, + 2, 71, :_reduce_43, + 2, 71, :_reduce_44, + 2, 71, :_reduce_45, + 2, 77, :_reduce_none, + 2, 77, :_reduce_47, + 2, 77, :_reduce_48, + 2, 77, :_reduce_49, + 2, 77, :_reduce_50, + 2, 77, :_reduce_51, + 2, 77, :_reduce_52, + 2, 77, :_reduce_53, + 0, 87, :_reduce_none, + 1, 87, :_reduce_none, + 1, 88, :_reduce_56, + 2, 88, :_reduce_57, + 2, 83, :_reduce_58, + 3, 83, :_reduce_59, + 0, 91, :_reduce_none, + 1, 91, :_reduce_none, + 3, 86, :_reduce_62, + 8, 78, :_reduce_63, + 5, 79, :_reduce_64, + 8, 79, :_reduce_65, + 1, 92, :_reduce_66, + 3, 92, :_reduce_67, + 1, 93, :_reduce_68, + 3, 93, :_reduce_69, + 0, 99, :_reduce_none, + 1, 99, :_reduce_none, + 0, 100, :_reduce_none, + 1, 100, :_reduce_none, + 1, 94, :_reduce_74, + 3, 94, :_reduce_75, + 3, 94, :_reduce_76, + 6, 94, :_reduce_77, + 3, 94, :_reduce_78, + 3, 94, :_reduce_79, + 0, 102, :_reduce_none, + 1, 102, :_reduce_none, + 1, 90, :_reduce_82, + 1, 103, :_reduce_83, + 2, 103, :_reduce_84, + 2, 84, :_reduce_85, + 3, 84, :_reduce_86, + 1, 80, :_reduce_none, + 1, 80, :_reduce_none, + 0, 104, :_reduce_89, + 0, 105, :_reduce_90, + 5, 75, :_reduce_91, + 1, 106, :_reduce_92, + 2, 106, :_reduce_93, + 2, 107, :_reduce_94, + 1, 108, :_reduce_95, + 2, 108, :_reduce_96, + 1, 85, :_reduce_97, + 1, 85, :_reduce_98, + 3, 85, :_reduce_99, + 1, 89, :_reduce_none, + 1, 89, :_reduce_none, + 1, 110, :_reduce_102, + 2, 110, :_reduce_103, + 2, 62, :_reduce_none, + 2, 62, :_reduce_none, + 4, 109, :_reduce_106, + 1, 111, :_reduce_107, + 3, 111, :_reduce_108, + 0, 112, :_reduce_109, + 2, 112, :_reduce_110, + 3, 112, :_reduce_111, + 5, 112, :_reduce_112, + 7, 112, :_reduce_113, + 4, 112, :_reduce_114, + 3, 112, :_reduce_115, + 1, 96, :_reduce_116, + 1, 96, :_reduce_117, + 1, 96, :_reduce_118, + 0, 113, :_reduce_none, + 1, 113, :_reduce_none, + 2, 97, :_reduce_121, + 3, 97, :_reduce_122, + 4, 97, :_reduce_123, + 0, 114, :_reduce_124, + 0, 115, :_reduce_125, + 5, 98, :_reduce_126, + 3, 95, :_reduce_127, + 0, 116, :_reduce_128, + 3, 63, :_reduce_129, + 1, 73, :_reduce_none, + 0, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 101, :_reduce_135 ] + +racc_reduce_n = 136 + +racc_shift_n = 231 racc_token_table = { false => 0, @@ -1043,52 +1076,53 @@ racc_token_table = { "%{" => 10, "%}" => 11, "%require" => 12, - "%expect" => 13, - "%define" => 14, - "%param" => 15, - "%lex-param" => 16, - "%parse-param" => 17, - "%code" => 18, - "%initial-action" => 19, - "%no-stdlib" => 20, - "%locations" => 21, - ";" => 22, - "%union" => 23, - "%destructor" => 24, - "%printer" => 25, - "%error-token" => 26, - "%after-shift" => 27, - "%before-reduce" => 28, - "%after-reduce" => 29, - "%after-shift-error-token" => 30, - "%after-pop-stack" => 31, - "-temp-group" => 32, - "%token" => 33, - "%type" => 34, - "%nterm" => 35, - "%left" => 36, - "%right" => 37, - "%precedence" => 38, - "%nonassoc" => 39, - "%rule" => 40, - "(" => 41, - ")" => 42, - ":" => 43, - "%inline" => 44, - "," => 45, - "|" => 46, - "%empty" => 47, - "%prec" => 48, - "{" => 49, - "}" => 50, - "?" => 51, - "+" => 52, - "*" => 53, - "[" => 54, - "]" => 55, - "{...}" => 56 } - -racc_nt_base = 57 + ";" => 13, + "%expect" => 14, + "%define" => 15, + "{" => 16, + "}" => 17, + "%param" => 18, + "%lex-param" => 19, + "%parse-param" => 20, + "%code" => 21, + "%initial-action" => 22, + "%no-stdlib" => 23, + "%locations" => 24, + "%union" => 25, + "%destructor" => 26, + "%printer" => 27, + "%error-token" => 28, + "%after-shift" => 29, + "%before-reduce" => 30, + "%after-reduce" => 31, + "%after-shift-error-token" => 32, + "%after-pop-stack" => 33, + "-temp-group" => 34, + "%token" => 35, + "%type" => 36, + "%nterm" => 37, + "%left" => 38, + "%right" => 39, + "%precedence" => 40, + "%nonassoc" => 41, + "%start" => 42, + "%rule" => 43, + "(" => 44, + ")" => 45, + ":" => 46, + "%inline" => 47, + "," => 48, + "|" => 49, + "%empty" => 50, + "%prec" => 51, + "?" => 52, + "+" => 53, + "*" => 54, + "[" => 55, + "]" => 56, + "{...}" => 57 } + +racc_nt_base = 58 racc_use_result_var = true @@ -1123,8 +1157,11 @@ Racc_token_to_s_table = [ "\"%{\"", "\"%}\"", "\"%require\"", + "\";\"", "\"%expect\"", "\"%define\"", + "\"{\"", + "\"}\"", "\"%param\"", "\"%lex-param\"", "\"%parse-param\"", @@ -1132,7 +1169,6 @@ Racc_token_to_s_table = [ "\"%initial-action\"", "\"%no-stdlib\"", "\"%locations\"", - "\";\"", "\"%union\"", "\"%destructor\"", "\"%printer\"", @@ -1150,6 +1186,7 @@ Racc_token_to_s_table = [ "\"%right\"", "\"%precedence\"", "\"%nonassoc\"", + "\"%start\"", "\"%rule\"", "\"(\"", "\")\"", @@ -1159,8 +1196,6 @@ Racc_token_to_s_table = [ "\"|\"", "\"%empty\"", "\"%prec\"", - "\"{\"", - "\"}\"", "\"?\"", "\"+\"", "\"*\"", @@ -1179,7 +1214,9 @@ Racc_token_to_s_table = [ "\"-option@epilogue_declaration\"", "@1", "@2", + "parser_option", "grammar_declaration", + "\"-many@;\"", "variable", "value", "param", @@ -1203,9 +1240,9 @@ Racc_token_to_s_table = [ "rule_rhs_list", "rule_rhs", "named_ref", - "parameterizing_suffix", - "parameterizing_args", - "midrule_action", + "parameterized_suffix", + "parameterized_args", + "action", "\"-option@%empty\"", "\"-option@named_ref\"", "string_as_id", @@ -1214,11 +1251,13 @@ Racc_token_to_s_table = [ "@3", "@4", "\"-many1@id\"", + "\"-group@TAG-\\\"-many1@id\\\"\"", + "\"-many1@-group@TAG-\\\"-many1@id\\\"\"", "rules", - "\"-option@;\"", + "\"-many1@;\"", "rhs_list", "rhs", - "\"-option@parameterizing_suffix\"", + "\"-option@parameterized_suffix\"", "@5", "@6", "@7" ] @@ -1278,10 +1317,9 @@ module_eval(<<'.,.,', 'parser.y', 11) # reduce 9 omitted -module_eval(<<'.,.,', 'parser.y', 12) +module_eval(<<'.,.,', 'parser.y', 13) def _reduce_10(val, _values, result) - begin_c_declaration("%}") - @grammar.prologue_first_lineno = @lexer.line + begin_c_declaration("%}") result end @@ -1289,7 +1327,7 @@ module_eval(<<'.,.,', 'parser.y', 12) module_eval(<<'.,.,', 'parser.y', 17) def _reduce_11(val, _values, result) - end_c_declaration + end_c_declaration result end @@ -1297,22 +1335,29 @@ module_eval(<<'.,.,', 'parser.y', 17) module_eval(<<'.,.,', 'parser.y', 21) def _reduce_12(val, _values, result) - @grammar.prologue = val[2].s_value + @grammar.prologue_first_lineno = val[0].first_line + @grammar.prologue = val[2].s_value result end .,., -# reduce 13 omitted +module_eval(<<'.,.,', 'parser.y', 26) + def _reduce_13(val, _values, result) + @grammar.required = true -module_eval(<<'.,.,', 'parser.y', 54) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) def _reduce_14(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 54) +module_eval(<<'.,.,', 'parser.y', 34) def _reduce_15(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result @@ -1321,145 +1366,140 @@ module_eval(<<'.,.,', 'parser.y', 54) # reduce 16 omitted -module_eval(<<'.,.,', 'parser.y', 26) - def _reduce_17(val, _values, result) - @grammar.expect = val[1] +# reduce 17 omitted + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_18(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -# reduce 18 omitted - -# reduce 19 omitted +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_19(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 31) +module_eval(<<'.,.,', 'parser.y', 36) def _reduce_20(val, _values, result) - val[1].each {|token| - @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value - } + @grammar.expect = val[1].s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 37) +module_eval(<<'.,.,', 'parser.y', 40) def _reduce_21(val, _values, result) - val[1].each {|token| - @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value - } + @grammar.define[val[1].s_value] = val[2]&.s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 43) +module_eval(<<'.,.,', 'parser.y', 44) def _reduce_22(val, _values, result) - @grammar.add_percent_code(id: val[1], code: val[2]) + @grammar.define[val[1].s_value] = val[3]&.s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 47) - def _reduce_23(val, _values, result) - @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) - - result - end -.,., +# reduce 23 omitted module_eval(<<'.,.,', 'parser.y', 49) def _reduce_24(val, _values, result) - @grammar.no_stdlib = true + val[1].each {|token| + @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value + } + result end .,., -module_eval(<<'.,.,', 'parser.y', 50) +module_eval(<<'.,.,', 'parser.y', 55) def _reduce_25(val, _values, result) - @grammar.locations = true + val[1].each {|token| + @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value + } + result end .,., -# reduce 26 omitted +module_eval(<<'.,.,', 'parser.y', 61) + def _reduce_26(val, _values, result) + @grammar.add_percent_code(id: val[1], code: val[2]) -module_eval(<<'.,.,', 'parser.y', 109) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 65) def _reduce_27(val, _values, result) - result = val + @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) + result end .,., -module_eval(<<'.,.,', 'parser.y', 109) +module_eval(<<'.,.,', 'parser.y', 69) def _reduce_28(val, _values, result) - result = val + @grammar.no_stdlib = true + result end .,., -module_eval(<<'.,.,', 'parser.y', 109) +module_eval(<<'.,.,', 'parser.y', 73) def _reduce_29(val, _values, result) - result = val[1] ? val[1].unshift(val[0]) : val + @grammar.locations = true + result end .,., -module_eval(<<'.,.,', 'parser.y', 109) +module_eval(<<'.,.,', 'parser.y', 133) def _reduce_30(val, _values, result) - result = val[1] ? val[1].unshift(val[0]) : val + result = val result end .,., -module_eval(<<'.,.,', 'parser.y', 55) +module_eval(<<'.,.,', 'parser.y', 133) def _reduce_31(val, _values, result) - @grammar.set_union( - Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), - val[1].line - ) - + result = val result end .,., -# reduce 32 omitted - -# reduce 33 omitted - -# reduce 34 omitted - -module_eval(<<'.,.,', 'parser.y', 65) - def _reduce_35(val, _values, result) - @grammar.add_destructor( - ident_or_tags: val[2].flatten, - token_code: val[1], - lineno: val[1].line - ) - +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_32(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 73) - def _reduce_36(val, _values, result) - @grammar.add_printer( - ident_or_tags: val[2].flatten, - token_code: val[1], - lineno: val[1].line - ) - +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_33(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 81) +# reduce 34 omitted + +# reduce 35 omitted + +# reduce 36 omitted + +module_eval(<<'.,.,', 'parser.y', 82) def _reduce_37(val, _values, result) - @grammar.add_error_token( - ident_or_tags: val[2].flatten, - token_code: val[1], - lineno: val[1].line - ) + @grammar.set_union( + Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), + val[1].line + ) result end @@ -1467,665 +1507,762 @@ module_eval(<<'.,.,', 'parser.y', 81) module_eval(<<'.,.,', 'parser.y', 89) def _reduce_38(val, _values, result) - @grammar.after_shift = val[1] + @grammar.add_destructor( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 93) +module_eval(<<'.,.,', 'parser.y', 97) def _reduce_39(val, _values, result) - @grammar.before_reduce = val[1] + @grammar.add_printer( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 97) +module_eval(<<'.,.,', 'parser.y', 105) def _reduce_40(val, _values, result) - @grammar.after_reduce = val[1] + @grammar.add_error_token( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 101) +module_eval(<<'.,.,', 'parser.y', 113) def _reduce_41(val, _values, result) - @grammar.after_shift_error_token = val[1] + @grammar.after_shift = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 105) +module_eval(<<'.,.,', 'parser.y', 117) def _reduce_42(val, _values, result) - @grammar.after_pop_stack = val[1] + @grammar.before_reduce = val[1] result end .,., -# reduce 43 omitted - -module_eval(<<'.,.,', 'parser.y', 111) - def _reduce_44(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - @grammar.add_type(id: id, tag: hash[:tag]) - } - } +module_eval(<<'.,.,', 'parser.y', 121) + def _reduce_43(val, _values, result) + @grammar.after_reduce = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 119) - def _reduce_45(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - if @grammar.find_term_by_s_value(id.s_value) - on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) - else - @grammar.add_type(id: id, tag: hash[:tag]) - end - } - } +module_eval(<<'.,.,', 'parser.y', 125) + def _reduce_44(val, _values, result) + @grammar.after_shift_error_token = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 131) - def _reduce_46(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_left(sym, @precedence_number) - } - } - @precedence_number += 1 +module_eval(<<'.,.,', 'parser.y', 129) + def _reduce_45(val, _values, result) + @grammar.after_pop_stack = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 141) +# reduce 46 omitted + +module_eval(<<'.,.,', 'parser.y', 136) def _reduce_47(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_right(sym, @precedence_number) - } - } - @precedence_number += 1 + val[1].each {|hash| + hash[:tokens].each {|id| + @grammar.add_type(id: id, tag: hash[:tag]) + } + } result end .,., -module_eval(<<'.,.,', 'parser.y', 151) +module_eval(<<'.,.,', 'parser.y', 144) def _reduce_48(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_precedence(sym, @precedence_number) - } - } - @precedence_number += 1 + val[1].each {|hash| + hash[:tokens].each {|id| + if @grammar.find_term_by_s_value(id.s_value) + on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) + else + @grammar.add_type(id: id, tag: hash[:tag]) + end + } + } result end .,., -module_eval(<<'.,.,', 'parser.y', 161) +module_eval(<<'.,.,', 'parser.y', 156) def _reduce_49(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_nonassoc(sym, @precedence_number) - } - } - @precedence_number += 1 + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_left(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 result end .,., -# reduce 50 omitted +module_eval(<<'.,.,', 'parser.y', 166) + def _reduce_50(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_right(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 -# reduce 51 omitted + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 176) + def _reduce_51(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_precedence(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 184) +module_eval(<<'.,.,', 'parser.y', 186) def _reduce_52(val, _values, result) - result = val[1] ? val[1].unshift(val[0]) : val + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_nonassoc(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + result end .,., -module_eval(<<'.,.,', 'parser.y', 184) +module_eval(<<'.,.,', 'parser.y', 196) def _reduce_53(val, _values, result) + @grammar.set_start_nterm(val[1]) + + result + end +.,., + +# reduce 54 omitted + +# reduce 55 omitted + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_56(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_57(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 172) - def _reduce_54(val, _values, result) - val[1].each {|token_declaration| - @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[0], replace: true) - } +module_eval(<<'.,.,', 'parser.y', 202) + def _reduce_58(val, _values, result) + val[1].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[0], replace: true) + } result end .,., -module_eval(<<'.,.,', 'parser.y', 178) - def _reduce_55(val, _values, result) - val[2].each {|token_declaration| - @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[1], replace: true) - } +module_eval(<<'.,.,', 'parser.y', 208) + def _reduce_59(val, _values, result) + val[2].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[1], replace: true) + } result end .,., -# reduce 56 omitted +# reduce 60 omitted -# reduce 57 omitted +# reduce 61 omitted -module_eval(<<'.,.,', 'parser.y', 183) - def _reduce_58(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 213) + def _reduce_62(val, _values, result) result = val result end .,., -module_eval(<<'.,.,', 'parser.y', 187) - def _reduce_59(val, _values, result) - rule = Grammar::ParameterizingRule::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) - @grammar.add_parameterizing_rule(rule) +module_eval(<<'.,.,', 'parser.y', 218) + def _reduce_63(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 193) - def _reduce_60(val, _values, result) - rule = Grammar::ParameterizingRule::Rule.new(val[2].s_value, [], val[4], is_inline: true) - @grammar.add_parameterizing_rule(rule) +module_eval(<<'.,.,', 'parser.y', 225) + def _reduce_64(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, [], val[4], is_inline: true) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 198) - def _reduce_61(val, _values, result) - rule = Grammar::ParameterizingRule::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) - @grammar.add_parameterizing_rule(rule) +module_eval(<<'.,.,', 'parser.y', 230) + def _reduce_65(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 202) - def _reduce_62(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 235) + def _reduce_66(val, _values, result) result = [val[0]] result end .,., -module_eval(<<'.,.,', 'parser.y', 203) - def _reduce_63(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 236) + def _reduce_67(val, _values, result) result = val[0].append(val[2]) result end .,., -module_eval(<<'.,.,', 'parser.y', 207) - def _reduce_64(val, _values, result) - builder = val[0] - result = [builder] +module_eval(<<'.,.,', 'parser.y', 241) + def _reduce_68(val, _values, result) + builder = val[0] + result = [builder] result end .,., -module_eval(<<'.,.,', 'parser.y', 212) - def _reduce_65(val, _values, result) - builder = val[2] - result = val[0].append(builder) +module_eval(<<'.,.,', 'parser.y', 246) + def _reduce_69(val, _values, result) + builder = val[2] + result = val[0].append(builder) result end .,., -# reduce 66 omitted +# reduce 70 omitted -# reduce 67 omitted +# reduce 71 omitted -# reduce 68 omitted +# reduce 72 omitted -# reduce 69 omitted +# reduce 73 omitted -module_eval(<<'.,.,', 'parser.y', 218) - def _reduce_70(val, _values, result) - reset_precs - result = Grammar::ParameterizingRule::Rhs.new +module_eval(<<'.,.,', 'parser.y', 253) + def _reduce_74(val, _values, result) + reset_precs + result = Grammar::Parameterized::Rhs.new result end .,., -module_eval(<<'.,.,', 'parser.y', 223) - def _reduce_71(val, _values, result) - token = val[1] - token.alias_name = val[2] - builder = val[0] - builder.symbols << token - result = builder +module_eval(<<'.,.,', 'parser.y', 258) + def _reduce_75(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.symbols << token + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 231) - def _reduce_72(val, _values, result) - builder = val[0] - builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) - result = builder +module_eval(<<'.,.,', 'parser.y', 267) + def _reduce_76(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 237) - def _reduce_73(val, _values, result) - builder = val[0] - builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) - result = builder +module_eval(<<'.,.,', 'parser.y', 274) + def _reduce_77(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 243) - def _reduce_74(val, _values, result) - user_code = val[1] - user_code.alias_name = val[2] - builder = val[0] - builder.user_code = user_code - result = builder +module_eval(<<'.,.,', 'parser.y', 281) + def _reduce_78(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + builder = val[0] + builder.user_code = user_code + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 251) - def _reduce_75(val, _values, result) - sym = @grammar.find_symbol_by_id!(val[2]) - @prec_seen = true - builder = val[0] - builder.precedence_sym = sym - result = builder +module_eval(<<'.,.,', 'parser.y', 289) + def _reduce_79(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder result end .,., -# reduce 76 omitted +# reduce 80 omitted -# reduce 77 omitted +# reduce 81 omitted -module_eval(<<'.,.,', 'parser.y', 258) - def _reduce_78(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 301) + def _reduce_82(val, _values, result) result = val[0].s_value if val[0] result end .,., -module_eval(<<'.,.,', 'parser.y', 271) - def _reduce_79(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_83(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 271) - def _reduce_80(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_84(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 262) - def _reduce_81(val, _values, result) - result = if val[0] - [{tag: val[0], tokens: val[1]}] - else - [{tag: nil, tokens: val[1]}] - end +module_eval(<<'.,.,', 'parser.y', 306) + def _reduce_85(val, _values, result) + result = if val[0] + [{tag: val[0], tokens: val[1]}] + else + [{tag: nil, tokens: val[1]}] + end result end .,., -module_eval(<<'.,.,', 'parser.y', 268) - def _reduce_82(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 312) + def _reduce_86(val, _values, result) result = val[0].append({tag: val[1], tokens: val[2]}) result end .,., -# reduce 83 omitted +# reduce 87 omitted -# reduce 84 omitted +# reduce 88 omitted -module_eval(<<'.,.,', 'parser.y', 274) - def _reduce_85(val, _values, result) - begin_c_declaration("}") +module_eval(<<'.,.,', 'parser.y', 321) + def _reduce_89(val, _values, result) + begin_c_declaration("}") result end .,., -module_eval(<<'.,.,', 'parser.y', 278) - def _reduce_86(val, _values, result) - end_c_declaration +module_eval(<<'.,.,', 'parser.y', 325) + def _reduce_90(val, _values, result) + end_c_declaration result end .,., -module_eval(<<'.,.,', 'parser.y', 282) - def _reduce_87(val, _values, result) - result = val[2] +module_eval(<<'.,.,', 'parser.y', 329) + def _reduce_91(val, _values, result) + result = val[2] result end .,., -module_eval(<<'.,.,', 'parser.y', 290) - def _reduce_88(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_92(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 290) - def _reduce_89(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_93(val, _values, result) result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 285) - def _reduce_90(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_94(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_95(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_96(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 333) + def _reduce_97(val, _values, result) result = [{tag: nil, tokens: val[0]}] result end .,., -module_eval(<<'.,.,', 'parser.y', 286) - def _reduce_91(val, _values, result) - result = [{tag: val[0], tokens: val[1]}] +module_eval(<<'.,.,', 'parser.y', 334) + def _reduce_98(val, _values, result) + result = val[0].map {|tag, ids| {tag: tag, tokens: ids} } result end .,., -module_eval(<<'.,.,', 'parser.y', 287) - def _reduce_92(val, _values, result) - result = val[0].append({tag: val[1], tokens: val[2]}) +module_eval(<<'.,.,', 'parser.y', 335) + def _reduce_99(val, _values, result) + result = [{tag: nil, tokens: val[0]}, {tag: val[1], tokens: val[2]}] result end .,., -module_eval(<<'.,.,', 'parser.y', 289) - def _reduce_93(val, _values, result) - on_action_error("ident after %prec", val[0]) if @prec_seen +# reduce 100 omitted + +# reduce 101 omitted + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_102(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 290) - def _reduce_94(val, _values, result) - on_action_error("char after %prec", val[0]) if @prec_seen +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_103(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -# reduce 95 omitted +# reduce 104 omitted -# reduce 96 omitted +# reduce 105 omitted -# reduce 97 omitted +module_eval(<<'.,.,', 'parser.y', 348) + def _reduce_106(val, _values, result) + lhs = val[0] + lhs.alias_name = val[1] + val[3].each do |builder| + builder.lhs = lhs + builder.complete_input + @grammar.add_rule_builder(builder) + end -# reduce 98 omitted + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 298) - def _reduce_99(val, _values, result) - lhs = val[0] - lhs.alias_name = val[1] - val[3].each do |builder| - builder.lhs = lhs - builder.complete_input - @grammar.add_rule_builder(builder) - end +module_eval(<<'.,.,', 'parser.y', 360) + def _reduce_107(val, _values, result) + if val[0].rhs.count > 1 + empties = val[0].rhs.select { |sym| sym.is_a?(Lrama::Lexer::Token::Empty) } + empties.each do |empty| + on_action_error("%empty on non-empty rule", empty) + end + end + builder = val[0] + if !builder.line + builder.line = @lexer.line - 1 + end + result = [builder] result end .,., -module_eval(<<'.,.,', 'parser.y', 309) - def _reduce_100(val, _values, result) - builder = val[0] - if !builder.line - builder.line = @lexer.line - 1 - end - result = [builder] +module_eval(<<'.,.,', 'parser.y', 374) + def _reduce_108(val, _values, result) + builder = val[2] + if !builder.line + builder.line = @lexer.line - 1 + end + result = val[0].append(builder) result end .,., -module_eval(<<'.,.,', 'parser.y', 317) - def _reduce_101(val, _values, result) - builder = val[2] - if !builder.line - builder.line = @lexer.line - 1 - end - result = val[0].append(builder) +module_eval(<<'.,.,', 'parser.y', 384) + def _reduce_109(val, _values, result) + reset_precs + result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) result end .,., -module_eval(<<'.,.,', 'parser.y', 326) - def _reduce_102(val, _values, result) - reset_precs - result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) +module_eval(<<'.,.,', 'parser.y', 389) + def _reduce_110(val, _values, result) + builder = val[0] + builder.add_rhs(Lrama::Lexer::Token::Empty.new(location: @lexer.location)) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 331) - def _reduce_103(val, _values, result) - token = val[1] - token.alias_name = val[2] - builder = val[0] - builder.add_rhs(token) - result = builder +module_eval(<<'.,.,', 'parser.y', 395) + def _reduce_111(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.add_rhs(token) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 339) - def _reduce_104(val, _values, result) - token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) - builder = val[0] - builder.add_rhs(token) - builder.line = val[1].first_line - result = builder +module_eval(<<'.,.,', 'parser.y', 404) + def _reduce_112(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 347) - def _reduce_105(val, _values, result) - token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) - builder = val[0] - builder.add_rhs(token) - builder.line = val[1].first_line - result = builder +module_eval(<<'.,.,', 'parser.y', 413) + def _reduce_113(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 355) - def _reduce_106(val, _values, result) - user_code = val[1] - user_code.alias_name = val[2] - user_code.tag = val[3] - builder = val[0] - builder.user_code = user_code - result = builder +module_eval(<<'.,.,', 'parser.y', 422) + def _reduce_114(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + user_code.tag = val[3] + builder = val[0] + builder.user_code = user_code + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 364) - def _reduce_107(val, _values, result) - sym = @grammar.find_symbol_by_id!(val[2]) - @prec_seen = true - builder = val[0] - builder.precedence_sym = sym - result = builder +module_eval(<<'.,.,', 'parser.y', 431) + def _reduce_115(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 371) - def _reduce_108(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 444) + def _reduce_116(val, _values, result) result = "option" result end .,., -module_eval(<<'.,.,', 'parser.y', 372) - def _reduce_109(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 445) + def _reduce_117(val, _values, result) result = "nonempty_list" result end .,., -module_eval(<<'.,.,', 'parser.y', 373) - def _reduce_110(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 446) + def _reduce_118(val, _values, result) result = "list" result end .,., -# reduce 111 omitted +# reduce 119 omitted -# reduce 112 omitted +# reduce 120 omitted -module_eval(<<'.,.,', 'parser.y', 377) - def _reduce_113(val, _values, result) - result = if val[1] - [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] - else - [val[0]] - end +module_eval(<<'.,.,', 'parser.y', 451) + def _reduce_121(val, _values, result) + result = if val[1] + [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] + else + [val[0]] + end result end .,., -module_eval(<<'.,.,', 'parser.y', 383) - def _reduce_114(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 457) + def _reduce_122(val, _values, result) result = val[0].append(val[2]) result end .,., -module_eval(<<'.,.,', 'parser.y', 384) - def _reduce_115(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 458) + def _reduce_123(val, _values, result) result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] result end .,., -module_eval(<<'.,.,', 'parser.y', 388) - def _reduce_116(val, _values, result) - if @prec_seen - on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec - @code_after_prec = true - end - begin_c_declaration("}") +module_eval(<<'.,.,', 'parser.y', 463) + def _reduce_124(val, _values, result) + if prec_seen? + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") result end .,., -module_eval(<<'.,.,', 'parser.y', 396) - def _reduce_117(val, _values, result) - end_c_declaration +module_eval(<<'.,.,', 'parser.y', 471) + def _reduce_125(val, _values, result) + end_c_declaration result end .,., -module_eval(<<'.,.,', 'parser.y', 400) - def _reduce_118(val, _values, result) - result = val[2] +module_eval(<<'.,.,', 'parser.y', 475) + def _reduce_126(val, _values, result) + result = val[2] result end .,., -module_eval(<<'.,.,', 'parser.y', 403) - def _reduce_119(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 478) + def _reduce_127(val, _values, result) result = val[1].s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 407) - def _reduce_120(val, _values, result) - begin_c_declaration('\Z') - @grammar.epilogue_first_lineno = @lexer.line + 1 +module_eval(<<'.,.,', 'parser.y', 483) + def _reduce_128(val, _values, result) + begin_c_declaration('\Z') result end .,., -module_eval(<<'.,.,', 'parser.y', 412) - def _reduce_121(val, _values, result) - end_c_declaration - @grammar.epilogue = val[2].s_value +module_eval(<<'.,.,', 'parser.y', 487) + def _reduce_129(val, _values, result) + end_c_declaration + @grammar.epilogue_first_lineno = val[0].first_line + 1 + @grammar.epilogue = val[2].s_value result end .,., -# reduce 122 omitted +# reduce 130 omitted -# reduce 123 omitted +# reduce 131 omitted -# reduce 124 omitted +# reduce 132 omitted -# reduce 125 omitted +# reduce 133 omitted -# reduce 126 omitted +# reduce 134 omitted -module_eval(<<'.,.,', 'parser.y', 423) - def _reduce_127(val, _values, result) - result = Lrama::Lexer::Token::Ident.new(s_value: val[0]) +module_eval(<<'.,.,', 'parser.y', 499) + def _reduce_135(val, _values, result) + result = Lrama::Lexer::Token::Ident.new(s_value: val[0].s_value) result end .,., diff --git a/tool/lrama/lib/lrama/report.rb b/tool/lrama/lib/lrama/report.rb deleted file mode 100644 index 890e5f1e8c..0000000000 --- a/tool/lrama/lib/lrama/report.rb +++ /dev/null @@ -1,4 +0,0 @@ -# frozen_string_literal: true - -require_relative 'report/duration' -require_relative 'report/profile' diff --git a/tool/lrama/lib/lrama/report/duration.rb b/tool/lrama/lib/lrama/report/duration.rb deleted file mode 100644 index fe09a0d028..0000000000 --- a/tool/lrama/lib/lrama/report/duration.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Report - module Duration - def self.enable - @_report_duration_enabled = true - end - - def self.enabled? - !!@_report_duration_enabled - end - - def report_duration(method_name) - time1 = Time.now.to_f - result = yield - time2 = Time.now.to_f - - if Duration.enabled? - puts sprintf("%s %10.5f s", method_name, time2 - time1) - end - - return result - end - end - end -end diff --git a/tool/lrama/lib/lrama/report/profile.rb b/tool/lrama/lib/lrama/report/profile.rb deleted file mode 100644 index 10488cf913..0000000000 --- a/tool/lrama/lib/lrama/report/profile.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class Report - module Profile - # See "Profiling Lrama" in README.md for how to use. - def self.report_profile - require "stackprof" - - StackProf.run(mode: :cpu, raw: true, out: 'tmp/stackprof-cpu-myapp.dump') do - yield - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/reporter.rb b/tool/lrama/lib/lrama/reporter.rb new file mode 100644 index 0000000000..ed25cc7f8f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'reporter/conflicts' +require_relative 'reporter/grammar' +require_relative 'reporter/precedences' +require_relative 'reporter/profile' +require_relative 'reporter/rules' +require_relative 'reporter/states' +require_relative 'reporter/terms' + +module Lrama + class Reporter + include Lrama::Tracer::Duration + + # @rbs (**bool options) -> void + def initialize(**options) + @options = options + @rules = Rules.new(**options) + @terms = Terms.new(**options) + @conflicts = Conflicts.new + @precedences = Precedences.new + @grammar = Grammar.new(**options) + @states = States.new(**options) + end + + # @rbs (File io, Lrama::States states) -> void + def report(io, states) + report_duration(:report) do + report_duration(:report_rules) { @rules.report(io, states) } + report_duration(:report_terms) { @terms.report(io, states) } + report_duration(:report_conflicts) { @conflicts.report(io, states) } + report_duration(:report_precedences) { @precedences.report(io, states) } + report_duration(:report_grammar) { @grammar.report(io, states) } + report_duration(:report_states) { @states.report(io, states, ielr: states.ielr_defined?) } + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/conflicts.rb b/tool/lrama/lib/lrama/reporter/conflicts.rb new file mode 100644 index 0000000000..f4d8c604c9 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/conflicts.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Conflicts + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_conflicts(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_conflicts(io, states) + has_conflict = false + + states.states.each do |state| + messages = format_conflict_messages(state.conflicts) + + unless messages.empty? + has_conflict = true + io << "State #{state.id} conflicts: #{messages.join(', ')}\n" + end + end + + io << "\n\n" if has_conflict + end + + # @rbs (Array[(Lrama::State::ShiftReduceConflict | Lrama::State::ReduceReduceConflict)] conflicts) -> Array[String] + def format_conflict_messages(conflicts) + conflict_types = { + shift_reduce: "shift/reduce", + reduce_reduce: "reduce/reduce" + } + + conflict_types.keys.map do |type| + type_conflicts = conflicts.select { |c| c.type == type } + "#{type_conflicts.count} #{conflict_types[type]}" unless type_conflicts.empty? + end.compact + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/grammar.rb b/tool/lrama/lib/lrama/reporter/grammar.rb new file mode 100644 index 0000000000..dc3f3f6bfd --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/grammar.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Grammar + # @rbs (?grammar: bool, **bool _) -> void + def initialize(grammar: false, **_) + @grammar = grammar + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @grammar + + io << "Grammar\n" + last_lhs = nil + + states.rules.each do |rule| + if rule.empty_rule? + r = "ε" + else + r = rule.rhs.map(&:display_name).join(" ") + end + + if rule.lhs == last_lhs + io << sprintf("%5d %s| %s", rule.id, " " * rule.lhs.display_name.length, r) << "\n" + else + io << "\n" + io << sprintf("%5d %s: %s", rule.id, rule.lhs.display_name, r) << "\n" + end + + last_lhs = rule.lhs + end + io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/precedences.rb b/tool/lrama/lib/lrama/reporter/precedences.rb new file mode 100644 index 0000000000..73c0888700 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/precedences.rb @@ -0,0 +1,54 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Precedences + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_precedences(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_precedences(io, states) + used_precedences = states.precedences.select(&:used_by?) + + return if used_precedences.empty? + + io << "Precedences\n\n" + + used_precedences.each do |precedence| + io << " precedence on #{precedence.symbol.display_name} is used to resolve conflict on\n" + + if precedence.used_by_lalr? + io << " LALR\n" + + precedence.used_by_lalr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + + if precedence.used_by_ielr? + io << " IELR\n" + + precedence.used_by_ielr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile.rb b/tool/lrama/lib/lrama/reporter/profile.rb new file mode 100644 index 0000000000..b569b94d4f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'profile/call_stack' +require_relative 'profile/memory' diff --git a/tool/lrama/lib/lrama/reporter/profile/call_stack.rb b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb new file mode 100644 index 0000000000..8a4d44b61c --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module CallStack + # See "Call-stack Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_stackprof + ex = nil #: Exception? + path = 'tmp/stackprof-cpu-myapp.dump' + + StackProf.run(mode: :cpu, raw: true, out: path) do + yield + rescue Exception => e + ex = e + end + + STDERR.puts("Call-stack Profiling result is generated on #{path}") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_stackprof + require "stackprof" + true + rescue LoadError + warn "stackprof is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile/memory.rb b/tool/lrama/lib/lrama/reporter/profile/memory.rb new file mode 100644 index 0000000000..a019581fdf --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/memory.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module Memory + # See "Memory Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_memory_profiler + ex = nil #: Exception? + + report = MemoryProfiler.report do # steep:ignore UnknownConstant + yield + rescue Exception => e + ex = e + end + + report.pretty_print(to_file: "tmp/memory_profiler.txt") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_memory_profiler + require "memory_profiler" + true + rescue LoadError + warn "memory_profiler is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/rules.rb b/tool/lrama/lib/lrama/reporter/rules.rb new file mode 100644 index 0000000000..3e8bf19a0a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/rules.rb @@ -0,0 +1,43 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Rules + # @rbs (?rules: bool, **bool _) -> void + def initialize(rules: false, **_) + @rules = rules + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @rules + + used_rules = states.rules.flat_map(&:rhs) + + unless used_rules.empty? + io << "Rule Usage Frequency\n\n" + frequency_counts = used_rules.each_with_object(Hash.new(0)) { |rule, counts| counts[rule] += 1 } + + frequency_counts + .select { |rule,| !rule.midrule? } + .sort_by { |rule, count| [-count, rule.name] } + .each_with_index { |(rule, count), i| io << sprintf("%5d %s (%d times)", i, rule.name, count) << "\n" } + io << "\n\n" + end + + unused_rules = states.rules.map(&:lhs).select do |rule| + !used_rules.include?(rule) && rule.token_id != 0 + end + + unless unused_rules.empty? + io << "#{unused_rules.count} Unused Rules\n\n" + unused_rules.each_with_index do |rule, index| + io << sprintf("%5d %s", index, rule.display_name) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/states.rb b/tool/lrama/lib/lrama/reporter/states.rb new file mode 100644 index 0000000000..d152d0511a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/states.rb @@ -0,0 +1,387 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class States + # @rbs (?itemsets: bool, ?lookaheads: bool, ?solved: bool, ?counterexamples: bool, ?verbose: bool, **bool _) -> void + def initialize(itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false, **_) + @itemsets = itemsets + @lookaheads = lookaheads + @solved = solved + @counterexamples = counterexamples + @verbose = verbose + end + + # @rbs (IO io, Lrama::States states, ielr: bool) -> void + def report(io, states, ielr: false) + cex = Counterexamples.new(states) if @counterexamples + + states.compute_la_sources_for_conflicted_states + report_split_states(io, states.states) if ielr + + states.states.each do |state| + report_state_header(io, state) + report_items(io, state) + report_conflicts(io, state) + report_shifts(io, state) + report_nonassoc_errors(io, state) + report_reduces(io, state) + report_nterm_transitions(io, state) + report_conflict_resolutions(io, state) if @solved + report_counterexamples(io, state, cex) if @counterexamples && state.has_conflicts? # @type var cex: Lrama::Counterexamples + report_verbose_info(io, state, states) if @verbose + # End of Report State + io << "\n" + end + end + + private + + # @rbs (IO io, Array[Lrama::State] states) -> void + def report_split_states(io, states) + ss = states.select(&:split_state?) + + return if ss.empty? + + io << "Split States\n\n" + + ss.each do |state| + io << " State #{state.id} is split from state #{state.lalr_isocore.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_state_header(io, state) + io << "State #{state.id}\n\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_items(io, state) + last_lhs = nil + list = @itemsets ? state.items : state.kernels + + list.sort_by {|i| [i.rule_id, i.position] }.each do |item| + r = item.empty_rule? ? "ε •" : item.rhs.map(&:display_name).insert(item.position, "•").join(" ") + + l = if item.lhs == last_lhs + " " * item.lhs.id.s_value.length + "|" + else + item.lhs.id.s_value + ":" + end + + la = "" + if @lookaheads && item.end_of_rule? + reduce = state.find_reduce_by_item!(item) + look_ahead = reduce.selected_look_ahead + unless look_ahead.empty? + la = " [#{look_ahead.compact.map(&:display_name).join(", ")}]" + end + end + + last_lhs = item.lhs + io << sprintf("%5i %s %s%s", item.rule_id, l, r, la) << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflicts(io, state) + return if state.conflicts.empty? + + state.conflicts.each do |conflict| + syms = conflict.symbols.map { |sym| sym.display_name } + io << " Conflict on #{syms.join(", ")}. " + + case conflict.type + when :shift_reduce + # @type var conflict: Lrama::State::ShiftReduceConflict + io << "shift/reduce(#{conflict.reduce.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + when :reduce_reduce + # @type var conflict: Lrama::State::ReduceReduceConflict + io << "reduce(#{conflict.reduce1.item.rule.lhs.display_name})/reduce(#{conflict.reduce2.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce1.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + + conflict.reduce2.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + else + raise "Unknown conflict type #{conflict.type}" + end + + io << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_shifts(io, state) + shifts = state.term_transitions.reject(&:not_selected) + + return if shifts.empty? + + next_syms = shifts.map(&:next_sym) + max_len = next_syms.map(&:display_name).map(&:length).max + shifts.each do |shift| + io << " #{shift.next_sym.display_name.ljust(max_len)} shift, and go to state #{shift.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nonassoc_errors(io, state) + error_symbols = state.resolved_conflicts.select { |resolved| resolved.which == :error }.map { |error| error.symbol.display_name } + + return if error_symbols.empty? + + max_len = error_symbols.map(&:length).max + error_symbols.each do |name| + io << " #{name.ljust(max_len)} error (nonassociative)\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_reduces(io, state) + reduce_pairs = [] #: Array[[Lrama::Grammar::Symbol, Lrama::State::Action::Reduce]] + + state.non_default_reduces.each do |reduce| + reduce.look_ahead&.each do |term| + reduce_pairs << [term, reduce] + end + end + + return if reduce_pairs.empty? && !state.default_reduction_rule + + max_len = [ + reduce_pairs.map(&:first).map(&:display_name).map(&:length).max || 0, + state.default_reduction_rule ? "$default".length : 0 + ].max + + reduce_pairs.sort_by { |term, _| term.number }.each do |term, reduce| + rule = reduce.item.rule + io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" + end + + if (r = state.default_reduction_rule) + s = "$default".ljust(max_len) + + if r.initial_rule? + io << " #{s} accept\n" + else + io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nterm_transitions(io, state) + return if state.nterm_transitions.empty? + + goto_transitions = state.nterm_transitions.sort_by do |goto| + goto.next_sym.number + end + + max_len = goto_transitions.map(&:next_sym).map do |nterm| + nterm.id.s_value.length + end.max + goto_transitions.each do |goto| + io << " #{goto.next_sym.id.s_value.ljust(max_len)} go to state #{goto.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflict_resolutions(io, state) + return if state.resolved_conflicts.empty? + + state.resolved_conflicts.each do |resolved| + io << " #{resolved.report_message}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::Counterexamples cex) -> void + def report_counterexamples(io, state, cex) + examples = cex.compute(state) + + examples.each do |example| + is_shift_reduce = example.type == :shift_reduce + label0 = is_shift_reduce ? "shift/reduce" : "reduce/reduce" + label1 = is_shift_reduce ? "Shift derivation" : "First Reduce derivation" + label2 = is_shift_reduce ? "Reduce derivation" : "Second Reduce derivation" + + io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" + io << " #{label1}\n" + + example.derivations1.render_strings_for_report.each do |str| + io << " #{str}\n" + end + + io << " #{label2}\n" + + example.derivations2.render_strings_for_report.each do |str| + io << " #{str}\n" + end + end + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_verbose_info(io, state, states) + report_direct_read_sets(io, state, states) + report_reads_relation(io, state, states) + report_read_sets(io, state, states) + report_includes_relation(io, state, states) + report_lookback_relation(io, state, states) + report_follow_sets(io, state, states) + report_look_ahead_sets(io, state, states) + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_direct_read_sets(io, state, states) + io << " [Direct Read sets]\n" + direct_read_sets = states.direct_read_sets + + state.nterm_transitions.each do |goto| + terms = direct_read_sets[goto] + next unless terms && !terms.empty? + + str = terms.map { |sym| sym.id.s_value }.join(", ") + io << " read #{goto.next_sym.id.s_value} shift #{str}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_reads_relation(io, state, states) + io << " [Reads Relation]\n" + + state.nterm_transitions.each do |goto| + goto2 = states.reads_relation[goto] + next unless goto2 + + goto2.each do |goto2| + io << " (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_read_sets(io, state, states) + io << " [Read sets]\n" + read_sets = states.read_sets + + state.nterm_transitions.each do |goto| + terms = read_sets[goto] + next unless terms && !terms.empty? + + terms.each do |sym| + io << " #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_includes_relation(io, state, states) + io << " [Includes Relation]\n" + + state.nterm_transitions.each do |goto| + gotos = states.includes_relation[goto] + next unless gotos + + gotos.each do |goto2| + io << " (State #{state.id}, #{goto.next_sym.id.s_value}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_lookback_relation(io, state, states) + io << " [Lookback Relation]\n" + + states.rules.each do |rule| + gotos = states.lookback_relation.dig(state.id, rule.id) + next unless gotos + + gotos.each do |goto2| + io << " (Rule: #{rule.display_name}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_follow_sets(io, state, states) + io << " [Follow sets]\n" + follow_sets = states.follow_sets + + state.nterm_transitions.each do |goto| + terms = follow_sets[goto] + next unless terms + + terms.each do |sym| + io << " #{goto.next_sym.id.s_value} -> #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_look_ahead_sets(io, state, states) + io << " [Look-Ahead Sets]\n" + look_ahead_rules = [] #: Array[[Lrama::Grammar::Rule, Array[Lrama::Grammar::Symbol]]] + + states.rules.each do |rule| + syms = states.la.dig(state.id, rule.id) + next unless syms + + look_ahead_rules << [rule, syms] + end + + return if look_ahead_rules.empty? + + max_len = look_ahead_rules.flat_map { |_, syms| syms.map { |s| s.id.s_value.length } }.max + + look_ahead_rules.each do |rule, syms| + syms.each do |sym| + io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/terms.rb b/tool/lrama/lib/lrama/reporter/terms.rb new file mode 100644 index 0000000000..f72d8b1a1a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/terms.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Terms + # @rbs (?terms: bool, **bool _) -> void + def initialize(terms: false, **_) + @terms = terms + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @terms + + look_aheads = states.states.each do |state| + state.reduces.flat_map do |reduce| + reduce.look_ahead unless reduce.look_ahead.nil? + end + end + + next_terms = states.states.flat_map do |state| + state.term_transitions.map {|shift| shift.next_sym } + end + + unused_symbols = states.terms.reject do |term| + (look_aheads + next_terms).include?(term) + end + + io << states.terms.count << " Terms\n\n" + + io << states.nterms.count << " Non-Terminals\n\n" + + unless unused_symbols.empty? + io << "#{unused_symbols.count} Unused Terms\n\n" + unused_symbols.each_with_index do |term, index| + io << sprintf("%5d %s", index, term.id.s_value) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb index c2623746aa..50912e094e 100644 --- a/tool/lrama/lib/lrama/state.rb +++ b/tool/lrama/lib/lrama/state.rb @@ -1,17 +1,62 @@ +# rbs_inline: enabled # frozen_string_literal: true -require_relative "state/reduce" +require_relative "state/action" +require_relative "state/inadequacy_annotation" +require_relative "state/item" require_relative "state/reduce_reduce_conflict" require_relative "state/resolved_conflict" -require_relative "state/shift" require_relative "state/shift_reduce_conflict" module Lrama class State - attr_reader :id, :accessing_symbol, :kernels, :conflicts, :resolved_conflicts, - :default_reduction_rule, :closure, :items - attr_accessor :shifts, :reduces + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type conflict = State::ShiftReduceConflict | State::ReduceReduceConflict + # type transition = Action::Shift | Action::Goto + # type lookahead_set = Hash[Item, Array[Grammar::Symbol]] + # + # @id: Integer + # @accessing_symbol: Grammar::Symbol + # @kernels: Array[Item] + # @items: Array[Item] + # @items_to_state: Hash[Array[Item], State] + # @conflicts: Array[conflict] + # @resolved_conflicts: Array[ResolvedConflict] + # @default_reduction_rule: Grammar::Rule? + # @closure: Array[Item] + # @nterm_transitions: Array[Action::Goto] + # @term_transitions: Array[Action::Shift] + # @transitions: Array[transition] + # @internal_dependencies: Hash[Action::Goto, Array[Action::Goto]] + # @successor_dependencies: Hash[Action::Goto, Array[Action::Goto]] + attr_reader :id #: Integer + attr_reader :accessing_symbol #: Grammar::Symbol + attr_reader :kernels #: Array[Item] + attr_reader :conflicts #: Array[conflict] + attr_reader :resolved_conflicts #: Array[ResolvedConflict] + attr_reader :default_reduction_rule #: Grammar::Rule? + attr_reader :closure #: Array[Item] + attr_reader :items #: Array[Item] + attr_reader :annotation_list #: Array[InadequacyAnnotation] + attr_reader :predecessors #: Array[State] + attr_reader :items_to_state #: Hash[Array[Item], State] + attr_reader :lane_items #: Hash[State, Array[[Item, Item]]] + + attr_accessor :_transitions #: Array[[Grammar::Symbol, Array[Item]]] + attr_accessor :reduces #: Array[Action::Reduce] + attr_accessor :ielr_isocores #: Array[State] + attr_accessor :lalr_isocore #: State + attr_accessor :lookaheads_recomputed #: bool + attr_accessor :follow_kernel_items #: Hash[Action::Goto, Hash[Item, bool]] + attr_accessor :always_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor :goto_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + + # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void def initialize(id, accessing_symbol, kernels) @id = id @accessing_symbol = accessing_symbol @@ -23,47 +68,77 @@ module Lrama @conflicts = [] @resolved_conflicts = [] @default_reduction_rule = nil + @predecessors = [] + @lalr_isocore = self + @ielr_isocores = [self] + @internal_dependencies = {} + @successor_dependencies = {} + @annotation_list = [] + @lookaheads_recomputed = false + @follow_kernel_items = {} + @always_follows = {} + @goto_follows = {} + @lhs_contributions = {} + @lane_items = {} + end + + # @rbs (State other) -> bool + def ==(other) + self.id == other.id end + # @rbs (Array[Item] closure) -> void def closure=(closure) @closure = closure @items = @kernels + @closure end + # @rbs () -> Array[Action::Reduce] def non_default_reduces reduces.reject do |reduce| reduce.rule == @default_reduction_rule end end - def compute_shifts_reduces - _shifts = {} + # @rbs () -> void + def compute_transitions_and_reduces + _transitions = {} + @_lane_items ||= {} reduces = [] items.each do |item| # TODO: Consider what should be pushed if item.end_of_rule? - reduces << Reduce.new(item) + reduces << Action::Reduce.new(item) else key = item.next_sym - _shifts[key] ||= [] - _shifts[key] << item.new_by_next_position + _transitions[key] ||= [] + @_lane_items[key] ||= [] + next_item = item.new_by_next_position + _transitions[key] << next_item + @_lane_items[key] << [item, next_item] end end # It seems Bison 3.8.2 iterates transitions order by symbol number - shifts = _shifts.sort_by do |next_sym, new_items| + transitions = _transitions.sort_by do |next_sym, to_items| next_sym.number - end.map do |next_sym, new_items| - Shift.new(next_sym, new_items.flatten) end - self.shifts = shifts.freeze + + self._transitions = transitions.freeze self.reduces = reduces.freeze end + # @rbs (Grammar::Symbol next_sym, State next_state) -> void + def set_lane_items(next_sym, next_state) + @lane_items[next_state] = @_lane_items[next_sym] + end + + # @rbs (Array[Item] items, State next_state) -> void def set_items_to_state(items, next_state) @items_to_state[items] = next_state end + # @rbs (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void def set_look_ahead(rule, look_ahead) reduce = reduces.find do |r| r.rule == rule @@ -72,38 +147,78 @@ module Lrama reduce.look_ahead = look_ahead end - def nterm_transitions - @nterm_transitions ||= transitions.select {|shift, _| shift.next_sym.nterm? } + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void + def set_look_ahead_sources(rule, sources) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead_sources = sources + end + + # @rbs () -> Array[Action::Goto] + def nterm_transitions # steep:ignore + @nterm_transitions ||= transitions.select {|transition| transition.is_a?(Action::Goto) } end - def term_transitions - @term_transitions ||= transitions.select {|shift, _| shift.next_sym.term? } + # @rbs () -> Array[Action::Shift] + def term_transitions # steep:ignore + @term_transitions ||= transitions.select {|transition| transition.is_a?(Action::Shift) } end + # @rbs () -> Array[transition] def transitions - @transitions ||= shifts.map {|shift| [shift, @items_to_state[shift.next_items]] } + @transitions ||= _transitions.map do |next_sym, to_items| + if next_sym.term? + Action::Shift.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + else + Action::Goto.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + end + end + end + + # @rbs (transition transition, State next_state) -> void + def update_transition(transition, next_state) + set_items_to_state(transition.to_items, next_state) + next_state.append_predecessor(self) + update_transitions_caches(transition) + end + + # @rbs () -> void + def update_transitions_caches(transition) + new_transition = + if transition.next_sym.term? + Action::Shift.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + else + Action::Goto.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + end + + @transitions.delete(transition) + @transitions << new_transition + @nterm_transitions = nil + @term_transitions = nil + + @follow_kernel_items[new_transition] = @follow_kernel_items.delete(transition) + @always_follows[new_transition] = @always_follows.delete(transition) end + # @rbs () -> Array[Action::Shift] def selected_term_transitions - term_transitions.reject do |shift, next_state| + term_transitions.reject do |shift| shift.not_selected end end # Move to next state by sym + # + # @rbs (Grammar::Symbol sym) -> State def transition(sym) result = nil if sym.term? - term_transitions.each do |shift, next_state| - term = shift.next_sym - result = next_state if term == sym - end + result = term_transitions.find {|shift| shift.next_sym == sym }.to_state else - nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - result = next_state if nterm == sym - end + result = nterm_transitions.find {|goto| goto.next_sym == sym }.to_state end raise "Can not transit by #{sym} #{self}" if result.nil? @@ -111,12 +226,14 @@ module Lrama result end + # @rbs (Item item) -> Action::Reduce def find_reduce_by_item!(item) reduces.find do |r| r.item == item end || (raise "reduce is not found. #{item}") end + # @rbs (Grammar::Rule default_reduction_rule) -> void def default_reduction_rule=(default_reduction_rule) @default_reduction_rule = default_reduction_rule @@ -127,20 +244,291 @@ module Lrama end end + # @rbs () -> bool def has_conflicts? !@conflicts.empty? end + # @rbs () -> Array[conflict] def sr_conflicts @conflicts.select do |conflict| conflict.type == :shift_reduce end end + # @rbs () -> Array[conflict] def rr_conflicts @conflicts.select do |conflict| conflict.type == :reduce_reduce end end + + # Clear information related to conflicts. + # IELR computation re-calculates conflicts and default reduction of states + # after LALR computation. + # Call this method before IELR computation to avoid duplicated conflicts information + # is stored. + # + # @rbs () -> void + def clear_conflicts + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + + term_transitions.each(&:clear_conflicts) + reduces.each(&:clear_conflicts) + end + + # @rbs () -> bool + def split_state? + @lalr_isocore != self + end + + # Definition 3.40 (propagate_lookaheads) + # + # @rbs (State next_state) -> lookahead_set + def propagate_lookaheads(next_state) + next_state.kernels.map {|next_kernel| + lookahead_sets = + if next_kernel.position > 1 + kernel = kernels.find {|k| k.predecessor_item_of?(next_kernel) } + item_lookahead_set[kernel] + else + goto_follow_set(next_kernel.lhs) + end + + [next_kernel, lookahead_sets & next_state.lookahead_set_filters[next_kernel]] + }.to_h + end + + # Definition 3.43 (is_compatible) + # + # @rbs (lookahead_set filtered_lookahead) -> bool + def is_compatible?(filtered_lookahead) + !lookaheads_recomputed || + @lalr_isocore.annotation_list.all? {|annotation| + a = annotation.dominant_contribution(item_lookahead_set) + b = annotation.dominant_contribution(filtered_lookahead) + a.nil? || b.nil? || a == b + } + end + + # Definition 3.38 (lookahead_set_filters) + # + # @rbs () -> lookahead_set + def lookahead_set_filters + @lookahead_set_filters ||= kernels.map {|kernel| + [kernel, @lalr_isocore.annotation_list.select {|annotation| annotation.contributed?(kernel) }.map(&:token)] + }.to_h + end + + # Definition 3.27 (inadequacy_lists) + # + # @rbs () -> Hash[Grammar::Symbol, Array[Action::Shift | Action::Reduce]] + def inadequacy_list + return @inadequacy_list if @inadequacy_list + + inadequacy_list = {} + + term_transitions.each do |shift| + inadequacy_list[shift.next_sym] ||= [] + inadequacy_list[shift.next_sym] << shift.dup + end + reduces.each do |reduce| + next if reduce.look_ahead.nil? + + reduce.look_ahead.each do |token| + inadequacy_list[token] ||= [] + inadequacy_list[token] << reduce.dup + end + end + + @inadequacy_list = inadequacy_list.select {|token, actions| actions.size > 1 } + end + + # Definition 3.30 (annotate_manifestation) + # + # @rbs () -> void + def annotate_manifestation + inadequacy_list.each {|token, actions| + contribution_matrix = actions.map {|action| + if action.is_a?(Action::Shift) + [action, nil] + else + [action, action.rule.empty_rule? ? lhs_contributions(action.rule.lhs, token) : kernels.map {|k| [k, k.rule == action.item.rule && k.end_of_rule?] }.to_h] + end + }.to_h + @annotation_list << InadequacyAnnotation.new(self, token, actions, contribution_matrix) + } + end + + # Definition 3.32 (annotate_predecessor) + # + # @rbs (State predecessor) -> void + def annotate_predecessor(predecessor) + propagating_list = annotation_list.map {|annotation| + contribution_matrix = annotation.contribution_matrix.map {|action, contributions| + if contributions.nil? + [action, nil] + elsif first_kernels.any? {|kernel| contributions[kernel] && predecessor.lhs_contributions(kernel.lhs, annotation.token).empty? } + [action, nil] + else + cs = predecessor.lane_items[self].map {|pred_kernel, kernel| + c = contributions[kernel] && ( + (kernel.position > 1 && predecessor.item_lookahead_set[pred_kernel].include?(annotation.token)) || + (kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, annotation.token)[pred_kernel]) + ) + [pred_kernel, c] + }.to_h + [action, cs] + end + }.to_h + + # Observation 3.33 (Simple Split-Stable Dominance) + # + # If all of contributions in the contribution_matrix are + # always contribution or never contribution, we can stop annotate propagations + # to the predecessor state. + next nil if contribution_matrix.all? {|_, contributions| contributions.nil? || contributions.all? {|_, contributed| !contributed } } + + InadequacyAnnotation.new(annotation.state, annotation.token, annotation.actions, contribution_matrix) + }.compact + predecessor.append_annotation_list(propagating_list) + end + + # @rbs () -> Array[Item] + def first_kernels + @first_kernels ||= kernels.select {|kernel| kernel.position == 1 } + end + + # @rbs (Array[InadequacyAnnotation] propagating_list) -> void + def append_annotation_list(propagating_list) + annotation_list.each do |annotation| + merging_list = propagating_list.select {|a| a.state == annotation.state && a.token == annotation.token && a.actions == annotation.actions } + annotation.merge_matrix(merging_list.map(&:contribution_matrix)) + propagating_list -= merging_list + end + + @annotation_list += propagating_list + end + + # Definition 3.31 (compute_lhs_contributions) + # + # @rbs (Grammar::Symbol sym, Grammar::Symbol token) -> (nil | Hash[Item, bool]) + def lhs_contributions(sym, token) + return @lhs_contributions[sym][token] unless @lhs_contributions.dig(sym, token).nil? + + transition = nterm_transitions.find {|goto| goto.next_sym == sym } + @lhs_contributions[sym] ||= {} + @lhs_contributions[sym][token] = + if always_follows[transition].include?(token) + {} + else + kernels.map {|kernel| [kernel, follow_kernel_items[transition][kernel] && item_lookahead_set[kernel].include?(token)] }.to_h + end + end + + # Definition 3.26 (item_lookahead_sets) + # + # @rbs () -> lookahead_set + def item_lookahead_set + return @item_lookahead_set if @item_lookahead_set + + @item_lookahead_set = kernels.map {|k| [k, []] }.to_h + @item_lookahead_set = kernels.map {|kernel| + value = + if kernel.lhs.accept_symbol? + [] + elsif kernel.position > 1 + prev_items = predecessors_with_item(kernel) + prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms } + elsif kernel.position == 1 + prev_state = @predecessors.find {|p| p.transitions.any? {|transition| transition.next_sym == kernel.lhs } } + goto = prev_state.nterm_transitions.find {|goto| goto.next_sym == kernel.lhs } + prev_state.goto_follows[goto] + end + [kernel, value] + }.to_h + end + + # @rbs (lookahead_set k) -> void + def item_lookahead_set=(k) + @item_lookahead_set = k + end + + # @rbs (Item item) -> Array[[State, Item]] + def predecessors_with_item(item) + result = [] + @predecessors.each do |pre| + pre.items.each do |i| + result << [pre, i] if i.predecessor_item_of?(item) + end + end + result + end + + # @rbs (State prev_state) -> void + def append_predecessor(prev_state) + @predecessors << prev_state + @predecessors.uniq! + end + + # Definition 3.39 (compute_goto_follow_set) + # + # @rbs (Grammar::Symbol nterm_token) -> Array[Grammar::Symbol] + def goto_follow_set(nterm_token) + return [] if nterm_token.accept_symbol? + goto = @lalr_isocore.nterm_transitions.find {|g| g.next_sym == nterm_token } + + @kernels + .select {|kernel| @lalr_isocore.follow_kernel_items[goto][kernel] } + .map {|kernel| item_lookahead_set[kernel] } + .reduce(@lalr_isocore.always_follows[goto]) {|result, terms| result |= terms } + end + + # Definition 3.8 (Goto Follows Internal Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def internal_dependencies(goto) + return @internal_dependencies[goto] if @internal_dependencies[goto] + + syms = @items.select {|i| + i.next_sym == goto.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0 + }.map(&:lhs).uniq + @internal_dependencies[goto] = nterm_transitions.select {|goto2| syms.include?(goto2.next_sym) } + end + + # Definition 3.5 (Goto Follows Successor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def successor_dependencies(goto) + return @successor_dependencies[goto] if @successor_dependencies[goto] + + @successor_dependencies[goto] = goto.to_state.nterm_transitions.select {|next_goto| next_goto.next_sym.nullable } + end + + # Definition 3.9 (Goto Follows Predecessor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def predecessor_dependencies(goto) + state_items = [] + @kernels.select {|kernel| + kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) + }.each do |item| + queue = predecessors_with_item(item) + until queue.empty? + st, i = queue.pop + if i.position == 0 + state_items << [st, i] + else + st.predecessors_with_item(i).each {|v| queue << v } + end + end + end + + state_items.map {|state, item| + state.nterm_transitions.find {|goto2| goto2.next_sym == item.lhs } + } + end end end diff --git a/tool/lrama/lib/lrama/state/action.rb b/tool/lrama/lib/lrama/state/action.rb new file mode 100644 index 0000000000..791685fc23 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "action/goto" +require_relative "action/reduce" +require_relative "action/shift" diff --git a/tool/lrama/lib/lrama/state/action/goto.rb b/tool/lrama/lib/lrama/state/action/goto.rb new file mode 100644 index 0000000000..4c2c82afdc --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/goto.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Goto + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/reduce.rb b/tool/lrama/lib/lrama/state/action/reduce.rb new file mode 100644 index 0000000000..9678ab0a98 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/reduce.rb @@ -0,0 +1,71 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Reduce + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @item: Item + # @look_ahead: Array[Grammar::Symbol]? + # @look_ahead_sources: Hash[Grammar::Symbol, Array[Action::Goto]]? + # @not_selected_symbols: Array[Grammar::Symbol] + + attr_reader :item #: Item + attr_reader :look_ahead #: Array[Grammar::Symbol]? + attr_reader :look_ahead_sources #: Hash[Grammar::Symbol, Array[Action::Goto]]? + attr_reader :not_selected_symbols #: Array[Grammar::Symbol] + + # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html + attr_accessor :default_reduction #: bool + + # @rbs (Item item) -> void + def initialize(item) + @item = item + @look_ahead = nil + @look_ahead_sources = nil + @not_selected_symbols = [] + end + + # @rbs () -> Grammar::Rule + def rule + @item.rule + end + + # @rbs (Array[Grammar::Symbol] look_ahead) -> Array[Grammar::Symbol] + def look_ahead=(look_ahead) + @look_ahead = look_ahead.freeze + end + + # @rbs (Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> Hash[Grammar::Symbol, Array[Action::Goto]] + def look_ahead_sources=(sources) + @look_ahead_sources = sources.freeze + end + + # @rbs (Grammar::Symbol sym) -> Array[Grammar::Symbol] + def add_not_selected_symbol(sym) + @not_selected_symbols << sym + end + + # @rbs () -> (::Array[Grammar::Symbol?]) + def selected_look_ahead + if look_ahead + look_ahead - @not_selected_symbols + else + [] + end + end + + # @rbs () -> void + def clear_conflicts + @not_selected_symbols = [] + @default_reduction = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/shift.rb b/tool/lrama/lib/lrama/state/action/shift.rb new file mode 100644 index 0000000000..52d9f8c4f0 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/shift.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Shift + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + attr_accessor :not_selected #: bool + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + + # @rbs () -> void + def clear_conflicts + @not_selected = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/inadequacy_annotation.rb b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb new file mode 100644 index 0000000000..3654fa4607 --- /dev/null +++ b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb @@ -0,0 +1,140 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class InadequacyAnnotation + # @rbs! + # type action = Action::Shift | Action::Reduce + + attr_accessor :state #: State + attr_accessor :token #: Grammar::Symbol + attr_accessor :actions #: Array[action] + attr_accessor :contribution_matrix #: Hash[action, Hash[Item, bool]] + + # @rbs (State state, Grammar::Symbol token, Array[action] actions, Hash[action, Hash[Item, bool]] contribution_matrix) -> void + def initialize(state, token, actions, contribution_matrix) + @state = state + @token = token + @actions = actions + @contribution_matrix = contribution_matrix + end + + # @rbs (Item item) -> bool + def contributed?(item) + @contribution_matrix.any? {|action, contributions| !contributions.nil? && contributions[item] } + end + + # @rbs (Array[Hash[action, Hash[Item, bool]]] another_matrixes) -> void + def merge_matrix(another_matrixes) + another_matrixes.each do |another_matrix| + @contribution_matrix.merge!(another_matrix) {|action, contributions, another_contributions| + next contributions if another_contributions.nil? + next another_contributions if contributions.nil? + + contributions.merge!(another_contributions) {|_, contributed, another_contributed| contributed || another_contributed } + } + end + end + + # Definition 3.42 (dominant_contribution) + # + # @rbs (State::lookahead_set lookaheads) -> Array[action]? + def dominant_contribution(lookaheads) + actions = @actions.select {|action| + contribution_matrix[action].nil? || contribution_matrix[action].any? {|item, contributed| contributed && lookaheads[item].include?(@token) } + } + return nil if actions.empty? + + resolve_conflict(actions) + end + + # @rbs (Array[action] actions) -> Array[action] + def resolve_conflict(actions) + # @type var shifts: Array[Action::Shift] + # @type var reduces: Array[Action::Reduce] + shifts = actions.select {|action| action.is_a?(Action::Shift)} + reduces = actions.select {|action| action.is_a?(Action::Reduce) } + + shifts.each do |shift| + reduces.each do |reduce| + sym = shift.next_sym + + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + actions.delete(shift) + next + when shift_prec > reduce_prec + # Shift is selected + actions.delete(reduce) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence&.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + next + when :right + # Shift is selected + actions.delete(reduce) + next + when :left + # Reduce is selected + actions.delete(shift) + next + when :nonassoc + # Can not resolve + # + # nonassoc creates "run-time" error, precedence creates "compile-time" error. + # Then omit both the shift and reduce. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + actions.delete(shift) + actions.delete(reduce) + else + raise "Unknown precedence type. #{sym}" + end + end + end + + actions + end + + # @rbs () -> String + def to_s + "State: #{@state.id}, Token: #{@token.id.s_value}, Actions: #{actions_to_s}, Contributions: #{contribution_matrix_to_s}" + end + + private + + # @rbs () -> String + def actions_to_s + '[' + @actions.map {|action| + if action.is_a?(Action::Shift) || action.is_a?(Action::Goto) + action.class.name + elsif action.is_a?(Action::Reduce) + "#{action.class.name}: (#{action.item})" + end + }.join(', ') + ']' + end + + # @rbs () -> String + def contribution_matrix_to_s + '[' + @contribution_matrix.map {|action, contributions| + "#{(action.is_a?(Action::Shift) || action.is_a?(Action::Goto)) ? action.class.name : "#{action.class.name}: (#{action.item})"}: " + contributions&.transform_keys(&:to_s).to_s + }.join(', ') + ']' + end + end + end +end diff --git a/tool/lrama/lib/lrama/states/item.rb b/tool/lrama/lib/lrama/state/item.rb index 5074e943b7..3ecdd70b76 100644 --- a/tool/lrama/lib/lrama/states/item.rb +++ b/tool/lrama/lib/lrama/state/item.rb @@ -1,3 +1,4 @@ +# rbs_inline: enabled # frozen_string_literal: true # TODO: Validate position is not over rule rhs @@ -5,79 +6,115 @@ require "forwardable" module Lrama - class States + class State class Item < Struct.new(:rule, :position, keyword_init: true) + # @rbs! + # include Grammar::Rule::_DelegatedMethods + # + # attr_accessor rule: Grammar::Rule + # attr_accessor position: Integer + # + # def initialize: (?rule: Grammar::Rule, ?position: Integer) -> void + extend Forwardable def_delegators "rule", :lhs, :rhs # Optimization for States#setup_state + # + # @rbs () -> Integer def hash [rule_id, position].hash end + # @rbs () -> Integer def rule_id rule.id end + # @rbs () -> bool def empty_rule? rule.empty_rule? end + # @rbs () -> Integer def number_of_rest_symbols - rhs.count - position + @number_of_rest_symbols ||= rhs.count - position end + # @rbs () -> Grammar::Symbol def next_sym rhs[position] end + # @rbs () -> Grammar::Symbol def next_next_sym - rhs[position + 1] + @next_next_sym ||= rhs[position + 1] end + # @rbs () -> Grammar::Symbol def previous_sym rhs[position - 1] end + # @rbs () -> bool def end_of_rule? rhs.count == position end + # @rbs () -> bool def beginning_of_rule? position == 0 end + # @rbs () -> bool def start_item? rule.initial_rule? && beginning_of_rule? end + # @rbs () -> State::Item def new_by_next_position Item.new(rule: rule, position: position + 1) end + # @rbs () -> Array[Grammar::Symbol] def symbols_before_dot # steep:ignore rhs[0...position] end + # @rbs () -> Array[Grammar::Symbol] def symbols_after_dot # steep:ignore rhs[position..-1] end + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_transition # steep:ignore + rhs[position+1..-1] + end + + # @rbs () -> ::String def to_s "#{lhs.id.s_value}: #{display_name}" end + # @rbs () -> ::String def display_name r = rhs.map(&:display_name).insert(position, "•").join(" ") "#{r} (rule #{rule_id})" end # Right after position + # + # @rbs () -> ::String def display_rest r = symbols_after_dot.map(&:display_name).join(" ") ". #{r} (rule #{rule_id})" end + + # @rbs (State::Item other_item) -> bool + def predecessor_item_of?(other_item) + rule == other_item.rule && position == other_item.position - 1 + end end end end diff --git a/tool/lrama/lib/lrama/state/reduce.rb b/tool/lrama/lib/lrama/state/reduce.rb deleted file mode 100644 index 54ab87b468..0000000000 --- a/tool/lrama/lib/lrama/state/reduce.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class State - class Reduce - # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html - attr_reader :item, :look_ahead, :not_selected_symbols - attr_accessor :default_reduction - - def initialize(item) - @item = item - @look_ahead = nil - @not_selected_symbols = [] - end - - def rule - @item.rule - end - - def look_ahead=(look_ahead) - @look_ahead = look_ahead.freeze - end - - def add_not_selected_symbol(sym) - @not_selected_symbols << sym - end - - def selected_look_ahead - if look_ahead - look_ahead - @not_selected_symbols - else - [] - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb index 736d08376a..55ecad40bd 100644 --- a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb +++ b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class State - class ReduceReduceConflict < Struct.new(:symbols, :reduce1, :reduce2, keyword_init: true) + class ReduceReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :reduce1 #: State::Action::Reduce + attr_reader :reduce2 #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], reduce1: State::Action::Reduce, reduce2: State::Action::Reduce) -> void + def initialize(symbols:, reduce1:, reduce2:) + @symbols = symbols + @reduce1 = reduce1 + @reduce2 = reduce2 + end + + # @rbs () -> :reduce_reduce def type :reduce_reduce end diff --git a/tool/lrama/lib/lrama/state/resolved_conflict.rb b/tool/lrama/lib/lrama/state/resolved_conflict.rb index 3bb3d1446e..014533c233 100644 --- a/tool/lrama/lib/lrama/state/resolved_conflict.rb +++ b/tool/lrama/lib/lrama/state/resolved_conflict.rb @@ -1,20 +1,54 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class State + # * state: A state on which the conflct is resolved # * symbol: A symbol under discussion # * reduce: A reduce under discussion # * which: For which a conflict is resolved. :shift, :reduce or :error (for nonassociative) - class ResolvedConflict < Struct.new(:symbol, :reduce, :which, :same_prec, keyword_init: true) + # * resolved_by_precedence: If the conflict is resolved by precedence definition or not + class ResolvedConflict + # @rbs! + # type which_enum = :reduce | :shift | :error + + attr_reader :state #: State + attr_reader :symbol #: Grammar::Symbol + attr_reader :reduce #: State::Action::Reduce + attr_reader :which #: which_enum + attr_reader :resolved_by_precedence #: bool + + # @rbs (state: State, symbol: Grammar::Symbol, reduce: State::Action::Reduce, which: which_enum, resolved_by_precedence: bool) -> void + def initialize(state:, symbol:, reduce:, which:, resolved_by_precedence:) + @state = state + @symbol = symbol + @reduce = reduce + @which = which + @resolved_by_precedence = resolved_by_precedence + end + + # @rbs () -> (::String | bot) def report_message + "Conflict between rule #{reduce.rule.id} and token #{symbol.display_name} #{how_resolved}." + end + + # @rbs () -> (::String | bot) + def report_precedences_message + "Conflict between reduce by \"#{reduce.rule.display_name}\" and shift #{symbol.display_name} #{how_resolved}." + end + + private + + # @rbs () -> (::String | bot) + def how_resolved s = symbol.display_name r = reduce.rule.precedence_sym&.display_name case - when which == :shift && same_prec + when which == :shift && resolved_by_precedence msg = "resolved as #{which} (%right #{s})" when which == :shift msg = "resolved as #{which} (#{r} < #{s})" - when which == :reduce && same_prec + when which == :reduce && resolved_by_precedence msg = "resolved as #{which} (%left #{s})" when which == :reduce msg = "resolved as #{which} (#{s} < #{r})" @@ -24,7 +58,7 @@ module Lrama raise "Unknown direction. #{self}" end - "Conflict between rule #{reduce.rule.id} and token #{s} #{msg}." + msg end end end diff --git a/tool/lrama/lib/lrama/state/shift.rb b/tool/lrama/lib/lrama/state/shift.rb deleted file mode 100644 index 81ef013a17..0000000000 --- a/tool/lrama/lib/lrama/state/shift.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class State - class Shift - attr_reader :next_sym, :next_items - attr_accessor :not_selected - - def initialize(next_sym, next_items) - @next_sym = next_sym - @next_items = next_items - end - end - end -end diff --git a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb index fd66834539..548f2de614 100644 --- a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb +++ b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama class State - class ShiftReduceConflict < Struct.new(:symbols, :shift, :reduce, keyword_init: true) + class ShiftReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :shift #: State::Action::Shift + attr_reader :reduce #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], shift: State::Action::Shift, reduce: State::Action::Reduce) -> void + def initialize(symbols:, shift:, reduce:) + @symbols = symbols + @shift = shift + @reduce = reduce + end + + # @rbs () -> :shift_reduce def type :shift_reduce end diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb index 0ed4bff9c1..ddce627df4 100644 --- a/tool/lrama/lib/lrama/states.rb +++ b/tool/lrama/lib/lrama/states.rb @@ -1,8 +1,9 @@ +# rbs_inline: enabled # frozen_string_literal: true require "forwardable" -require_relative "report/duration" -require_relative "states/item" +require_relative "tracer/duration" +require_relative "state/item" module Lrama # States is passed to a template file @@ -10,17 +11,42 @@ module Lrama # "Efficient Computation of LALR(1) Look-Ahead Sets" # https://dl.acm.org/doi/pdf/10.1145/69622.357187 class States + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type state_id = Integer + # type rule_id = Integer + # + # include Grammar::_DelegatedMethods + # + # @grammar: Grammar + # @tracer: Tracer + # @states: Array[State] + # @direct_read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @reads_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @includes_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @lookback_relation: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + # @follow_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @la: Hash[state_id, Hash[rule_id, Bitmap::bitmap]] + extend Forwardable - include Lrama::Report::Duration + include Lrama::Tracer::Duration - def_delegators "@grammar", :symbols, :terms, :nterms, :rules, - :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value! + def_delegators "@grammar", :symbols, :terms, :nterms, :rules, :precedences, + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined? - attr_reader :states, :reads_relation, :includes_relation, :lookback_relation + attr_reader :states #: Array[State] + attr_reader :reads_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :includes_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :lookback_relation #: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] - def initialize(grammar, trace_state: false) + # @rbs (Grammar grammar, Tracer tracer) -> void + def initialize(grammar, tracer) @grammar = grammar - @trace_state = trace_state + @tracer = tracer @states = [] @@ -28,7 +54,7 @@ module Lrama # where p is state, A is nterm, t is term. # # `@direct_read_sets` is a hash whose - # key is [state.id, nterm.token_id], + # key is goto, # value is bitmap of term. @direct_read_sets = {} @@ -37,12 +63,14 @@ module Lrama # where p, r are state, A, C are nterm. # # `@reads_relation` is a hash whose - # key is [state.id, nterm.token_id], - # value is array of [state.id, nterm.token_id]. + # key is goto, + # value is array of goto. @reads_relation = {} + # `Read(p, A) =s DR(p, A) ∪ ∪{Read(r, C) | (p, A) reads (r, C)}` + # # `@read_sets` is a hash whose - # key is [state.id, nterm.token_id], + # key is goto, # value is bitmap of term. @read_sets = {} @@ -50,96 +78,163 @@ module Lrama # where p, p' are state, A, B are nterm, β, γ is sequence of symbol. # # `@includes_relation` is a hash whose - # key is [state.id, nterm.token_id], - # value is array of [state.id, nterm.token_id]. + # key is goto, + # value is array of goto. @includes_relation = {} # `(q, A -> ω) lookback (p, A) iff p -(ω)-> q` # where p, q are state, A -> ω is rule, A is nterm, ω is sequence of symbol. # - # `@lookback_relation` is a hash whose - # key is [state.id, rule.id], - # value is array of [state.id, nterm.token_id]. + # `@lookback_relation` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is array of goto. @lookback_relation = {} + # `Follow(p, A) =s Read(p, A) ∪ ∪{Follow(p', B) | (p, A) includes (p', B)}` + # # `@follow_sets` is a hash whose - # key is [state.id, rule.id], + # key is goto, # value is bitmap of term. @follow_sets = {} # `LA(q, A -> ω) = ∪{Follow(p, A) | (q, A -> ω) lookback (p, A)` # - # `@la` is a hash whose - # key is [state.id, rule.id], + # `@la` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, # value is bitmap of term. @la = {} end + # @rbs () -> void def compute - # Look Ahead Sets report_duration(:compute_lr0_states) { compute_lr0_states } - report_duration(:compute_direct_read_sets) { compute_direct_read_sets } - report_duration(:compute_reads_relation) { compute_reads_relation } - report_duration(:compute_read_sets) { compute_read_sets } - report_duration(:compute_includes_relation) { compute_includes_relation } - report_duration(:compute_lookback_relation) { compute_lookback_relation } - report_duration(:compute_follow_sets) { compute_follow_sets } + + # Look Ahead Sets report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } # Conflicts - report_duration(:compute_conflicts) { compute_conflicts } + report_duration(:compute_conflicts) { compute_conflicts(:lalr) } report_duration(:compute_default_reduction) { compute_default_reduction } end - def reporter - StatesReporter.new(self) + # @rbs () -> void + def compute_ielr + # Preparation + report_duration(:clear_conflicts) { clear_conflicts } + # Phase 1 + report_duration(:compute_predecessors) { compute_predecessors } + report_duration(:compute_follow_kernel_items) { compute_follow_kernel_items } + report_duration(:compute_always_follows) { compute_always_follows } + report_duration(:compute_goto_follows) { compute_goto_follows } + # Phase 2 + report_duration(:compute_inadequacy_annotations) { compute_inadequacy_annotations } + # Phase 3 + report_duration(:split_states) { split_states } + # Phase 4 + report_duration(:clear_look_ahead_sets) { clear_look_ahead_sets } + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + # Phase 5 + report_duration(:compute_conflicts) { compute_conflicts(:ielr) } + report_duration(:compute_default_reduction) { compute_default_reduction } end + # @rbs () -> Integer def states_count @states.count end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def direct_read_sets - @direct_read_sets.transform_values do |v| + @_direct_read_sets ||= @direct_read_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def read_sets - @read_sets.transform_values do |v| + @_read_sets ||= @read_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def follow_sets - @follow_sets.transform_values do |v| + @_follow_sets ||= @follow_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[state_id, Hash[rule_id, Array[Grammar::Symbol]]] def la - @la.transform_values do |v| - bitmap_to_terms(v) + @_la ||= @la.transform_values do |second_hash| + second_hash.transform_values do |v| + bitmap_to_terms(v) + end end end + # @rbs () -> Integer def sr_conflicts_count @sr_conflicts_count ||= @states.flat_map(&:sr_conflicts).count end + # @rbs () -> Integer def rr_conflicts_count @rr_conflicts_count ||= @states.flat_map(&:rr_conflicts).count end - private + # @rbs (Logger logger) -> void + def validate!(logger) + validate_conflicts_within_threshold!(logger) + end - def trace_state - if @trace_state - yield STDERR + def compute_la_sources_for_conflicted_states + reflexive = {} + @states.each do |state| + state.nterm_transitions.each do |goto| + reflexive[goto] = [goto] + end + end + + # compute_read_sets + read_sets = Digraph.new(nterm_transitions, @reads_relation, reflexive).compute + # compute_follow_sets + follow_sets = Digraph.new(nterm_transitions, @includes_relation, read_sets).compute + + @states.select(&:has_conflicts?).each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + sources = {} + + ary.each do |goto| + source = follow_sets[goto] + + next unless source + + source.each do |goto2| + tokens = direct_read_sets[goto2] + tokens.each do |token| + sources[token] ||= [] + sources[token] |= [goto2] + end + end + end + + state.set_look_ahead_sources(rule, sources) + end end end + private + + # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] def create_state(accessing_symbol, kernels, states_created) # A item can appear in some states, # so need to use `kernels` (not `kernels.first`) as a key. @@ -186,27 +281,25 @@ module Lrama return [state, true] end + # @rbs (State state) -> void def setup_state(state) # closure closure = [] - visited = {} queued = {} items = state.kernels.dup items.each do |item| - queued[item] = true + queued[item.rule_id] = true if item.position == 0 end while (item = items.shift) do - visited[item] = true - if (sym = item.next_sym) && sym.nterm? @grammar.find_rules_by_symbol!(sym).each do |rule| - i = Item.new(rule: rule, position: 0) - next if queued[i] + next if queued[rule.id] + i = State::Item.new(rule: rule, position: 0) closure << i items << i - queued[i] = true + queued[i.rule_id] = true end end end @@ -214,116 +307,107 @@ module Lrama state.closure = closure.sort_by {|i| i.rule.id } # Trace - trace_state do |out| - out << "Closure: input\n" - state.kernels.each do |item| - out << " #{item.display_rest}\n" - end - out << "\n\n" - out << "Closure: output\n" - state.items.each do |item| - out << " #{item.display_rest}\n" - end - out << "\n\n" - end + @tracer.trace_closure(state) # shift & reduce - state.compute_shifts_reduces + state.compute_transitions_and_reduces end + # @rbs (Array[State] states, State state) -> void def enqueue_state(states, state) # Trace - previous = state.kernels.first.previous_sym - trace_state do |out| - out << sprintf("state_list_append (state = %d, symbol = %d (%s))", - @states.count, previous.number, previous.display_name) - end + @tracer.trace_state_list_append(@states.count, state) states << state end + # @rbs () -> void def compute_lr0_states # State queue states = [] states_created = {} - state, _ = create_state(symbols.first, [Item.new(rule: @grammar.rules.first, position: 0)], states_created) + state, _ = create_state(symbols.first, [State::Item.new(rule: @grammar.rules.first, position: 0)], states_created) enqueue_state(states, state) while (state = states.shift) do # Trace - # - # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but - # I think it is not correct... - previous = state.kernels.first.previous_sym - trace_state do |out| - out << "Processing state #{state.id} (reached by #{previous.display_name})\n" - end + @tracer.trace_state(state) setup_state(state) - state.shifts.each do |shift| - new_state, created = create_state(shift.next_sym, shift.next_items, states_created) - state.set_items_to_state(shift.next_items, new_state) + # `State#transitions` can not be used here + # because `items_to_state` of the `state` is not set yet. + state._transitions.each do |next_sym, to_items| + new_state, created = create_state(next_sym, to_items, states_created) + state.set_items_to_state(to_items, new_state) + state.set_lane_items(next_sym, new_state) enqueue_state(states, new_state) if created end end end + # @rbs () -> Array[State::Action::Goto] def nterm_transitions a = [] @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - a << [state, nterm, next_state] + state.nterm_transitions.each do |goto| + a << goto end end a end + # @rbs () -> void + def compute_look_ahead_sets + report_duration(:compute_direct_read_sets) { compute_direct_read_sets } + report_duration(:compute_reads_relation) { compute_reads_relation } + report_duration(:compute_read_sets) { compute_read_sets } + report_duration(:compute_includes_relation) { compute_includes_relation } + report_duration(:compute_lookback_relation) { compute_lookback_relation } + report_duration(:compute_follow_sets) { compute_follow_sets } + report_duration(:compute_la) { compute_la } + end + + # @rbs () -> void def compute_direct_read_sets @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - - ary = next_state.term_transitions.map do |shift, _| + state.nterm_transitions.each do |goto| + ary = goto.to_state.term_transitions.map do |shift| shift.next_sym.number end - key = [state.id, nterm.token_id] - @direct_read_sets[key] = Bitmap.from_array(ary) + @direct_read_sets[goto] = Bitmap.from_array(ary) end end end + # @rbs () -> void def compute_reads_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - next_state.nterm_transitions.each do |shift2, _next_state2| - nterm2 = shift2.next_sym + state.nterm_transitions.each do |goto| + goto.to_state.nterm_transitions.each do |goto2| + nterm2 = goto2.next_sym if nterm2.nullable - key = [state.id, nterm.token_id] - @reads_relation[key] ||= [] - @reads_relation[key] << [next_state.id, nterm2.token_id] + @reads_relation[goto] ||= [] + @reads_relation[goto] << goto2 end end end end end + # @rbs () -> void def compute_read_sets - sets = nterm_transitions.map do |state, nterm, next_state| - [state.id, nterm.token_id] - end - - @read_sets = Digraph.new(sets, @reads_relation, @direct_read_sets).compute + @read_sets = Digraph.new(nterm_transitions, @reads_relation, @direct_read_sets).compute end # Execute transition of state by symbols # then return final state. + # + # @rbs (State state, Array[Grammar::Symbol] symbols) -> State def transition(state, symbols) symbols.each do |sym| state = state.transition(sym) @@ -332,10 +416,11 @@ module Lrama state end + # @rbs () -> void def compute_includes_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym + state.nterm_transitions.each do |goto| + nterm = goto.next_sym @grammar.find_rules_by_symbol!(nterm).each do |rule| i = rule.rhs.count - 1 @@ -345,10 +430,12 @@ module Lrama break if sym.term? state2 = transition(state, rule.rhs[0...i]) # p' = state, B = nterm, p = state2, A = sym - key = [state2.id, sym.token_id] + key = state2.nterm_transitions.find do |goto2| + goto2.next_sym.token_id == sym.token_id + end || (raise "Goto by #{sym.name} on state #{state2.id} is not found") # TODO: need to omit if state == state2 ? @includes_relation[key] ||= [] - @includes_relation[key] << [state.id, nterm.token_id] + @includes_relation[key] << goto break unless sym.nullable i -= 1 end @@ -357,45 +444,46 @@ module Lrama end end + # @rbs () -> void def compute_lookback_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym + state.nterm_transitions.each do |goto| + nterm = goto.next_sym @grammar.find_rules_by_symbol!(nterm).each do |rule| state2 = transition(state, rule.rhs) # p = state, A = nterm, q = state2, A -> ω = rule - key = [state2.id, rule.id] - @lookback_relation[key] ||= [] - @lookback_relation[key] << [state.id, nterm.token_id] + @lookback_relation[state2.id] ||= {} + @lookback_relation[state2.id][rule.id] ||= [] + @lookback_relation[state2.id][rule.id] << goto end end end end + # @rbs () -> void def compute_follow_sets - sets = nterm_transitions.map do |state, nterm, next_state| - [state.id, nterm.token_id] - end - - @follow_sets = Digraph.new(sets, @includes_relation, @read_sets).compute + @follow_sets = Digraph.new(nterm_transitions, @includes_relation, @read_sets).compute end - def compute_look_ahead_sets + # @rbs () -> void + def compute_la @states.each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state rules.each do |rule| - ary = @lookback_relation[[state.id, rule.id]] + ary = lookback_relation_on_state[rule.id] next unless ary - ary.each do |state2_id, nterm_token_id| + ary.each do |goto| # q = state, A -> ω = rule, p = state2, A = nterm - follows = @follow_sets[[state2_id, nterm_token_id]] + follows = @follow_sets[goto] next if follows == 0 - key = [state.id, rule.id] - @la[key] ||= 0 - look_ahead = @la[key] | follows - @la[key] |= look_ahead + @la[state.id] ||= {} + @la[state.id][rule.id] ||= 0 + look_ahead = @la[state.id][rule.id] | follows + @la[state.id][rule.id] |= look_ahead # No risk of conflict when # * the state only has single reduce @@ -408,6 +496,7 @@ module Lrama end end + # @rbs (Bitmap::bitmap bit) -> Array[Grammar::Symbol] def bitmap_to_terms(bit) ary = Bitmap.to_array(bit) ary.map do |i| @@ -415,14 +504,16 @@ module Lrama end end - def compute_conflicts - compute_shift_reduce_conflicts + # @rbs () -> void + def compute_conflicts(lr_type) + compute_shift_reduce_conflicts(lr_type) compute_reduce_reduce_conflicts end - def compute_shift_reduce_conflicts + # @rbs () -> void + def compute_shift_reduce_conflicts(lr_type) states.each do |state| - state.shifts.each do |shift| + state.term_transitions.each do |shift| state.reduces.each do |reduce| sym = shift.next_sym @@ -442,43 +533,57 @@ module Lrama case when shift_prec < reduce_prec # Reduce is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :reduce) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when shift_prec > reduce_prec # Shift is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next end # shift_prec == reduce_prec, then check associativity case sym.precedence.type when :precedence + # Can not resolve the conflict + # # %precedence only specifies precedence and not specify associativity # then a conflict is unresolved if precedence is same. state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) next when :right # Shift is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift, same_prec: true) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when :left # Reduce is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :reduce, same_prec: true) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when :nonassoc - # Can not resolve + # The conflict is resolved # - # nonassoc creates "run-time" error, precedence creates "compile-time" error. - # Then omit both the shift and reduce. + # %nonassoc creates "run-time" error by removing both shift and reduce from + # the state. This makes the state to get syntax error if the conflicted token appears. + # On the other hand, %precedence creates "compile-time" error by keeping both + # shift and reduce on the state. This makes the state to be conflicted on the token. # # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :error) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :error, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict shift.not_selected = true reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) else raise "Unknown precedence type. #{sym}" end @@ -487,35 +592,41 @@ module Lrama end end + # @rbs (Grammar::Precedence shift_prec, Grammar::Precedence reduce_prec, State::ResolvedConflict resolved_conflict) -> void + def mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + case lr_type + when :lalr + shift_prec.mark_used_by_lalr(resolved_conflict) + reduce_prec.mark_used_by_lalr(resolved_conflict) + when :ielr + shift_prec.mark_used_by_ielr(resolved_conflict) + reduce_prec.mark_used_by_ielr(resolved_conflict) + end + end + + # @rbs () -> void def compute_reduce_reduce_conflicts states.each do |state| - count = state.reduces.count - - (0...count).each do |i| - reduce1 = state.reduces[i] - next if reduce1.look_ahead.nil? + state.reduces.combination(2) do |reduce1, reduce2| + next if reduce1.look_ahead.nil? || reduce2.look_ahead.nil? - ((i+1)...count).each do |j| - reduce2 = state.reduces[j] - next if reduce2.look_ahead.nil? + intersection = reduce1.look_ahead & reduce2.look_ahead - intersection = reduce1.look_ahead & reduce2.look_ahead - - unless intersection.empty? - state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) - end + unless intersection.empty? + state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) end end end end + # @rbs () -> void def compute_default_reduction states.each do |state| next if state.reduces.empty? # Do not set, if conflict exist next unless state.conflicts.empty? # Do not set, if shift with `error` exists. - next if state.shifts.map(&:next_sym).include?(@grammar.error_symbol) + next if state.term_transitions.map {|shift| shift.next_sym }.include?(@grammar.error_symbol) state.default_reduction_rule = state.reduces.map do |r| [r.rule, r.rule.id, (r.look_ahead || []).count] @@ -524,5 +635,233 @@ module Lrama end.first end end + + # @rbs () -> void + def clear_conflicts + states.each(&:clear_conflicts) + end + + # Definition 3.15 (Predecessors) + # + # @rbs () -> void + def compute_predecessors + @states.each do |state| + state.transitions.each do |transition| + transition.to_state.append_predecessor(state) + end + end + end + + # Definition 3.16 (follow_kernel_items) + # + # @rbs () -> void + def compute_follow_kernel_items + set = nterm_transitions + relation = compute_goto_internal_relation + base_function = compute_goto_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, follow_kernel_items| + state = goto.from_state + state.follow_kernel_items[goto] = state.kernels.map {|kernel| + [kernel, Bitmap.to_bool_array(follow_kernel_items, state.kernels.count)] + }.to_h + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_goto_bitmaps + nterm_transitions.map {|goto| + bools = goto.from_state.kernels.map.with_index {|kernel, i| i if kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) }.compact + [goto, Bitmap.from_array(bools)] + }.to_h + end + + # Definition 3.20 (always_follows, one closure) + # + # @rbs () -> void + def compute_always_follows + set = nterm_transitions + relation = compute_goto_successor_or_internal_relation + base_function = compute_transition_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, always_follows_bitmap| + goto.from_state.always_follows[goto] = bitmap_to_terms(always_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_successor_or_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.successor_dependencies(goto) + state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_transition_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.to_state.term_transitions.map {|shift| shift.next_sym.number })] + }.to_h + end + + # Definition 3.24 (goto_follows, via always_follows) + # + # @rbs () -> void + def compute_goto_follows + set = nterm_transitions + relation = compute_goto_internal_or_predecessor_dependencies + base_function = compute_always_follows_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, goto_follows_bitmap| + goto.from_state.goto_follows[goto] = bitmap_to_terms(goto_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_or_predecessor_dependencies + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + state.predecessor_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_always_follows_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.from_state.always_follows[goto].map(&:number))] + }.to_h + end + + # @rbs () -> void + def split_states + @states.each do |state| + state.transitions.each do |transition| + compute_state(state, transition, transition.to_state) + end + end + end + + # @rbs () -> void + def compute_inadequacy_annotations + @states.each do |state| + state.annotate_manifestation + end + + queue = @states.reject {|state| state.annotation_list.empty? } + + while (curr = queue.shift) do + curr.predecessors.each do |pred| + cache = pred.annotation_list.dup + curr.annotate_predecessor(pred) + queue << pred if cache != pred.annotation_list && !queue.include?(pred) + end + end + end + + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> void + def merge_lookaheads(state, filtered_lookaheads) + return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? } + + state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 } + state.transitions.each do |transition| + next if transition.to_state.lookaheads_recomputed + compute_state(state, transition, transition.to_state) + end + end + + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void + def compute_state(state, transition, next_state) + propagating_lookaheads = state.propagate_lookaheads(next_state) + s = next_state.ielr_isocores.find {|st| st.is_compatible?(propagating_lookaheads) } + + if s.nil? + s = next_state.lalr_isocore + new_state = State.new(@states.count, s.accessing_symbol, s.kernels) + new_state.closure = s.closure + new_state.compute_transitions_and_reduces + s.transitions.each do |transition| + new_state.set_items_to_state(transition.to_items, transition.to_state) + end + @states << new_state + new_state.lalr_isocore = s + s.ielr_isocores << new_state + s.ielr_isocores.each do |st| + st.ielr_isocores = s.ielr_isocores + end + new_state.lookaheads_recomputed = true + new_state.item_lookahead_set = propagating_lookaheads + state.update_transition(transition, new_state) + elsif(!s.lookaheads_recomputed) + s.lookaheads_recomputed = true + s.item_lookahead_set = propagating_lookaheads + else + merge_lookaheads(s, propagating_lookaheads) + state.update_transition(transition, s) if state.items_to_state[transition.to_items].id != s.id + end + end + + # @rbs (Logger logger) -> void + def validate_conflicts_within_threshold!(logger) + exit false unless conflicts_within_threshold?(logger) + end + + # @rbs (Logger logger) -> bool + def conflicts_within_threshold?(logger) + return true unless @grammar.expect + + [sr_conflicts_within_threshold?(logger), rr_conflicts_within_threshold?(logger)].all? + end + + # @rbs (Logger logger) -> bool + def sr_conflicts_within_threshold?(logger) + return true if @grammar.expect == sr_conflicts_count + + logger.error("shift/reduce conflicts: #{sr_conflicts_count} found, #{@grammar.expect} expected") + false + end + + # @rbs (Logger logger) -> bool + def rr_conflicts_within_threshold?(logger, expected: 0) + return true if expected == rr_conflicts_count + + logger.error("reduce/reduce conflicts: #{rr_conflicts_count} found, #{expected} expected") + false + end + + # @rbs () -> void + def clear_look_ahead_sets + @direct_read_sets.clear + @reads_relation.clear + @read_sets.clear + @includes_relation.clear + @lookback_relation.clear + @follow_sets.clear + @la.clear + + @_direct_read_sets = nil + @_read_sets = nil + @_follow_sets = nil + @_la = nil + end end end diff --git a/tool/lrama/lib/lrama/states_reporter.rb b/tool/lrama/lib/lrama/states_reporter.rb deleted file mode 100644 index 64ff4de100..0000000000 --- a/tool/lrama/lib/lrama/states_reporter.rb +++ /dev/null @@ -1,362 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class StatesReporter - include Lrama::Report::Duration - - def initialize(states) - @states = states - end - - def report(io, **options) - report_duration(:report) do - _report(io, **options) - end - end - - private - - def _report(io, grammar: false, rules: false, terms: false, states: false, itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false) - report_unused_rules(io) if rules - report_unused_terms(io) if terms - report_conflicts(io) - report_grammar(io) if grammar - report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) - end - - def report_unused_terms(io) - look_aheads = @states.states.each do |state| - state.reduces.flat_map do |reduce| - reduce.look_ahead unless reduce.look_ahead.nil? - end - end - - next_terms = @states.states.flat_map do |state| - state.shifts.map(&:next_sym).select(&:term?) - end - - unused_symbols = @states.terms.select do |term| - !(look_aheads + next_terms).include?(term) - end - - unless unused_symbols.empty? - io << "#{unused_symbols.count} Unused Terms\n\n" - unused_symbols.each_with_index do |term, index| - io << sprintf("%5d %s\n", index, term.id.s_value) - end - io << "\n\n" - end - end - - def report_unused_rules(io) - used_rules = @states.rules.flat_map(&:rhs) - - unused_rules = @states.rules.map(&:lhs).select do |rule| - !used_rules.include?(rule) && rule.token_id != 0 - end - - unless unused_rules.empty? - io << "#{unused_rules.count} Unused Rules\n\n" - unused_rules.each_with_index do |rule, index| - io << sprintf("%5d %s\n", index, rule.display_name) - end - io << "\n\n" - end - end - - def report_conflicts(io) - has_conflict = false - - @states.states.each do |state| - messages = [] - cs = state.conflicts.group_by(&:type) - if cs[:shift_reduce] - messages << "#{cs[:shift_reduce].count} shift/reduce" - end - - if cs[:reduce_reduce] - messages << "#{cs[:reduce_reduce].count} reduce/reduce" - end - - unless messages.empty? - has_conflict = true - io << "State #{state.id} conflicts: #{messages.join(', ')}\n" - end - end - - if has_conflict - io << "\n\n" - end - end - - def report_grammar(io) - io << "Grammar\n" - last_lhs = nil - - @states.rules.each do |rule| - if rule.empty_rule? - r = "ε" - else - r = rule.rhs.map(&:display_name).join(" ") - end - - if rule.lhs == last_lhs - io << sprintf("%5d %s| %s\n", rule.id, " " * rule.lhs.display_name.length, r) - else - io << "\n" - io << sprintf("%5d %s: %s\n", rule.id, rule.lhs.display_name, r) - end - - last_lhs = rule.lhs - end - io << "\n\n" - end - - def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) - if counterexamples - cex = Counterexamples.new(@states) - end - - @states.states.each do |state| - # Report State - io << "State #{state.id}\n\n" - - # Report item - last_lhs = nil - list = itemsets ? state.items : state.kernels - list.sort_by {|i| [i.rule_id, i.position] }.each do |item| - if item.empty_rule? - r = "ε •" - else - r = item.rhs.map(&:display_name).insert(item.position, "•").join(" ") - end - if item.lhs == last_lhs - l = " " * item.lhs.id.s_value.length + "|" - else - l = item.lhs.id.s_value + ":" - end - la = "" - if lookaheads && item.end_of_rule? - reduce = state.find_reduce_by_item!(item) - look_ahead = reduce.selected_look_ahead - unless look_ahead.empty? - la = " [#{look_ahead.map(&:display_name).join(", ")}]" - end - end - last_lhs = item.lhs - - io << sprintf("%5i %s %s%s\n", item.rule_id, l, r, la) - end - io << "\n" - - # Report shifts - tmp = state.term_transitions.reject do |shift, _| - shift.not_selected - end.map do |shift, next_state| - [shift.next_sym, next_state.id] - end - max_len = tmp.map(&:first).map(&:display_name).map(&:length).max - tmp.each do |term, state_id| - io << " #{term.display_name.ljust(max_len)} shift, and go to state #{state_id}\n" - end - io << "\n" unless tmp.empty? - - # Report error caused by %nonassoc - nl = false - tmp = state.resolved_conflicts.select do |resolved| - resolved.which == :error - end.map do |error| - error.symbol.display_name - end - max_len = tmp.map(&:length).max - tmp.each do |name| - nl = true - io << " #{name.ljust(max_len)} error (nonassociative)\n" - end - io << "\n" unless tmp.empty? - - # Report reduces - nl = false - max_len = state.non_default_reduces.flat_map(&:look_ahead).compact.map(&:display_name).map(&:length).max || 0 - max_len = [max_len, "$default".length].max if state.default_reduction_rule - ary = [] - - state.non_default_reduces.each do |reduce| - reduce.look_ahead.each do |term| - ary << [term, reduce] - end - end - - ary.sort_by do |term, reduce| - term.number - end.each do |term, reduce| - rule = reduce.item.rule - io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" - nl = true - end - - if (r = state.default_reduction_rule) - nl = true - s = "$default".ljust(max_len) - - if r.initial_rule? - io << " #{s} accept\n" - else - io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" - end - end - io << "\n" if nl - - # Report nonterminal transitions - tmp = [] - max_len = 0 - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - tmp << [nterm, next_state.id] - max_len = [max_len, nterm.id.s_value.length].max - end - tmp.uniq! - tmp.sort_by! do |nterm, state_id| - nterm.number - end - tmp.each do |nterm, state_id| - io << " #{nterm.id.s_value.ljust(max_len)} go to state #{state_id}\n" - end - io << "\n" unless tmp.empty? - - if solved - # Report conflict resolutions - state.resolved_conflicts.each do |resolved| - io << " #{resolved.report_message}\n" - end - io << "\n" unless state.resolved_conflicts.empty? - end - - if counterexamples && state.has_conflicts? - # Report counterexamples - examples = cex.compute(state) - examples.each do |example| - label0 = example.type == :shift_reduce ? "shift/reduce" : "reduce/reduce" - label1 = example.type == :shift_reduce ? "Shift derivation" : "First Reduce derivation" - label2 = example.type == :shift_reduce ? "Reduce derivation" : "Second Reduce derivation" - - io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" - io << " #{example.path1_item}\n" - io << " #{example.path2_item}\n" - io << " #{label1}\n" - example.derivations1.render_strings_for_report.each do |str| - io << " #{str}\n" - end - io << " #{label2}\n" - example.derivations2.render_strings_for_report.each do |str| - io << " #{str}\n" - end - end - end - - if verbose - # Report direct_read_sets - io << " [Direct Read sets]\n" - direct_read_sets = @states.direct_read_sets - @states.nterms.each do |nterm| - terms = direct_read_sets[[state.id, nterm.token_id]] - next unless terms - next if terms.empty? - - str = terms.map {|sym| sym.id.s_value }.join(", ") - io << " read #{nterm.id.s_value} shift #{str}\n" - end - io << "\n" - - # Report reads_relation - io << " [Reads Relation]\n" - @states.nterms.each do |nterm| - a = @states.reads_relation[[state.id, nterm.token_id]] - next unless a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report read_sets - io << " [Read sets]\n" - read_sets = @states.read_sets - @states.nterms.each do |nterm| - terms = read_sets[[state.id, nterm.token_id]] - next unless terms - next if terms.empty? - - terms.each do |sym| - io << " #{sym.id.s_value}\n" - end - end - io << "\n" - - # Report includes_relation - io << " [Includes Relation]\n" - @states.nterms.each do |nterm| - a = @states.includes_relation[[state.id, nterm.token_id]] - next unless a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (State #{state.id}, #{nterm.id.s_value}) -> (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report lookback_relation - io << " [Lookback Relation]\n" - @states.rules.each do |rule| - a = @states.lookback_relation[[state.id, rule.id]] - next unless a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (Rule: #{rule.display_name}) -> (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report follow_sets - io << " [Follow sets]\n" - follow_sets = @states.follow_sets - @states.nterms.each do |nterm| - terms = follow_sets[[state.id, nterm.token_id]] - - next unless terms - - terms.each do |sym| - io << " #{nterm.id.s_value} -> #{sym.id.s_value}\n" - end - end - io << "\n" - - # Report LA - io << " [Look-Ahead Sets]\n" - tmp = [] - max_len = 0 - @states.rules.each do |rule| - syms = @states.la[[state.id, rule.id]] - next unless syms - - tmp << [rule, syms] - max_len = ([max_len] + syms.map {|s| s.id.s_value.length }).max - end - tmp.each do |rule, syms| - syms.each do |sym| - io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" - end - end - io << "\n" unless tmp.empty? - end - - # End of Report State - io << "\n" - end - end - end -end diff --git a/tool/lrama/lib/lrama/trace_reporter.rb b/tool/lrama/lib/lrama/trace_reporter.rb deleted file mode 100644 index 87c01a4c8a..0000000000 --- a/tool/lrama/lib/lrama/trace_reporter.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Lrama - class TraceReporter - def initialize(grammar) - @grammar = grammar - end - - def report(**options) - _report(**options) - end - - private - - def _report(rules: false, actions: false, **_) - report_rules if rules - report_actions if actions - end - - def report_rules - puts "Grammar rules:" - @grammar.rules.each { |rule| puts rule.display_name } - end - - def report_actions - puts "Grammar rules with actions:" - @grammar.rules.each { |rule| puts rule.with_actions } - end - end -end diff --git a/tool/lrama/lib/lrama/tracer.rb b/tool/lrama/lib/lrama/tracer.rb new file mode 100644 index 0000000000..fda699a665 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer.rb @@ -0,0 +1,51 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "tracer/actions" +require_relative "tracer/closure" +require_relative "tracer/duration" +require_relative "tracer/only_explicit_rules" +require_relative "tracer/rules" +require_relative "tracer/state" + +module Lrama + class Tracer + # @rbs (IO io, **bool options) -> void + def initialize(io, **options) + @io = io + @options = options + @only_explicit_rules = OnlyExplicitRules.new(io, **options) + @rules = Rules.new(io, **options) + @actions = Actions.new(io, **options) + @closure = Closure.new(io, **options) + @state = State.new(io, **options) + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + @only_explicit_rules.trace(grammar) + @rules.trace(grammar) + @actions.trace(grammar) + end + + # @rbs (Lrama::State state) -> void + def trace_closure(state) + @closure.trace(state) + end + + # @rbs (Lrama::State state) -> void + def trace_state(state) + @state.trace(state) + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_state_list_append(state_count, state) + @state.trace_list_append(state_count, state) + end + + # @rbs () -> void + def enable_duration + Duration.enable if @options[:time] + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/actions.rb b/tool/lrama/lib/lrama/tracer/actions.rb new file mode 100644 index 0000000000..7b9c9b9f53 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/actions.rb @@ -0,0 +1,22 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Actions + # @rbs (IO io, ?actions: bool, **bool options) -> void + def initialize(io, actions: false, **options) + @io = io + @actions = actions + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @actions + + @io << "Grammar rules with actions:" << "\n" + grammar.rules.each { |rule| @io << rule.with_actions << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/closure.rb b/tool/lrama/lib/lrama/tracer/closure.rb new file mode 100644 index 0000000000..5b2f0b27e6 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/closure.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Closure + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @closure = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @closure + + @io << "Closure: input" << "\n" + state.kernels.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + @io << "Closure: output" << "\n" + state.items.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/duration.rb b/tool/lrama/lib/lrama/tracer/duration.rb new file mode 100644 index 0000000000..91c49625b2 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/duration.rb @@ -0,0 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + module Duration + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @_report_duration_enabled: bool + + # @rbs () -> void + def self.enable + @_report_duration_enabled = true + end + + # @rbs () -> bool + def self.enabled? + !!@_report_duration_enabled + end + + # @rbs [T] (_ToS message) { -> T } -> T + def report_duration(message) + time1 = Time.now.to_f + result = yield + time2 = Time.now.to_f + + if Duration.enabled? + STDERR.puts sprintf("%s %10.5f s", message, time2 - time1) + end + + return result + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb new file mode 100644 index 0000000000..4f64e7d2f4 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class OnlyExplicitRules + # @rbs (IO io, ?only_explicit: bool, **bool) -> void + def initialize(io, only_explicit: false, **_) + @io = io + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each do |rule| + @io << rule.display_name_without_action << "\n" if rule.lhs.first_set.any? + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/rules.rb b/tool/lrama/lib/lrama/tracer/rules.rb new file mode 100644 index 0000000000..d6e85b8432 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Rules + # @rbs (IO io, ?rules: bool, ?only_explicit: bool, **bool) -> void + def initialize(io, rules: false, only_explicit: false, **_) + @io = io + @rules = rules + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return if !@rules || @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each { |rule| @io << rule.display_name << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/state.rb b/tool/lrama/lib/lrama/tracer/state.rb new file mode 100644 index 0000000000..21c0047f8e --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/state.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class State + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @state = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @state + + # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but + # I think it is not correct... + previous = state.kernels.first.previous_sym + @io << "Processing state #{state.id} (reached by #{previous.display_name})" << "\n" + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_list_append(state_count, state) + return unless @state + + previous = state.kernels.first.previous_sym + @io << sprintf("state_list_append (state = %d, symbol = %d (%s))", + state_count, previous.number, previous.display_name) << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb index ff55e34b51..d649b74939 100644 --- a/tool/lrama/lib/lrama/version.rb +++ b/tool/lrama/lib/lrama/version.rb @@ -1,5 +1,6 @@ +# rbs_inline: enabled # frozen_string_literal: true module Lrama - VERSION = "0.6.11".freeze + VERSION = "0.7.1".freeze #: String end diff --git a/tool/lrama/lib/lrama/warnings.rb b/tool/lrama/lib/lrama/warnings.rb new file mode 100644 index 0000000000..52f09144ef --- /dev/null +++ b/tool/lrama/lib/lrama/warnings.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'warnings/conflicts' +require_relative 'warnings/implicit_empty' +require_relative 'warnings/name_conflicts' +require_relative 'warnings/redefined_rules' +require_relative 'warnings/required' +require_relative 'warnings/useless_precedence' + +module Lrama + class Warnings + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @conflicts = Conflicts.new(logger, warnings) + @implicit_empty = ImplicitEmpty.new(logger, warnings) + @name_conflicts = NameConflicts.new(logger, warnings) + @redefined_rules = RedefinedRules.new(logger, warnings) + @required = Required.new(logger, warnings) + @useless_precedence = UselessPrecedence.new(logger, warnings) + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + @conflicts.warn(states) + @implicit_empty.warn(grammar) + @name_conflicts.warn(grammar) + @redefined_rules.warn(grammar) + @required.warn(grammar) + @useless_precedence.warn(grammar, states) + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/conflicts.rb b/tool/lrama/lib/lrama/warnings/conflicts.rb new file mode 100644 index 0000000000..6ba0de6f9c --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/conflicts.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Conflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::States states) -> void + def warn(states) + return unless @warnings + + if states.sr_conflicts_count != 0 + @logger.warn("shift/reduce conflicts: #{states.sr_conflicts_count} found") + end + + if states.rr_conflicts_count != 0 + @logger.warn("reduce/reduce conflicts: #{states.rr_conflicts_count} found") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/implicit_empty.rb b/tool/lrama/lib/lrama/warnings/implicit_empty.rb new file mode 100644 index 0000000000..ba81adca01 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/implicit_empty.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Empty rules are easily overlooked and ambiguous + # - Empty alternatives like `rule: | "token";` can be missed during code reading + # - Difficult to distinguish between intentional empty rules vs. omissions + # - Explicit marking with %empty directive comment improves clarity + class ImplicitEmpty + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.rule_builders.each do |builder| + if builder.rhs.empty? + @logger.warn("warning: empty rule without %empty") + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/name_conflicts.rb b/tool/lrama/lib/lrama/warnings/name_conflicts.rb new file mode 100644 index 0000000000..c0754ab551 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/name_conflicts.rb @@ -0,0 +1,63 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Parameterized rule names conflicting with symbol names + # - When a %rule name is identical to a terminal or non-terminal symbol name, + # it reduces grammar readability and may cause unintended behavior + # - Detecting these conflicts helps improve grammar definition quality + class NameConflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return if grammar.parameterized_rules.empty? + + symbol_names = collect_symbol_names(grammar) + check_conflicts(grammar.parameterized_rules, symbol_names) + end + + private + + # @rbs (Lrama::Grammar grammar) -> Set[String] + def collect_symbol_names(grammar) + symbol_names = Set.new + + collect_term_names(grammar.terms, symbol_names) + collect_nterm_names(grammar.nterms, symbol_names) + + symbol_names + end + + # @rbs (Array[untyped] terms, Set[String] symbol_names) -> void + def collect_term_names(terms, symbol_names) + terms.each do |term| + symbol_names.add(term.id.s_value) + symbol_names.add(term.alias_name) if term.alias_name + end + end + + # @rbs (Array[untyped] nterms, Set[String] symbol_names) -> void + def collect_nterm_names(nterms, symbol_names) + nterms.each do |nterm| + symbol_names.add(nterm.id.s_value) + end + end + + # @rbs (Array[untyped] parameterized_rules, Set[String] symbol_names) -> void + def check_conflicts(parameterized_rules, symbol_names) + parameterized_rules.each do |param_rule| + next unless symbol_names.include?(param_rule.name) + + @logger.warn("warning: parameterized rule name \"#{param_rule.name}\" conflicts with symbol name") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/redefined_rules.rb b/tool/lrama/lib/lrama/warnings/redefined_rules.rb new file mode 100644 index 0000000000..8ac2f1f103 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/redefined_rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class RedefinedRules + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.parameterized_resolver.redefined_rules.each do |rule| + @logger.warn("parameterized rule redefined: #{rule}") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/required.rb b/tool/lrama/lib/lrama/warnings/required.rb new file mode 100644 index 0000000000..4ab1ed787e --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/required.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Required + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings = false, **_) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + if grammar.required + @logger.warn("currently, %require is simply valid as a grammar but does nothing") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/useless_precedence.rb b/tool/lrama/lib/lrama/warnings/useless_precedence.rb new file mode 100644 index 0000000000..2913d6d7e5 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/useless_precedence.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class UselessPrecedence + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + return unless @warnings + + grammar.precedences.each do |precedence| + unless precedence.used_by? + @logger.warn("Precedence #{precedence.s_value} (line: #{precedence.lineno}) is defined but not used in any rule.") + end + end + end + end + end +end diff --git a/tool/lrama/template/bison/_yacc.h b/tool/lrama/template/bison/_yacc.h index 34ed6d81f5..3e270c9171 100644 --- a/tool/lrama/template/bison/_yacc.h +++ b/tool/lrama/template/bison/_yacc.h @@ -28,6 +28,7 @@ extern int yydebug; <%-# b4_declare_yylstype -%> <%-# b4_value_type_define -%> /* Value type. */ +<% if output.grammar.union %> #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED union YYSTYPE { @@ -40,6 +41,13 @@ typedef union YYSTYPE YYSTYPE; # define YYSTYPE_IS_TRIVIAL 1 # define YYSTYPE_IS_DECLARED 1 #endif +<% else %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef int YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% end %> <%-# b4_location_type_define -%> /* Location type. */ diff --git a/tool/lrama/template/diagram/diagram.html b/tool/lrama/template/diagram/diagram.html new file mode 100644 index 0000000000..3e87e6e519 --- /dev/null +++ b/tool/lrama/template/diagram/diagram.html @@ -0,0 +1,102 @@ +<!DOCTYPE html> +<html> +<head> + <title>Lrama syntax diagrams</title> + + <style> + <%= output.default_style %> + .diagram-header { + display: inline-block; + font-weight: bold; + font-size: 18px; + margin-bottom: -8px; + text-align: center; + } + + svg { + width: 100%; + } + + svg.railroad-diagram g.non-terminal text { + cursor: pointer; + } + + h2.hover-header { + background-color: #90ee90; + } + + svg.railroad-diagram g.non-terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + + svg.railroad-diagram g.terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + </style> +</head> + +<body align="center"> + <%= output.diagrams %> + <script> + document.addEventListener("DOMContentLoaded", () => { + function addHoverEffect(selector, hoverClass, relatedSelector, relatedHoverClass, getTextElements) { + document.querySelectorAll(selector).forEach(element => { + element.addEventListener("mouseenter", () => { + element.classList.add(hoverClass); + getTextElements(element).forEach(textEl => { + if (!relatedSelector) return; + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.add(relatedHoverClass); + }); + }); + }); + + element.addEventListener("mouseleave", () => { + element.classList.remove(hoverClass); + if (!relatedSelector) return; + getTextElements(element).forEach(textEl => { + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.remove(relatedHoverClass); + }); + }); + }); + }); + } + + function getElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => el.textContent.trim() === text.trim()); + } + + function getParentElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => + [...el.querySelectorAll("text")].some(textEl => textEl.textContent.trim() === text.trim()) + ); + } + + function scrollToMatchingHeader() { + document.querySelectorAll("g.non-terminal").forEach(element => { + element.addEventListener("click", () => { + const textElements = [...element.querySelectorAll("text")]; + for (const textEl of textElements) { + const targetHeader = getElementsByText("h2", textEl.textContent)[0]; + if (targetHeader) { + targetHeader.scrollIntoView({ behavior: "smooth", block: "start" }); + break; + } + } + }); + }); + } + + addHoverEffect("h2", "hover-header", "g.non-terminal", "hover-g", element => [element]); + addHoverEffect("g.non-terminal", "hover-g", "h2", "hover-header", + element => [...element.querySelectorAll("text")] + ); + addHoverEffect("g.terminal", "hover-g", "", "", element => [element]); + scrollToMatchingHeader(); + }); + </script> +</body> +</html> diff --git a/tool/m4/ruby_append_option.m4 b/tool/m4/ruby_append_option.m4 index 98359fa1f9..8cd2741ae8 100644 --- a/tool/m4/ruby_append_option.m4 +++ b/tool/m4/ruby_append_option.m4 @@ -4,6 +4,6 @@ AC_DEFUN([RUBY_APPEND_OPTION], AS_CASE([" [$]{$1-} "], [*" $2 "*], [], [' '], [ $1="$2"], [ $1="[$]$1 $2"])])dnl AC_DEFUN([RUBY_PREPEND_OPTION], - [# RUBY_APPEND_OPTION($1) + [# RUBY_PREPEND_OPTION($1) AS_CASE([" [$]{$1-} "], [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_defint.m4 b/tool/m4/ruby_defint.m4 index e9ed68e5b8..7f262a73fc 100644 --- a/tool/m4/ruby_defint.m4 +++ b/tool/m4/ruby_defint.m4 @@ -17,7 +17,8 @@ typedef $1 t; int s = sizeof(t) == 42;])], ["${ac_cv_sizeof___int128@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int128"], [ rb_cv_type_$1=no])])]) AS_IF([test "${rb_cv_type_$1}" != no], [ - type="${rb_cv_type_$1@%:@@%:@unsigned }" + type="${rb_cv_type_$1@%:@@%:@*signed }" + AS_IF([test "$type" = "long long"], [type=long_long]) AS_IF([test "$type" != yes && eval 'test -n "${ac_cv_sizeof_'$type'+set}"'], [ eval cond='"${ac_cv_sizeof_'$type'}"' AS_CASE([$cond], [*:*], [ diff --git a/tool/make-snapshot b/tool/make-snapshot index 3479e2ac85..4af6a855eb 100755 --- a/tool/make-snapshot +++ b/tool/make-snapshot @@ -10,6 +10,7 @@ require 'fileutils' require 'shellwords' require 'tmpdir' require 'pathname' +require 'date' require 'yaml' require 'json' require File.expand_path("../lib/vcs", __FILE__) @@ -71,16 +72,6 @@ ENV["LC_ALL"] = ENV["LANG"] = "C" GITURL = URI.parse("https://github.com/ruby/ruby.git") RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ -ENV["VPATH"] ||= "include/ruby" -YACC = ENV["YACC"] ||= "#{$tooldir}/lrama/exe/lrama" -ENV["BASERUBY"] ||= "ruby" -ENV["RUBY"] ||= "ruby" -ENV["MV"] ||= "mv" -ENV["RM"] ||= "rm -f" -ENV["MINIRUBY"] ||= "ruby" -ENV["PROGRAM"] ||= "ruby" -ENV["AUTOCONF"] ||= "autoconf" -ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" ENV["TZ"] = "UTC" class String @@ -117,22 +108,32 @@ $digests &&= $digests.split(/[, ]+/).tap {|dig| $digests ||= DIGESTS $patch_file &&= File.expand_path($patch_file) -path = ENV["PATH"].split(File::PATH_SEPARATOR) -%w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| - cmd, = ENV[var].shellsplit - unless path.any? {|dir| +PATH = ENV["PATH"].split(File::PATH_SEPARATOR) +def PATH.executable_env(var, command = nil) + command = ENV[var] ||= (command or return) + cmd, = command.shellsplit + unless any? {|dir| file = File.expand_path(cmd, dir) File.file?(file) and File.executable?(file) } abort "#{File.basename $0}: #{var} command not found - #{cmd}" end + command end +PATH.executable_env("MV", "mv") +PATH.executable_env("RM", "rm -f") +PATH.executable_env("AUTOCONF", "autoconf") + %w[BASERUBY RUBY MINIRUBY].each do |var| - %x[#{ENV[var]} --disable-gem -e1 2>&1] - if $?.success? - ENV[var] += ' --disable-gem' + cmd = PATH.executable_env(var, "ruby") + help = IO.popen("#{cmd} --help", err: %i[child out], &:read) + unless $?.success? and /ruby/ =~ help + abort "#{File.basename $0}: #{var} ruby not found - #{cmd}" end + IO.popen("#{cmd} --disable-gem -eexit", err: %i[child out], &:read) + cmd += ' --disable-gem' if $?.success? + ENV[var] = cmd end if defined?($help) or defined?($_help) @@ -252,7 +253,6 @@ end def package(vcs, rev, destdir, tmp = nil) pwd = Dir.pwd - patchlevel = false prerelease = false if rev and revision = rev[/@(\h+)\z/, 1] rev = $` @@ -269,22 +269,23 @@ def package(vcs, rev, destdir, tmp = nil) when /\Astable\z/ vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} url &&= vcs.branch(url) - when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + when /\A(\d+)\.(\d+)\.(\d+)-(preview|rc)(\d+)/ prerelease = true tag = "#{$4}#{$5}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") - when /\A(.*)\.(.*)\.(.*)-p(\d+)/ - patchlevel = true - tag = "p#{$4}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") - when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ - if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") - patchlevel = true - tag = "" - url = vcs.tag("v#{$1}_#{$2}_#{$3}") + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") else - url = vcs.branch("ruby_#{rev.tr('.', '_')}") + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") + end + when /\A(\d+)\.(\d+)\.(\d+)\z/ + tag = "" + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") + else + url = vcs.tag("v#{$1}_#{$2}_#{$3}") end + when /\A(\d+)\.(\d+)\z/ + url = vcs.branch("ruby_#{rev.tr('.', '_')}") else warn "#{$0}: unknown version - #{rev}" return @@ -334,7 +335,7 @@ def package(vcs, rev, destdir, tmp = nil) FileUtils.rm(file, verbose: $VERBOSE) end - status = IO.read(File.dirname(__FILE__) + "/prereq.status") + status = File.read(File.dirname(__FILE__) + "/prereq.status") Dir.chdir(tmp) if tmp if !File.directory?(v) @@ -346,26 +347,20 @@ def package(vcs, rev, destdir, tmp = nil) File.open("#{v}/revision.h", "wb") {|f| f.puts vcs.revision_header(revision, modified) } - version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version ||= (versionhdr = File.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] version ||= begin - include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h") + include_ruby_versionhdr = File.read("#{v}/include/ruby/version.h") api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] [api_major_version, api_minor_version, version_teeny].join('.') end version or return - if patchlevel - unless tag.empty? - versionhdr ||= IO.read("#{v}/version.h") - patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] - tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision)) - end - elsif prerelease - versionhdr ||= IO.read("#{v}/version.h") + if prerelease + versionhdr ||= File.read("#{v}/version.h") versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace" - IO.write("#{v}/version.h", versionhdr) + File.write("#{v}/version.h", versionhdr) else tag ||= vcs.revision_name(revision) end @@ -430,7 +425,7 @@ def package(vcs, rev, destdir, tmp = nil) puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE unless File.exist?("configure") print "creating configure..." - unless system([ENV["AUTOCONF"]]*2) + unless system(File.exist?(gen = "./autogen.sh") ? gen : [ENV["AUTOCONF"]]*2) puts $colorize.fail(" failed") return end @@ -439,11 +434,11 @@ def package(vcs, rev, destdir, tmp = nil) clean.add("autom4te.cache") clean.add("enc/unicode/data") print "creating prerequisites..." - if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + if File.file?("common.mk") && /^prereq/ =~ commonmk = File.read("common.mk") puts extout = clean.add('tmp') begin - status = IO.read("tool/prereq.status") + status = File.read("tool/prereq.status") rescue Errno::ENOENT # use fallback file end @@ -456,7 +451,7 @@ def package(vcs, rev, destdir, tmp = nil) File.binwrite("#{defaults}/ruby.rb", "") miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] - mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). + mk = (File.read("template/Makefile.in") rescue File.read("Makefile.in")). gsub(/^@.*\n/, '') vars = { "EXTOUT"=>extout, @@ -469,6 +464,9 @@ def package(vcs, rev, destdir, tmp = nil) "MAJOR"=>api_major_version, "MINOR"=>api_minor_version, "TEENY"=>version_teeny, + "VPATH"=>(ENV["VPATH"] || "include/ruby"), + "PROGRAM"=>(ENV["PROGRAM"] || "ruby"), + "BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"), } status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do vars[$2] ||= $3 @@ -477,6 +475,13 @@ def package(vcs, rev, destdir, tmp = nil) vars["UNICODE_VERSION"] = $unicode_version if $unicode_version args = vars.dup mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} + commonmk.gsub!(/^!(?:include \$\(srcdir\)\/(.*))?/) do + if inc = $1 and File.exist?(inc) + File.binread(inc).gsub(/^!/, '# !') + else + "#" + end + end mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') mk << <<-'APPEND' @@ -514,7 +519,7 @@ touch-unicode-files: end print "prerequisites" else - system(*%W"#{YACC} -o parse.c parse.y") + system(*%W[#{PATH.executable_env("YACC", "bison")} -o parse.c parse.y]) end vcs.after_export(".") if exported clean.concat(Dir.glob("ext/**/autom4te.cache")) @@ -631,7 +636,7 @@ revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| key = basename[/\A(.*)\.(?:tar|zip)/, 1] info[key] ||= Hash.new{|h,k|h[k]={}} info[key]['version'] = version if version - info[key]['date'] = release_date.strftime('%Y-%m-%d') + info[key]['date'] = release_date.to_date if version info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" diff --git a/tool/merger.rb b/tool/merger.rb index e45a3ac586..4c096087fc 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -65,7 +65,8 @@ class << Merger = Object.new if teeny v[2].succ! end - if pl != '-1' # trunk does not have patchlevel + # We stopped bumping RUBY_PATCHLEVEL at Ruby 4.0.0. + if Integer(v[0]) <= 3 pl.succ! end @@ -113,8 +114,17 @@ class << Merger = Object.new abort 'no relname is given and not in a release branch even if this is patch release' end end - tagname = "v#{v.join('_')}#{("_#{pl}" if v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl)}" + if /^(?:preview|rc)/ =~ pl + tagname = "v#{v.join('.')}-#{pl}" + elsif Integer(v[0]) >= 4 + tagname = "v#{v.join('.')}" + else + tagname = "v#{v.join('_')}" + end + unless execute('git', 'diff', '--exit-code') + abort 'uncommitted changes' + end unless execute('git', 'tag', tagname) abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' end @@ -132,10 +142,12 @@ class << Merger = Object.new unless relname raise ArgumentError, 'relname is not specified' end - if /^v/ !~ relname - tagname = "v#{relname.gsub(/[.-]/, '_')}" - else + if relname.start_with?('v') tagname = relname + elsif Integer(relname.split('.', 2).first) >= 4 + tagname = "v#{relname}" + else + tagname = "v#{relname.gsub(/[.-]/, '_')}" end execute('git', 'tag', '-d', tagname) @@ -260,7 +272,7 @@ else end # Merge revision from Git patch - git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" + git_uri = "https://github.com/ruby/ruby/commit/#{git_rev}.patch" resp = Net::HTTP.get_response(URI(git_uri)) if resp.code != '200' abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat index fcc75ea902..d39568fe86 100755 --- a/tool/missing-baseruby.bat +++ b/tool/missing-baseruby.bat @@ -18,6 +18,13 @@ : ; abort () { exit 1; } call :warn "executable host ruby is required. use --with-baseruby option." -call :warn "Note that BASERUBY must be Ruby 3.0.0 or later." +call :warn "Note that BASERUBY must be Ruby 3.1.0 or later." call :abort -: || (:^; abort if RUBY_VERSION < s[%r"warn .*Ruby ([\d.]+)(?:\.0)?",1]) +(goto :eof ^;) +verbose = true if ARGV[0] == "--verbose" +case +when !defined?(RubyVM::InstructionSequence) + abort(*(["BASERUBY must be CRuby"] if verbose)) +when RUBY_VERSION < s[%r[warn .*\KBASERUBY .*Ruby ([\d.]+)(?:\.0)?.*(?=\")],1] + abort(*(["#{$&}. Found: #{RUBY_VERSION}"] if verbose)) +end diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index 6e1f5c666a..5aa07962f9 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -282,17 +282,22 @@ def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_nam # Avoid generating fetches of lvars we don't need. This is imperfect as it # will match text inside strings or other false positives. - local_candidates = text.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + local_ptrs = [] + local_candidates = text.gsub(/\bLOCAL_PTR\(\K[a-zA-Z_][a-zA-Z0-9_]*(?=\))/) { + local_ptrs << $&; '' + }.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) f.puts '{' lineno += 1 # locals is nil outside methods locals&.reverse_each&.with_index{|param, i| next unless Symbol === param - next unless local_candidates.include?(param.to_s) + param = param.to_s + lvar = local_candidates.include?(param) + next unless lvar or local_ptrs.include?(param) f.puts "VALUE *const #{param}__ptr = (VALUE *)&ec->cfp->ep[#{-3 - i}];" - f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" - lineno += 1 + f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" if lvar + lineno += lvar ? 2 : 1 } f.puts "#line #{body_lineno} \"#{line_file}\"" lineno += 1 diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb index a83a3563f9..db74115730 100755 --- a/tool/mkconfig.rb +++ b/tool/mkconfig.rb @@ -64,8 +64,6 @@ File.foreach "config.status" do |line| when /^(?:X|(?:MINI|RUN|(?:HAVE_)?BASE|BOOTSTRAP|BTEST)RUBY(?:_COMMAND)?$)/; next when /^INSTALLDOC|TARGET$/; next when /^DTRACE/; next - when /^RJIT_(CC|SUPPORT)$/; # pass - when /^RJIT_/; next when /^(?:MAJOR|MINOR|TEENY)$/; vars[name] = val; next when /^LIBRUBY_D?LD/; next when /^RUBY_INSTALL_NAME$/; next vars[name] = (install_name = val).dup if $install_name @@ -396,6 +394,7 @@ print <<EOS ) end end +# Non-nil if configured for cross compiling. CROSS_COMPILING = nil unless defined? CROSS_COMPILING EOS diff --git a/tool/notes-github-pr.rb b/tool/notes-github-pr.rb new file mode 100644 index 0000000000..d69d479cdf --- /dev/null +++ b/tool/notes-github-pr.rb @@ -0,0 +1,138 @@ +#!/usr/bin/env ruby +# Add GitHub pull request reference / author info to git notes. + +require 'net/http' +require 'uri' +require 'tmpdir' +require 'json' +require 'yaml' + +# Conversion for people whose GitHub account name and SVN_ACCOUNT_NAME are different. +GITHUB_TO_SVN = { + 'amatsuda' => 'a_matsuda', + 'matzbot' => 'git', + 'jeremyevans' => 'jeremy', + 'znz' => 'kazu', + 'k-tsj' => 'ktsj', + 'nurse' => 'naruse', + 'ioquatix' => 'samuel', + 'suketa' => 'suke', + 'unak' => 'usa', +} + +EMAIL_YML_URL = 'https://raw.githubusercontent.com/ruby/git.ruby-lang.org/refs/heads/master/config/email.yml' +SVN_TO_EMAILS = YAML.safe_load(Net::HTTP.get_response(URI(EMAIL_YML_URL)).tap(&:value).body) + +class GitHub + ENDPOINT = URI.parse('https://api.github.com') + + def initialize(access_token) + @access_token = access_token + end + + # https://developer.github.com/changes/2019-04-11-pulls-branches-for-commit/ + def pulls(owner:, repo:, commit_sha:) + resp = get("/repos/#{owner}/#{repo}/commits/#{commit_sha}/pulls", accept: 'application/vnd.github.groot-preview+json') + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/pulls/#get-a-single-pull-request + def pull_request(owner:, repo:, number:) + resp = get("/repos/#{owner}/#{repo}/pulls/#{number}") + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/users/#get-a-single-user + def user(username:) + resp = get("/users/#{username}") + JSON.parse(resp.body) + end + + private + + def get(path, accept: 'application/vnd.github.v3+json') + Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: ENDPOINT.scheme == 'https') do |http| + headers = { 'Accept': accept, 'Authorization': "bearer #{@access_token}" } + http.get(path, headers).tap(&:value) + end + end +end + +module Git + class << self + def abbrev_ref(refname, repo_path:) + git('rev-parse', '--symbolic', '--abbrev-ref', refname, repo_path: repo_path).strip + end + + def rev_list(arg, first_parent: false, repo_path: nil) + git('rev-list', *[('--first-parent' if first_parent)].compact, arg, repo_path: repo_path).lines.map(&:chomp) + end + + def commit_message(sha) + git('log', '-1', '--pretty=format:%B', sha) + end + + def notes_message(sha) + git('log', '-1', '--pretty=format:%N', sha) + end + + def committer_name(sha) + git('log', '-1', '--pretty=format:%cn', sha) + end + + def committer_email(sha) + git('log', '-1', '--pretty=format:%cE', sha) + end + + private + + def git(*cmd, repo_path: nil) + env = {} + if repo_path + env['GIT_DIR'] = repo_path + end + out = IO.popen(env, ['git', *cmd], &:read) + unless $?.success? + abort "Failed to execute: git #{cmd.join(' ')}\n#{out}" + end + out + end + end +end + +github = GitHub.new(ENV.fetch('GITHUB_TOKEN')) + +repo_path, *rest = ARGV +rest.each_slice(3).map do |oldrev, newrev, _refname| + system('git', 'fetch', 'origin', 'refs/notes/commits:refs/notes/commits', exception: true) + + updated = false + Git.rev_list("#{oldrev}..#{newrev}", first_parent: true).each do |sha| + github.pulls(owner: 'ruby', repo: 'ruby', commit_sha: sha).each do |pull| + number = pull.fetch('number') + url = pull.fetch('html_url') + next unless url.start_with?('https://github.com/ruby/ruby/pull/') + + # "Merged" notes for "Squash and merge" + message = Git.commit_message(sha) + notes = Git.notes_message(sha) + if !message.include?(url) && !message.match(/[ (]##{number}[) ]/) && !notes.include?(url) + system('git', 'notes', 'append', '-m', "Merged: #{url}", sha, exception: true) + updated = true + end + + # "Merged-By" notes for "Rebase and merge" + if Git.committer_name(sha) == 'GitHub' && Git.committer_email(sha) == 'noreply@github.com' + username = github.pull_request(owner: 'ruby', repo: 'ruby', number: number).fetch('merged_by').fetch('login') + email = github.user(username: username).fetch('email') + email ||= SVN_TO_EMAILS[GITHUB_TO_SVN.fetch(username, username)]&.first + system('git', 'notes', 'append', '-m', "Merged-By: #{username}#{(" <#{email}>" if email)}", sha, exception: true) + updated = true + end + end + end + + if updated + system('git', 'push', 'origin', 'refs/notes/commits', exception: true) + end +end diff --git a/tool/notify-slack-commits.rb b/tool/notify-slack-commits.rb new file mode 100644 index 0000000000..73e22b9a03 --- /dev/null +++ b/tool/notify-slack-commits.rb @@ -0,0 +1,87 @@ +#!/usr/bin/env ruby + +require "net/https" +require "open3" +require "json" +require "digest/md5" + +SLACK_WEBHOOK_URLS = [ + ENV.fetch("SLACK_WEBHOOK_URL_ALERTS").chomp, # ruby-lang#alerts + ENV.fetch("SLACK_WEBHOOK_URL_COMMITS").chomp, # ruby-lang#commits + ENV.fetch("SLACK_WEBHOOK_URL_RUBY_JP").chomp, # ruby-jp#ruby-commits +] +GRAVATAR_OVERRIDES = { + "nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars0.githubusercontent.com/u/21976", + "noreply@github.com" => "https://avatars1.githubusercontent.com/u/9919", + "nurse@users.noreply.github.com" => "https://avatars1.githubusercontent.com/u/13423", + "svn-admin@ruby-lang.org" => "https://avatars1.githubusercontent.com/u/29403229", + "svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars1.githubusercontent.com/u/29403229", + "usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars2.githubusercontent.com/u/17790", + "usa@ruby-lang.org" => "https://avatars2.githubusercontent.com/u/17790", + "yui-knk@ruby-lang.org" => "https://avatars0.githubusercontent.com/u/5356517", + "znz@users.noreply.github.com" => "https://avatars3.githubusercontent.com/u/11857", +} + +def escape(s) + s.gsub(/[&<>]/, "&" => "&", "<" => "<", ">" => ">") +end + +ARGV.each_slice(3) do |oldrev, newrev, refname| + out, = Open3.capture2("git", "rev-parse", "--symbolic", "--abbrev-ref", refname) + branch = out.strip + + out, = Open3.capture2("git", "log", "--pretty=format:%H\n%h\n%cn\n%ce\n%ct\n%B", "--abbrev=10", "-z", "#{oldrev}..#{newrev}") + + attachments = [] + out.split("\0").reverse_each do |s| + sha, sha_abbr, committer, committeremail, committertime, body = s.split("\n", 6) + subject, body = body.split("\n", 2) + + # Append notes content to `body` if it's notes + if refname.match(%r[\Arefs/notes/\w+\z]) + # `--diff-filter=AM -M` to exclude rename by git's directory optimization + object = IO.popen(["git", "diff", "--diff-filter=AM", "-M", "--name-only", "#{sha}^..#{sha}"], &:read).chomp + if md = object.match(/\A(?<prefix>\h{2})\/?(?<rest>\h{38})\z/) + body = [body, IO.popen(["git", "notes", "show", md[:prefix] + md[:rest]], &:read)].join + end + end + + gravatar = GRAVATAR_OVERRIDES.fetch(committeremail) do + "https://www.gravatar.com/avatar/#{ Digest::MD5.hexdigest(committeremail.downcase) }" + end + + attachments << { + title: "#{ sha_abbr } (#{ branch }): #{ escape(subject) }", + title_link: "https://github.com/ruby/ruby/commit/#{ sha }", + text: escape((body || "").strip), + footer: committer, + footer_icon: gravatar, + ts: committertime.to_i, + color: '#24282D', + } + end + + # 100 attachments cannot be exceeded. 20 is recommended. https://api.slack.com/docs/message-attachments + attachments.each_slice(20).each do |attachments_group| + payload = { attachments: attachments_group } + + #Net::HTTP.post( + # URI.parse(SLACK_WEBHOOK_URL), + # JSON.generate(payload), + # "Content-Type" => "application/json" + #) + responses = SLACK_WEBHOOK_URLS.map do |url| + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.start do + req = Net::HTTP::Post.new(uri.path) + req.set_form_data(payload: payload.to_json) + http.request(req) + end + end + + results = responses.map { |resp| "#{resp.code} (#{resp.body})" }.join(', ') + puts "#{results} -- #{payload.to_json}" + end +end diff --git a/tool/outdate-bundled-gems.rb b/tool/outdate-bundled-gems.rb index c82d31d743..47ee80bc89 100755 --- a/tool/outdate-bundled-gems.rb +++ b/tool/outdate-bundled-gems.rb @@ -115,7 +115,7 @@ srcdir = Removal.new(ARGV.shift) curdir = !srcdir.base || File.identical?(srcdir.base, ".") ? srcdir : Removal.new bundled = File.readlines("#{srcdir.base}gems/bundled_gems"). - grep(/^(\w\S+)\s+\S+(?:\s+\S+\s+(\S+))?/) {$~.captures}.to_h rescue nil + grep(/^(\w[^\#\s]+)\s+[^\#\s]+(?:\s+[^\#\s]+\s+([^\#\s]+))?/) {$~.captures}.to_h rescue nil srcdir.glob(".bundle/gems/*/") do |dir| base = File.basename(dir) diff --git a/tool/prereq.status b/tool/prereq.status index 6de00c8a92..6aca615e90 100644 --- a/tool/prereq.status +++ b/tool/prereq.status @@ -41,4 +41,5 @@ s,@rubylibprefix@,,g s,@srcdir@,.,g s/@[A-Za-z][A-Za-z0-9_]*@//g -s/{\$([A-Za-z]*)}//g +s/{\$([^(){}]*)}//g +s/^!/#!/ diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index b194f2c9c2..874c3ef1d9 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -150,6 +150,7 @@ def parse_args(argv = ARGV) end $destdir ||= $mflags.defined?("DESTDIR") + $destdir = File.expand_path($destdir) unless $destdir.empty? if $extout ||= $mflags.defined?("EXTOUT") RbConfig.expand($extout) end @@ -172,6 +173,16 @@ def parse_args(argv = ARGV) end end +Compressors = {".gz"=>"gzip", ".bz2"=>"bzip2"} +def Compressors.for(type) + ext = File.extname(type) + if compress = fetch(ext, nil) + [type.chomp(ext), ext, compress] + else + [type, *find {|_, z| system(z, in: IO::NULL, out: IO::NULL)}] + end +end + $install_procs = Hash.new {[]} def install?(*types, &block) unless types.delete(:nodefault) @@ -516,6 +527,8 @@ module RbInstall const_set(:FileUtils, fu::NoWrite) fu end + # RubyGems 3.0.0 or later supports `dir_mode`, but it uses + # `File` method to apply it, not `FileUtils`. dir_mode = options.delete(:dir_mode) if options end yield @@ -648,6 +661,17 @@ module RbInstall "#{srcdir}/lib" end end + + class UnpackedGem < self + def collect + base = @srcdir or return [] + Dir.glob("**/*", File::FNM_DOTMATCH, base: base).select do |n| + next if n == "." + next if File.fnmatch?("*.gemspec", n, File::FNM_DOTMATCH|File::FNM_PATHNAME) + !File.directory?(File.join(base, n)) + end + end + end end end @@ -685,6 +709,77 @@ module RbInstall end class UnpackedInstaller < Gem::Installer + # This method is mostly copied from old version of Gem::Installer#install + def install_with_default_gem + verify_gem_home + + # The name and require_paths must be verified first, since it could contain + # ruby code that would be eval'ed in #ensure_loadable_spec + verify_spec + + ensure_loadable_spec + + if options[:install_as_default] + Gem.ensure_default_gem_subdirectories gem_home + else + Gem.ensure_gem_subdirectories gem_home + end + + return true if @force + + ensure_dependencies_met unless @ignore_dependencies + + run_pre_install_hooks + + # Set loaded_from to ensure extension_dir is correct + if @options[:install_as_default] + spec.loaded_from = default_spec_file + else + spec.loaded_from = spec_file + end + + # Completely remove any previous gem files + FileUtils.rm_rf gem_dir + FileUtils.rm_rf spec.extension_dir + + dir_mode = options[:dir_mode] + FileUtils.mkdir_p gem_dir, mode: dir_mode && 0o755 + + if @options[:install_as_default] + extract_bin + write_default_spec + else + extract_files + + build_extensions + write_build_info_file + run_post_build_hooks + end + + generate_bin + generate_plugins + + unless @options[:install_as_default] + write_spec + write_cache_file + end + + File.chmod(dir_mode, gem_dir) if dir_mode + + say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? + + Gem::Specification.add_spec(spec) unless @install_dir + + load_plugin + + run_post_install_hooks + + spec + rescue Errno::EACCES => e + # Permission denied - /path/to/foo + raise Gem::FilePermissionError, e.message.split(" - ").last + end + def write_cache_file end @@ -730,7 +825,7 @@ module RbInstall def install spec.post_install_message = nil dir_creating(without_destdir(gem_dir)) - RbInstall.no_write(options) {super} + RbInstall.no_write(options) { install_with_default_gem } end # Now build-ext builds all extensions including bundled gems. @@ -759,32 +854,45 @@ module RbInstall $installed_list.puts(d+"/") if $installed_list end end + + def load_plugin + # Suppress warnings for constant re-assignment + verbose, $VERBOSE = $VERBOSE, nil + super + ensure + $VERBOSE = verbose + end end end -def load_gemspec(file, base = nil) +def load_gemspec(file, base = nil, files: nil) file = File.realpath(file) code = File.read(file, encoding: "utf-8:-") - files = [] - Dir.glob("**/*", File::FNM_DOTMATCH, base: base) do |n| - case File.basename(n); when ".", ".."; next; end - next if File.directory?(File.join(base, n)) - files << n.dump - end if base + code.gsub!(/^ *#.*/, "") + spec_files = files ? files.map(&:dump).join(", ") : "" code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split(\([^\)]*\))?/m) do - "[" + files.join(", ") + "]" - end + "[" + spec_files + "]" + end \ + or code.gsub!(/IO\.popen\(.*git.*?\)/) do - "[" + files.join(", ") + "] || itself" + "[" + spec_files + "] || itself" end spec = eval(code, binding, file) + # for out-of-place build + collected_files = files ? spec.files.concat(files).uniq : spec.files + spec.files = collected_files.map do |f| + if !File.exist?(File.join(base || ".", f)) && f.end_with?(".rb") + "lib/#{f}" + else + f + end + end unless Gem::Specification === spec raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." end spec.loaded_from = base ? File.join(base, File.basename(file)) : file - spec.files.reject! {|n| n.end_with?(".gemspec") or n.start_with?(".git")} spec.date = RUBY_RELEASE_DATE spec @@ -795,6 +903,7 @@ def install_default_gem(dir, srcdir, bindir) install_dir = with_destdir(gem_dir) prepare "default gems from #{dir}", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_default_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -813,14 +922,11 @@ def install_default_gem(dir, srcdir, bindir) base = "#{srcdir}/#{dir}" gems = Dir.glob("**/*.gemspec", base: base).map {|src| - spec = load_gemspec("#{base}/#{src}") - file_collector = RbInstall::Specs::FileCollector.for(srcdir, dir, src) - files = file_collector.collect + files = RbInstall::Specs::FileCollector.for(srcdir, dir, src).collect if files.empty? next end - spec.files = files - spec + load_gemspec("#{base}/#{src}", files: files) } gems.compact.sort_by(&:name).each do |gemspec| old_gemspecs = Dir[File.join(with_destdir(default_spec_dir), "#{gemspec.name}-*.gemspec")] @@ -839,6 +945,10 @@ def install_default_gem(dir, srcdir, bindir) end end +def mdoc_file?(mdoc) + /^\.Nm / =~ File.read(mdoc, 1024) +end + # :startdoc: install?(:local, :arch, :bin, :'bin-arch') do @@ -926,8 +1036,6 @@ end install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do prepare "extension headers", archhdrdir install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_rjit_header-*.obj", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_rjit_header-*.pch", :mode => $data_mode) end install?(:ext, :comm, :'ext-comm') do @@ -992,14 +1100,9 @@ install?(:local, :comm, :man) do mdocs = Dir["#{srcdir}/man/*.[1-9]"] prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} - case $mantype - when /\.(?:(gz)|bz2)\z/ - compress = $1 ? "gzip" : "bzip2" - suffix = $& - end - mandir = File.join(mandir, "man") + mantype, suffix, compress = Compressors.for($mantype) has_goruby = File.exist?(goruby_install_name+exeext) - require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ mantype mdocs.each do |mdoc| next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' base = File.basename(mdoc) @@ -1007,11 +1110,11 @@ install?(:local, :comm, :man) do next unless has_goruby end - destdir = mandir + (section = mdoc[/\d+$/]) - destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) + destdir = File.join(mandir, "man" + (section = mdoc[/\d+$/])) + destname = $script_installer.transform(base.chomp(".#{section}")) destfile = File.join(destdir, "#{destname}.#{section}") - if /\Adoc\b/ =~ $mantype + if /\Adoc\b/ =~ mantype or !mdoc_file?(mdoc) if compress begin w = IO.popen(compress, "rb", in: mdoc, &:read) @@ -1029,13 +1132,8 @@ install?(:local, :comm, :man) do class << (w = []) alias print push end - if File.basename(mdoc).start_with?('bundle') || - File.basename(mdoc).start_with?('gemfile') - w = File.read(mdoc) - else - File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} - w = w.join("") - end + File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + w = w.join("") if compress begin w = IO.popen(compress, "r+b") do |f| @@ -1099,6 +1197,7 @@ install?(:ext, :comm, :gem, :'bundled-gems') do install_dir = with_destdir(gem_dir) prepare "bundled gems", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -1127,28 +1226,30 @@ install?(:ext, :comm, :gem, :'bundled-gems') do # the newly installed ruby. ENV.delete('RUBYOPT') + collector = RbInstall::Specs::FileCollector::UnpackedGem File.foreach("#{srcdir}/gems/bundled_gems") do |name| next if /^\s*(?:#|$)/ =~ name next unless /^(\S+)\s+(\S+).*/ =~ name gem = $1 gem_name = "#$1-#$2" - # Try to find the original gemspec file - path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec" - unless File.exist?(path) - # Try to find the gemspec file for C ext gems + path = [ + # gemspec that removed duplicated dependencies of bundled gems + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec", + # gemspec for C ext gems, It has the original dependencies # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec - # This gemspec keep the original dependencies - path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec" - unless File.exist?(path) - # Try to find the gemspec file for gems that hasn't own gemspec - path = "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" - unless File.exist?(path) - skipped[gem_name] = "gemspec not found" - next - end - end + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec", + # original gemspec generated by rubygems + "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + ].find { |gemspec| File.exist?(gemspec) } + if path.nil? + skipped[gem_name] = "gemspec not found" + next end - spec = load_gemspec(path, "#{srcdir}/.bundle/gems/#{gem_name}") + base = "#{srcdir}/.bundle/gems/#{gem_name}" + files = collector.new(path, base, nil).collect + files.delete("#{gem}.gemspec") + files.delete("#{gem_name}.gemspec") + spec = load_gemspec(path, base, files: files) unless spec.platform == Gem::Platform::RUBY skipped[gem_name] = "not ruby platform (#{spec.platform})" next @@ -1163,6 +1264,7 @@ install?(:ext, :comm, :gem, :'bundled-gems') do next end spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + package = RbInstall::DirPackage.new spec ins = RbInstall::UnpackedInstaller.new(package, options) puts "#{INDENT}#{spec.name} #{spec.version}" @@ -1182,7 +1284,8 @@ install?(:ext, :comm, :gem, :'bundled-gems') do skipped.default = "not found in bundled_gems" puts "skipped bundled gems:" gems.each do |gem| - printf " %-32s%s\n", File.basename(gem), skipped[gem] + gem = File.basename(gem) + printf " %-31s %s\n", gem, skipped[gem.chomp(".gem")] end end end diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests index 873db5a641..4bcb5707a5 100644 --- a/tool/rbs_skip_tests +++ b/tool/rbs_skip_tests @@ -37,6 +37,9 @@ TestInstanceNetHTTPResponse depending on external resources test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` +# Failing because ObjectSpace.count_nodes has been removed +test_count_nodes(ObjectSpaceTest) + ## Unknown failures # NoMethodError: undefined method 'inspect' for an instance of RBS::UnitTest::Convertibles::ToInt @@ -45,4 +48,10 @@ test_linear_time?(RegexpSingletonTest) test_new(RegexpSingletonTest) ## Failed tests caused by unreleased version of Ruby +test_source_location(MethodInstanceTest) +test_source_location(ProcInstanceTest) +test_source_location(UnboundMethodInstanceTest) +# Errno::ENOENT: No such file or directory - bundle +test_collection_install__pathname_set(RBS::CliTest) +test_collection_install__set_pathname__manifest(RBS::CliTest) diff --git a/tool/rbs_skip_tests_windows b/tool/rbs_skip_tests_windows new file mode 100644 index 0000000000..db12c69419 --- /dev/null +++ b/tool/rbs_skip_tests_windows @@ -0,0 +1,111 @@ +ARGFTest Failing on Windows + +RactorSingletonTest Hangs up on Windows +RactorInstanceTest Hangs up on Windows + +# NotImplementedError: fileno() function is unimplemented on this machine +test_fileno(DirInstanceTest) +test_fchdir(DirSingletonTest) +test_for_fd(DirSingletonTest) + +# ArgumentError: user root doesn't exist +test_home(DirSingletonTest) + +# NameError: uninitialized constant Etc::CS_PATH +test_confstr(EtcSingletonTest) + +# NameError: uninitialized constant Etc::SC_ARG_MAX +test_sysconf(EtcSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-udw6rx/chmod +test_chmod(FileInstanceTest) +test_chmod(FileInstanceTest) +test_truncate(FileInstanceTest) + +# Errno::EISDIR: Is a directory @ rb_sysopen - C:/a/ruby/ruby/src/gems/src/rbs/test/stdlib +test_directory?(FileSingletonTest) + +# NotImplementedError: lutime() function is unimplemented on this machine +test_lutime(FileSingletonTest) + +# NotImplementedError: mkfifo() function is unimplemented on this machine +test_mkfifo(FileSingletonTest) + +# Returns `nil` on Windows +test_getgrgid(EtcSingletonTest) +test_getgrnam(EtcSingletonTest) +test_getpwnam(EtcSingletonTest) +test_getpwuid(EtcSingletonTest) + +# Returns `false` +test_setgid?(FileSingletonTest) +test_setuid?(FileSingletonTest) +test_sticky?(FileSingletonTest) + +test_world_readable?(FileSingletonTest) # Returns `420` +test_world_readable?(FileStatInstanceTest) # Returns `420` +test_world_writable?(FileSingletonTest) # Returns `nil` +test_dev_major(FileStatInstanceTest) # Returns `nil` +test_dev_minor(FileStatInstanceTest) # Returns `nil` +test_rdev_major(FileStatInstanceTest) # Returns `nil` +test_rdev_minor(FileStatInstanceTest) # Returns `nil` + +# ArgumentError: wrong number of arguments (given -403772944, expected 0+) +test_curry(MethodInstanceTest) + +# ArgumentError: no output encoding given +test_tolocale(KconvSingletonTest) + +# Errno::EINVAL: Invalid argument - : +test_system(KernelInstanceTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_load(OpenSSLConfigSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - +test_parse(OpenSSLConfigSingletonTest) +test_parse_config(OpenSSLConfigSingletonTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_each(OpenSSLConfigTest) +test_lookup_and_set(OpenSSLConfigTest) +test_sections(OpenSSLConfigTest) + +# OpenSSL::SSL::SSLError: SSL_connect returned=1 errno=0 peeraddr=185.199.108.153:443 state=error: certificate verify failed (unable to get local issuer certificate) +test_URI_open(OpenURISingletonTest) + +# ArgumentError: both textmode and binmode specified +test_binwrite(PathnameInstanceTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-delete-test20250813-10156-mb3e9i +test_delete(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-binwrite-test20250813-10156-sh8145 +test_open(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_truncate(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_unlink(PathnameInstanceTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /etc/resolv.conf +test_parse_resolv_conf(ResolvDNSConfigSingletonTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvInstanceTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvSingletonTest) + +# ArgumentError: unsupported signal 'SIGUSR2' +test_trap(SignalSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-mgr4tx +test_create(TempfileSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-hp9nzu +test_initialize(TempfileSingletonTest) +test_new(TempfileSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-f8z9pn/test.gz +test_open(ZlibGzipReaderSingletonTest) + +# Errno::EACCES: Permission denied @ rb_file_s_rename +# D:/a/ruby/ruby/src/lib/rubygems/util/atomic_file_writer.rb:42:in 'File.rename' +test_write_binary(GemSingletonTest) diff --git a/tool/rdoc-srcdir b/tool/rdoc-srcdir index 03e5171650..ecc49b4b2c 100755 --- a/tool/rdoc-srcdir +++ b/tool/rdoc-srcdir @@ -1,6 +1,9 @@ #!ruby -W0 -$:.unshift(File.expand_path("../lib", __dir__)) +%w[tsort rdoc].each do |lib| + path = Dir.glob("#{File.dirname(__dir__)}/.bundle/gems/#{lib}-*").first + $LOAD_PATH.unshift("#{path}/lib") +end require 'rdoc/rdoc' # Make only the output directory relative to the invoked directory. @@ -16,7 +19,7 @@ options.title = options.title.sub(/Ruby \K.*version/) { .sort # "MAJOR" < "MINOR", fortunately .to_h.values.join(".") } -options.parse ARGV +options.parse ARGV + ["#{invoked}/rbconfig.rb"] options.singleton_class.define_method(:finish) do super() diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index 7f08eb8d1a..95a9688cb2 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -190,7 +190,7 @@ def backport_command_string next false if c.match(/\A\d{1,6}\z/) # skip SVN revision # check if the Git revision is included in master - has_commit(c, "master") + has_commit(c, "origin/master") end.sort_by do |changeset| Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) end diff --git a/tool/releng/gen-mail.rb b/tool/releng/gen-mail.rb index 6dc0e4cec1..17fa499d69 100755 --- a/tool/releng/gen-mail.rb +++ b/tool/releng/gen-mail.rb @@ -10,7 +10,7 @@ end # Confirm current directory is www.ruby-lang.org's working directory def confirm_w_r_l_o_wd File.foreach('.git/config') do |line| - return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + return true if line.include?('ruby/www.ruby-lang.org.git') end abort "Run this script in www.ruby-lang.org's working directory" end diff --git a/tool/releng/update-www-meta.rb b/tool/releng/update-www-meta.rb index 8a5651dcd0..0dd5b25631 100755 --- a/tool/releng/update-www-meta.rb +++ b/tool/releng/update-www-meta.rb @@ -1,6 +1,7 @@ #!/usr/bin/env ruby require "open-uri" require "yaml" +require_relative "../ruby-version" class Tarball attr_reader :version, :size, :sha1, :sha256, :sha512 @@ -41,22 +42,7 @@ eom unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" info = YAML.load(URI(uri).read) @@ -74,9 +60,10 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) rubydir = File.expand_path(File.join(__FILE__, '../../../')) puts %`git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` @@ -155,7 +142,7 @@ eom date = Time.now.utc # use utc to use previous day in midnight entry = <<eom - version: #{ver} - tag: v#{ver.tr('-.', '_')} + tag: #{RubyVersion.tag(ver)} date: #{date.strftime("%Y-%m-%d")} post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ stats: diff --git a/tool/rjit/bindgen.rb b/tool/rjit/bindgen.rb deleted file mode 100755 index 4105052dd2..0000000000 --- a/tool/rjit/bindgen.rb +++ /dev/null @@ -1,666 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -ENV['GEM_HOME'] = File.expand_path('./.bundle', __dir__) -require 'rubygems/source' -require 'bundler/inline' -gemfile(true) do - source 'https://rubygems.org' - gem 'ffi-clang', '0.7.0', require: false -end - -# Help ffi-clang find libclang -# Hint: apt install libclang1 -ENV['LIBCLANG'] ||= Dir.glob("/usr/lib/llvm-*/lib/libclang.so.1").grep_v(/-cpp/).sort.last -require 'ffi/clang' - -require 'etc' -require 'fiddle/import' -require 'set' - -unless build_dir = ARGV.first - abort "Usage: #{$0} BUILD_DIR" -end - -class Node < Struct.new( - :kind, - :spelling, - :type, - :typedef_type, - :bitwidth, - :sizeof_type, - :offsetof, - :enum_value, - :children, - keyword_init: true, -) -end - -# Parse a C header with ffi-clang and return Node objects. -# To ease the maintenance, ffi-clang should be used only inside this class. -class HeaderParser - def initialize(header, cflags:) - @translation_unit = FFI::Clang::Index.new.parse_translation_unit(header, cflags, [], {}) - end - - def parse - parse_children(@translation_unit.cursor) - end - - private - - def parse_children(cursor) - children = [] - cursor.visit_children do |cursor, _parent| - children << parse_cursor(cursor) - next :continue - end - children - end - - def parse_cursor(cursor) - unless cursor.kind.start_with?('cursor_') - raise "unexpected cursor kind: #{cursor.kind}" - end - kind = cursor.kind.to_s.delete_prefix('cursor_').to_sym - children = parse_children(cursor) - - offsetof = {} - if kind == :struct - children.select { |c| c.kind == :field_decl }.each do |child| - offsetof[child.spelling] = cursor.type.offsetof(child.spelling) - end - end - - sizeof_type = nil - if %i[struct union].include?(kind) - sizeof_type = cursor.type.sizeof - end - - enum_value = nil - if kind == :enum_constant_decl - enum_value = cursor.enum_value - end - - Node.new( - kind: kind, - spelling: cursor.spelling, - type: cursor.type.spelling, - typedef_type: cursor.typedef_type.spelling, - bitwidth: cursor.bitwidth, - sizeof_type: sizeof_type, - offsetof: offsetof, - enum_value: enum_value, - children: children, - ) - end -end - -# Convert Node objects to a Ruby binding source. -class BindingGenerator - BINDGEN_BEG = '### RJIT bindgen begin ###' - BINDGEN_END = '### RJIT bindgen end ###' - DEFAULTS = { '_Bool' => 'CType::Bool.new' } - DEFAULTS.default_proc = proc { |_h, k| "CType::Stub.new(:#{k})" } - - attr_reader :src - - # @param src_path [String] - # @param consts [Hash{ Symbol => Array<String> }] - # @param values [Hash{ Symbol => Array<String> }] - # @param funcs [Array<String>] - # @param types [Array<String>] - # @param dynamic_types [Array<String>] #ifdef-dependent immediate types, which need Primitive.cexpr! for type detection - # @param skip_fields [Hash{ Symbol => Array<String> }] Struct fields that are skipped from bindgen - # @param ruby_fields [Hash{ Symbol => Array<String> }] Struct VALUE fields that are considered Ruby objects - def initialize(src_path:, consts:, values:, funcs:, types:, dynamic_types:, skip_fields:, ruby_fields:) - @preamble, @postamble = split_ambles(src_path) - @src = String.new - @consts = consts.transform_values(&:sort) - @values = values.transform_values(&:sort) - @funcs = funcs.sort - @types = types.sort - @dynamic_types = dynamic_types.sort - @skip_fields = skip_fields.transform_keys(&:to_s) - @ruby_fields = ruby_fields.transform_keys(&:to_s) - @references = Set.new - end - - def generate(nodes) - println @preamble - - # Define macros/enums - @consts.each do |type, values| - values.each do |value| - raise "#{value} isn't a valid constant name" unless ('A'..'Z').include?(value[0]) - println " C::#{value} = Primitive.cexpr! %q{ #{type}2NUM(#{value}) }" - end - end - println - - # Define variables - @values.each do |type, values| - values.each do |value| - println " def C.#{value} = Primitive.cexpr!(%q{ #{type}2NUM(#{value}) })" - end - end - println - - # Define function pointers - @funcs.each do |func| - println " def C.#{func}" - println " Primitive.cexpr! %q{ SIZET2NUM((size_t)#{func}) }" - println " end" - println - end - - # Build a hash table for type lookup by name - nodes_index = flatten_nodes(nodes).group_by(&:spelling).transform_values do |values| - # Try to search a declaration with definitions - node_with_children = values.find { |v| !v.children.empty? } - next node_with_children if node_with_children - - # Otherwise, assume the last one is the main declaration - values.last - end - - # Define types - @types.each do |type| - unless definition = generate_node(nodes_index[type]) - raise "Failed to find or generate type: #{type}" - end - println " def C.#{type}" - println "@#{type} ||= #{definition}".gsub(/^/, " ").chomp - println " end" - println - end - - # Define dynamic types - @dynamic_types.each do |type| - unless generate_node(nodes_index[type])&.start_with?('CType::Immediate') - raise "Non-immediate type is given to dynamic_types: #{type}" - end - # Only one Primitive.cexpr! is allowed for each line: https://github.com/ruby/ruby/pull/9612 - println " def C.#{type}" - println " @#{type} ||= CType::Immediate.find(" - println " Primitive.cexpr!(\"SIZEOF(#{type})\")," - println " Primitive.cexpr!(\"SIGNED_TYPE_P(#{type})\")," - println " )" - println " end" - println - end - - # Leave a stub for types that are referenced but not targeted - (@references - @types - @dynamic_types).each do |type| - println " def C.#{type}" - println " #{DEFAULTS[type]}" - println " end" - println - end - - print @postamble - end - - private - - # Make an array that includes all top-level and nested nodes - def flatten_nodes(nodes) - result = [] - nodes.each do |node| - unless node.children.empty? - result.concat(flatten_nodes(node.children)) - end - end - result.concat(nodes) # prioritize top-level nodes - result - end - - # Return code before BINDGEN_BEG and code after BINDGEN_END - def split_ambles(src_path) - lines = File.read(src_path).lines - - preamble_end = lines.index { |l| l.include?(BINDGEN_BEG) } - raise "`#{BINDGEN_BEG}` was not found in '#{src_path}'" if preamble_end.nil? - - postamble_beg = lines.index { |l| l.include?(BINDGEN_END) } - raise "`#{BINDGEN_END}` was not found in '#{src_path}'" if postamble_beg.nil? - raise "`#{BINDGEN_BEG}` was found after `#{BINDGEN_END}`" if preamble_end >= postamble_beg - - return lines[0..preamble_end].join, lines[postamble_beg..-1].join - end - - # Generate code from a node. Used for constructing a complex nested node. - # @param node [Node] - def generate_node(node, sizeof_type: nil) - case node&.kind - when :struct, :union - # node.spelling is often empty for union, but we'd like to give it a name when it has one. - buf = +"CType::#{node.kind.to_s.sub(/\A[a-z]/, &:upcase)}.new(\n" - buf << " \"#{node.spelling}\", Primitive.cexpr!(\"SIZEOF(#{sizeof_type || node.type})\"),\n" - bit_fields_end = node.children.index { |c| c.bitwidth == -1 } || node.children.size # first non-bit field index - node.children.each_with_index do |child, i| - skip_type = sizeof_type&.gsub(/\(\(struct ([^\)]+) \*\)NULL\)->/, '\1.') || node.spelling - next if @skip_fields.fetch(skip_type, []).include?(child.spelling) - field_builder = proc do |field, type| - if node.kind == :struct - to_ruby = @ruby_fields.fetch(node.spelling, []).include?(field) - if child.bitwidth > 0 - if bit_fields_end <= i # give up offsetof calculation for non-leading bit fields - raise "non-leading bit fields are not supported. consider including '#{field}' in skip_fields." - end - offsetof = node.offsetof.fetch(field) - else - off_type = sizeof_type || "(*((#{node.type} *)NULL))" - offsetof = "Primitive.cexpr!(\"OFFSETOF(#{off_type}, #{field})\")" - end - " #{field}: [#{type}, #{offsetof}#{', true' if to_ruby}],\n" - else - " #{field}: #{type},\n" - end - end - - case child - # BitField is struct-specific. So it must be handled here. - in Node[kind: :field_decl, spelling:, bitwidth:, children: [_grandchild, *]] if bitwidth > 0 - buf << field_builder.call(spelling, "CType::BitField.new(#{bitwidth}, #{node.offsetof.fetch(spelling) % 8})") - # "(unnamed ...)" struct and union are handled here, which are also struct-specific. - in Node[kind: :field_decl, spelling:, type:, children: [grandchild]] if type.match?(/\((unnamed|anonymous) [^)]+\)\z/) - if sizeof_type - child_type = "#{sizeof_type}.#{child.spelling}" - else - child_type = "((#{node.type} *)NULL)->#{child.spelling}" - end - buf << field_builder.call(spelling, generate_node(grandchild, sizeof_type: child_type).gsub(/^/, ' ').sub(/\A +/, '')) - # In most cases, we'd like to let generate_type handle the type unless it's "(unnamed ...)". - in Node[kind: :field_decl, spelling:, type:] if !type.empty? - buf << field_builder.call(spelling, generate_type(type)) - else # forward declarations are ignored - end - end - buf << ")" - when :typedef_decl - case node.children - in [child] - generate_node(child) - in [child, Node[kind: :integer_literal]] - generate_node(child) - in _ unless node.typedef_type.empty? - generate_type(node.typedef_type) - end - when :enum_decl - generate_type('int') - when :type_ref - generate_type(node.spelling) - end - end - - # Generate code from a type name. Used for resolving the name of a simple leaf node. - # @param type [String] - def generate_type(type) - if type.match?(/\[\d+\]\z/) - return "CType::Array.new { #{generate_type(type.sub!(/\[\d+\]\z/, ''))} }" - end - type = type.delete_suffix('const') - if type.end_with?('*') - if type == 'const void *' - # `CType::Pointer.new { CType::Immediate.parse("void") }` is never useful, - # so specially handle that case here. - return 'CType::Immediate.parse("void *")' - end - return "CType::Pointer.new { #{generate_type(type.delete_suffix('*').rstrip)} }" - end - - type = type.gsub(/((const|volatile) )+/, '').rstrip - if type.start_with?(/(struct|union|enum) /) - target = type.split(' ', 2).last - push_target(target) - "self.#{target}" - else - begin - ctype = Fiddle::Importer.parse_ctype(type) - rescue Fiddle::DLError - push_target(type) - "self.#{type}" - else - # Convert any function pointers to void* to workaround FILE* vs int* - if ctype == Fiddle::TYPE_VOIDP - "CType::Immediate.parse(\"void *\")" - else - "CType::Immediate.parse(#{type.dump})" - end - end - end - end - - def print(str) - @src << str - end - - def println(str = "") - @src << str << "\n" - end - - def chomp - @src.delete_suffix!("\n") - end - - def rstrip! - @src.rstrip! - end - - def push_target(target) - unless target.match?(/\A\w+\z/) - raise "invalid target: #{target}" - end - @references << target - end -end - -src_dir = File.expand_path('../..', __dir__) -src_path = File.join(src_dir, 'rjit_c.rb') -build_dir = File.expand_path(build_dir) -cflags = [ - src_dir, - build_dir, - File.join(src_dir, 'include'), - File.join(build_dir, ".ext/include/#{RUBY_PLATFORM}"), -].map { |dir| "-I#{dir}" } - -# Clear .cache/clangd created by the language server, which could break this bindgen -clangd_cache = File.join(src_dir, '.cache/clangd') -if Dir.exist?(clangd_cache) - system('rm', '-rf', clangd_cache, exception: true) -end - -# Parse rjit_c.h and generate rjit_c.rb -nodes = HeaderParser.new(File.join(src_dir, 'rjit_c.h'), cflags: cflags).parse -generator = BindingGenerator.new( - src_path: src_path, - consts: { - LONG: %w[ - UNLIMITED_ARGUMENTS - VM_ENV_DATA_INDEX_ME_CREF - VM_ENV_DATA_INDEX_SPECVAL - ], - SIZET: %w[ - ARRAY_REDEFINED_OP_FLAG - BOP_AND - BOP_AREF - BOP_EQ - BOP_EQQ - BOP_FREEZE - BOP_GE - BOP_GT - BOP_LE - BOP_LT - BOP_MINUS - BOP_MOD - BOP_OR - BOP_PLUS - BUILTIN_ATTR_LEAF - HASH_REDEFINED_OP_FLAG - INTEGER_REDEFINED_OP_FLAG - INVALID_SHAPE_ID - METHOD_VISI_PRIVATE - METHOD_VISI_PROTECTED - METHOD_VISI_PUBLIC - METHOD_VISI_UNDEF - OBJ_TOO_COMPLEX_SHAPE_ID - OPTIMIZED_METHOD_TYPE_BLOCK_CALL - OPTIMIZED_METHOD_TYPE_CALL - OPTIMIZED_METHOD_TYPE_SEND - OPTIMIZED_METHOD_TYPE_STRUCT_AREF - OPTIMIZED_METHOD_TYPE_STRUCT_ASET - RARRAY_EMBED_FLAG - RARRAY_EMBED_LEN_MASK - RARRAY_EMBED_LEN_SHIFT - RMODULE_IS_REFINEMENT - ROBJECT_EMBED - RSTRUCT_EMBED_LEN_MASK - RUBY_EVENT_CLASS - RUBY_EVENT_C_CALL - RUBY_EVENT_C_RETURN - RUBY_FIXNUM_FLAG - RUBY_FLONUM_FLAG - RUBY_FLONUM_MASK - RUBY_IMMEDIATE_MASK - RUBY_SPECIAL_SHIFT - RUBY_SYMBOL_FLAG - RUBY_T_ARRAY - RUBY_T_CLASS - RUBY_T_ICLASS - RUBY_T_HASH - RUBY_T_MASK - RUBY_T_MODULE - RUBY_T_STRING - RUBY_T_SYMBOL - RUBY_T_OBJECT - SHAPE_FLAG_SHIFT - SHAPE_FROZEN - SHAPE_ID_NUM_BITS - SHAPE_IVAR - SHAPE_MASK - SHAPE_ROOT - STRING_REDEFINED_OP_FLAG - T_OBJECT - VM_BLOCK_HANDLER_NONE - VM_CALL_ARGS_BLOCKARG - VM_CALL_ARGS_SPLAT - VM_CALL_FCALL - VM_CALL_FORWARDING - VM_CALL_KWARG - VM_CALL_KW_SPLAT - VM_CALL_KW_SPLAT_MUT - VM_CALL_KW_SPLAT_bit - VM_CALL_OPT_SEND - VM_CALL_TAILCALL - VM_CALL_TAILCALL_bit - VM_CALL_ZSUPER - VM_ENV_DATA_INDEX_FLAGS - VM_ENV_DATA_SIZE - VM_ENV_FLAG_LOCAL - VM_ENV_FLAG_WB_REQUIRED - VM_FRAME_FLAG_BMETHOD - VM_FRAME_FLAG_CFRAME - VM_FRAME_FLAG_CFRAME_KW - VM_FRAME_FLAG_LAMBDA - VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM - VM_FRAME_MAGIC_BLOCK - VM_FRAME_MAGIC_CFUNC - VM_FRAME_MAGIC_METHOD - VM_METHOD_TYPE_ALIAS - VM_METHOD_TYPE_ATTRSET - VM_METHOD_TYPE_BMETHOD - VM_METHOD_TYPE_CFUNC - VM_METHOD_TYPE_ISEQ - VM_METHOD_TYPE_IVAR - VM_METHOD_TYPE_MISSING - VM_METHOD_TYPE_NOTIMPLEMENTED - VM_METHOD_TYPE_OPTIMIZED - VM_METHOD_TYPE_REFINED - VM_METHOD_TYPE_UNDEF - VM_METHOD_TYPE_ZSUPER - VM_SPECIAL_OBJECT_VMCORE - RUBY_ENCODING_MASK - RUBY_FL_FREEZE - RHASH_PASS_AS_KEYWORDS - ], - }, - values: { - SIZET: %w[ - block_type_iseq - imemo_iseq - imemo_callinfo - rb_block_param_proxy - rb_cArray - rb_cFalseClass - rb_cFloat - rb_cInteger - rb_cNilClass - rb_cString - rb_cSymbol - rb_cTrueClass - rb_rjit_global_events - rb_mRubyVMFrozenCore - rb_vm_insns_count - idRespond_to_missing - ], - }, - funcs: %w[ - rb_ary_entry_internal - rb_ary_push - rb_ary_resurrect - rb_ary_store - rb_ec_ary_new_from_values - rb_ec_str_resurrect - rb_ensure_iv_list_size - rb_fix_aref - rb_fix_div_fix - rb_fix_mod_fix - rb_fix_mul_fix - rb_gc_writebarrier - rb_get_symbol_id - rb_hash_aref - rb_hash_aset - rb_hash_bulk_insert - rb_hash_new - rb_hash_new_with_size - rb_hash_resurrect - rb_ivar_get - rb_obj_as_string_result - rb_obj_is_kind_of - rb_str_concat_literals - rb_str_eql_internal - rb_str_getbyte - rb_vm_bh_to_procval - rb_vm_concat_array - rb_vm_defined - rb_vm_get_ev_const - rb_vm_getclassvariable - rb_vm_ic_hit_p - rb_vm_opt_newarray_min - rb_vm_opt_newarray_max - rb_vm_opt_newarray_hash - rb_vm_opt_newarray_pack - rb_vm_setinstancevariable - rb_vm_splat_array - rjit_full_cfunc_return - rjit_optimized_call - rjit_str_neq_internal - rjit_record_exit_stack - rb_ivar_defined - rb_vm_throw - rb_backref_get - rb_reg_last_match - rb_reg_match_pre - rb_reg_match_post - rb_reg_match_last - rb_reg_nth_match - rb_gvar_get - rb_range_new - rb_ary_tmp_new_from_values - rb_reg_new_ary - rb_ary_clear - rb_str_intern - rb_vm_setclassvariable - rb_str_bytesize - rjit_str_simple_append - rb_str_buf_append - rb_str_dup - rb_vm_yield_with_cfunc - rb_vm_set_ivar_id - rb_ary_dup - rjit_rb_ary_subseq_length - rb_ary_unshift_m - rjit_build_kwhash - rb_rjit_entry_stub_hit - rb_rjit_branch_stub_hit - rb_sym_to_proc - ], - types: %w[ - CALL_DATA - IC - ID - IVC - RArray - RB_BUILTIN - RBasic - RObject - RStruct - RString - attr_index_t - iseq_inline_constant_cache - iseq_inline_constant_cache_entry - iseq_inline_iv_cache_entry - iseq_inline_storage_entry - method_optimized_type - rb_block - rb_block_type - rb_builtin_function - rb_call_data - rb_callable_method_entry_struct - rb_callable_method_entry_t - rb_callcache - rb_callinfo - rb_captured_block - rb_cfunc_t - rb_control_frame_t - rb_cref_t - rb_execution_context_struct - rb_execution_context_t - rb_iseq_constant_body - rb_iseq_location_t - rb_iseq_struct - rb_iseq_t - rb_method_attr_t - rb_method_bmethod_t - rb_method_cfunc_t - rb_method_definition_struct - rb_method_entry_t - rb_method_iseq_t - rb_method_optimized_t - rb_method_type_t - rb_proc_t - rb_rjit_runtime_counters - rb_serial_t - rb_shape - rb_shape_t - rb_thread_struct - rb_jit_func_t - rb_iseq_param_keyword - rb_rjit_options - rb_callinfo_kwarg - ], - # #ifdef-dependent immediate types, which need Primitive.cexpr! for type detection - dynamic_types: %w[ - VALUE - shape_id_t - ], - skip_fields: { - 'rb_execution_context_struct.machine': %w[regs], # differs between macOS and Linux - rb_execution_context_struct: %w[method_missing_reason], # non-leading bit fields not supported - rb_iseq_constant_body: %w[jit_exception jit_exception_calls yjit_payload yjit_calls_at_interv], # conditionally defined - rb_thread_struct: %w[status has_dedicated_nt to_kill abort_on_exception report_on_exception pending_interrupt_queue_checked], - :'' => %w[is_from_method is_lambda is_isolated], # rb_proc_t - }, - ruby_fields: { - rb_iseq_constant_body: %w[ - rjit_blocks - ], - rb_iseq_location_struct: %w[ - base_label - label - pathobj - ], - rb_callable_method_entry_t: %w[ - defined_class - ], - rb_callable_method_entry_struct: %w[ - defined_class - ], - }, -) -generator.generate(nodes) - -# Write rjit_c.rb -File.write(src_path, generator.src) diff --git a/tool/ruby-version.rb b/tool/ruby-version.rb new file mode 100755 index 0000000000..3bbec576e1 --- /dev/null +++ b/tool/ruby-version.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby + +module RubyVersion + def self.tag(version) + major_version = Integer(version.split('.', 2)[0]) + if major_version >= 4 + "v#{version}" + else + "v#{version.tr('.-', '_')}" + end + end + + # Return the previous version to be used for release diff links. + # For a ".0" version, it returns the previous ".0" version. + # For a non-".0" version, it returns the previous teeny version. + def self.previous(version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + major = Integer($1) + minor = Integer($2) + teeny = Integer($3) + + if teeny != 0 + "#{major}.#{minor}.#{teeny-1}" + elsif minor != 0 # && teeny == 0 + "#{major}.#{minor-1}.#{teeny}" + else # minor == 0 && teeny == 0 + case major + when 3 + "2.7.0" + when 4 + "3.4.0" + else + raise "it doesn't know what is the previous version of '#{version}'" + end + end + end +end + +if __FILE__ == $0 + case ARGV[0] + when "tag" + print RubyVersion.tag(ARGV[1]) + when "previous" + print RubyVersion.previous(ARGV[1]) + when "previous-tag" + print RubyVersion.tag(RubyVersion.previous(ARGV[1])) + else + "#{$0}: unexpected command #{ARGV[0].inspect}" + end +end diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb index e03e54e397..f6c0e39600 100644 --- a/tool/ruby_vm/controllers/application_controller.rb +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb index 2a99e408da..628cb0428b 100644 --- a/tool/ruby_vm/helpers/c_escape.rb +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb index 8a04041da9..f0758dd44f 100644 --- a/tool/ruby_vm/helpers/dumper.rb +++ b/tool/ruby_vm/helpers/dumper.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/scanner.rb b/tool/ruby_vm/helpers/scanner.rb index ef6de8120e..8998abb2d3 100644 --- a/tool/ruby_vm/helpers/scanner.rb +++ b/tool/ruby_vm/helpers/scanner.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb index 034905f74e..d45d0ba83c 100644 --- a/tool/ruby_vm/loaders/insns_def.rb +++ b/tool/ruby_vm/loaders/insns_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb index aa6fd79e79..0750f1823a 100644 --- a/tool/ruby_vm/loaders/opt_insn_unif_def.rb +++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb index 29aef8a325..e08509a433 100644 --- a/tool/ruby_vm/loaders/opt_operand_def.rb +++ b/tool/ruby_vm/loaders/opt_operand_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb index 3f05c270ee..d626ea0296 100644 --- a/tool/ruby_vm/loaders/vm_opts_h.rb +++ b/tool/ruby_vm/loaders/vm_opts_h.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index ac4122f3ac..177b701b92 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instruction.rb index a810d89f3c..f87dd74179 100755..100644 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -15,7 +14,7 @@ require_relative 'c_expr' require_relative 'typemap' require_relative 'attribute' -class RubyVM::BareInstructions +class RubyVM::BareInstruction attr_reader :template, :name, :operands, :pops, :rets, :decls, :expr def initialize opts = {} @@ -108,14 +107,6 @@ class RubyVM::BareInstructions /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr end - def always_leaf? - @attrs.fetch('leaf').expr.expr == 'true;' - end - - def leaf_without_check_ints? - @attrs.fetch('leaf').expr.expr == 'leafness_of_check_ints;' - end - def handle_canary stmt # Stack canary is basically a good thing that we want to add, however: # @@ -149,6 +140,10 @@ class RubyVM::BareInstructions @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } end + def zjit_profile? + @attrs.fetch('zjit_profile').expr.expr != 'false;' + end + private def check_attribute_consistency @@ -187,6 +182,7 @@ class RubyVM::BareInstructions generate_attribute 'rb_snum_t', 'sp_inc', rets.size - pops.size generate_attribute 'bool', 'handles_sp', default_definition_of_handles_sp generate_attribute 'bool', 'leaf', default_definition_of_leaf + generate_attribute 'bool', 'zjit_profile', false end def default_definition_of_handles_sp @@ -228,13 +224,13 @@ class RubyVM::BareInstructions new h.merge(:template => h) } - def self.fetch name + def self.find(name) @instances.find do |insn| insn.name == name end or raise IndexError, "instruction not found: #{name}" end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb index 4b5aec58dd..095ff4f1d9 100644 --- a/tool/ruby_vm/models/c_expr.rb +++ b/tool/ruby_vm/models/c_expr.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/instructions.rb b/tool/ruby_vm/models/instructions.rb index 1198c7a4a6..7be7064b14 100644 --- a/tool/ruby_vm/models/instructions.rb +++ b/tool/ruby_vm/models/instructions.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -10,13 +9,15 @@ # conditions mentioned in the file COPYING are met. Consult the file for # details. -require_relative 'bare_instructions' -require_relative 'operands_unifications' -require_relative 'instructions_unifications' +require_relative 'bare_instruction' +require_relative 'operands_unification' +require_relative 'instructions_unification' +require_relative 'trace_instruction' +require_relative 'zjit_instruction' -RubyVM::Instructions = RubyVM::BareInstructions.to_a + \ - RubyVM::OperandsUnifications.to_a + \ - RubyVM::InstructionsUnifications.to_a - -require_relative 'trace_instructions' +RubyVM::Instructions = RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all + + RubyVM::TraceInstruction.all + + RubyVM::ZJITInstruction.all RubyVM::Instructions.freeze diff --git a/tool/ruby_vm/models/instructions_unifications.rb b/tool/ruby_vm/models/instructions_unification.rb index 214ba5fcc2..5c798e6d54 100644 --- a/tool/ruby_vm/models/instructions_unifications.rb +++ b/tool/ruby_vm/models/instructions_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,9 +11,9 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_insn_unif_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::InstructionsUnifications +class RubyVM::InstructionsUnification include RubyVM::CEscape attr_reader :name @@ -23,7 +22,7 @@ class RubyVM::InstructionsUnifications @location = opts[:location] @name = namegen opts[:signature] @series = opts[:signature].map do |i| - RubyVM::BareInstructions.fetch i # Misshit is fatal + RubyVM::BareInstruction.find(i) # Misshit is fatal end end @@ -37,7 +36,7 @@ class RubyVM::InstructionsUnifications new h end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/operands_unifications.rb b/tool/ruby_vm/models/operands_unification.rb index 10e5742897..ce118648ca 100644 --- a/tool/ruby_vm/models/operands_unifications.rb +++ b/tool/ruby_vm/models/operands_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,16 +11,16 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_operand_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::OperandsUnifications < RubyVM::BareInstructions +class RubyVM::OperandsUnification < RubyVM::BareInstruction include RubyVM::CEscape attr_reader :preamble, :original, :spec def initialize opts = {} name = opts[:signature][0] - @original = RubyVM::BareInstructions.fetch name + @original = RubyVM::BareInstruction.find(name) template = @original.template parts = compose opts[:location], opts[:signature], template[:signature] json = template.dup @@ -130,12 +129,12 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions new h end - def self.to_a + def self.all @instances end def self.each_group - to_a.group_by(&:original).each_pair do |k, v| + all.group_by(&:original).each_pair do |k, v| yield k, v end end diff --git a/tool/ruby_vm/models/trace_instructions.rb b/tool/ruby_vm/models/trace_instruction.rb index 4ed4c8cb42..6a3ad53c44 100644 --- a/tool/ruby_vm/models/trace_instructions.rb +++ b/tool/ruby_vm/models/trace_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -11,9 +10,9 @@ # details. require_relative '../helpers/c_escape' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::TraceInstructions +class RubyVM::TraceInstruction include RubyVM::CEscape attr_reader :name @@ -61,11 +60,11 @@ class RubyVM::TraceInstructions private - @instances = RubyVM::Instructions.map {|i| new i } + @instances = (RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all).map {|i| new(i) } - def self.to_a + def self.all @instances end - - RubyVM::Instructions.push(*to_a) end diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb index d762dd3321..68ef5a41a5 100644 --- a/tool/ruby_vm/models/typemap.rb +++ b/tool/ruby_vm/models/typemap.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/zjit_instruction.rb b/tool/ruby_vm/models/zjit_instruction.rb new file mode 100644 index 0000000000..04764e4c61 --- /dev/null +++ b/tool/ruby_vm/models/zjit_instruction.rb @@ -0,0 +1,56 @@ +require_relative '../helpers/c_escape' +require_relative 'bare_instruction' + +# Profile YARV instructions to optimize code generated by ZJIT +class RubyVM::ZJITInstruction + include RubyVM::CEscape + + attr_reader :name + + def initialize(orig) + @orig = orig + @name = as_tr_cpp "zjit @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute?(*) + return false + end + + @instances = RubyVM::BareInstruction.all.filter(&:zjit_profile?).map {|i| new(i) } + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb index 47d8da5513..ad8603b1a8 100644 --- a/tool/ruby_vm/scripts/insns2vm.rb +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/tests/.gitkeep b/tool/ruby_vm/tests/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 --- a/tool/ruby_vm/tests/.gitkeep +++ /dev/null diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb index cb895815ce..8bb28db1c1 100644 --- a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -6,6 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. %# +% +% stack_increase = proc do |i| +% if i.has_attribute?('sp_inc') +% '-127' +% else +% sprintf("%4d", i.rets.size - i.pops.size) +% end +% end +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); @@ -13,15 +23,14 @@ rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) { static const signed char t[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map { |i| - if i.has_attribute?('sp_inc') - '-127' - else - sprintf("%4d", i.rets.size - i.pops.size) - end - }.join(', ') -%>, +% insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, % end +#endif }; signed char c = t[insn]; diff --git a/tool/ruby_vm/views/_insn_leaf_info.erb b/tool/ruby_vm/views/_insn_leaf_info.erb new file mode 100644 index 0000000000..f30366ffda --- /dev/null +++ b/tool/ruby_vm/views/_insn_leaf_info.erb @@ -0,0 +1,18 @@ +MAYBE_UNUSED(static bool insn_leaf(int insn, const VALUE *opes)); +static bool +insn_leaf(int insn, const VALUE *opes) +{ + switch (insn) { +% RubyVM::Instructions.each do |insn| +% next if insn.is_a?(RubyVM::TraceInstruction) || insn.is_a?(RubyVM::ZJITInstruction) + case <%= insn.bin %>: + return attr_leaf_<%= insn.name %>(<%= + insn.operands.map.with_index do |ope, i| + "(#{ope[:type]})opes[#{i}]" + end.join(', ') + %>); +% end + default: + return false; + } +} diff --git a/tool/ruby_vm/views/_insn_len_info.erb b/tool/ruby_vm/views/_insn_len_info.erb index 569dca5845..b29a405918 100644 --- a/tool/ruby_vm/views/_insn_len_info.erb +++ b/tool/ruby_vm/views/_insn_len_info.erb @@ -5,6 +5,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% CONSTFUNC(MAYBE_UNUSED(static int insn_len(VALUE insn))); RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ @@ -13,9 +16,14 @@ RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO const uint8_t rb_vm_insn_len_info[] = { -% RubyVM::Instructions.each_slice 23 do |a| - <%= a.map(&:width).join(', ') -%>, +% insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, % end +#if USE_ZJIT +% zjit_insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_len_info); diff --git a/tool/ruby_vm/views/_insn_name_info.erb b/tool/ruby_vm/views/_insn_name_info.erb index e7ded75e65..2862908631 100644 --- a/tool/ruby_vm/views/_insn_name_info.erb +++ b/tool/ruby_vm/views/_insn_name_info.erb @@ -6,10 +6,14 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.name } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% next_offset = 0 +% name_offset = proc do |i| +% offset = sprintf("%4d", next_offset) +% next_offset += i.name.length + 1 # insn.name + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_name(VALUE insn))); @@ -20,18 +24,29 @@ extern const unsigned short rb_vm_insn_name_offset[VM_INSTRUCTION_SIZE]; RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO -const int rb_vm_max_insn_name_size = <%= a.map(&:size).max %>; +%# "trace_" is longer than "zjit_", so USE_ZJIT doesn't impact the max name size. +const int rb_vm_max_insn_name_size = <%= RubyVM::Instructions.map { |i| i.name.size }.max %>; const char rb_vm_insn_name_base[] = -% a.each do |i| - <%=cstr i%> "\0" +% insns.each do |i| + <%= cstr i.name %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each do |i| + <%= cstr i.name %> "\0" % end +#endif ; const unsigned short rb_vm_insn_name_offset[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%4d", i) }.join(', ') %>, +% insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, % end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_name_offset); diff --git a/tool/ruby_vm/views/_insn_operand_info.erb b/tool/ruby_vm/views/_insn_operand_info.erb index 996c33e960..410869fcd3 100644 --- a/tool/ruby_vm/views/_insn_operand_info.erb +++ b/tool/ruby_vm/views/_insn_operand_info.erb @@ -6,10 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.operands_info } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% operands_info = proc { |i| sprintf("%-6s", cstr(i.operands_info)) } +% +% next_offset = 0 +% op_offset = proc do |i| +% offset = sprintf("%3d", next_offset) +% next_offset += i.operands_info.length + 1 # insn.operands_info + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_op_types(VALUE insn))); CONSTFUNC(MAYBE_UNUSED(static int insn_op_type(VALUE insn, long pos))); @@ -21,15 +27,25 @@ RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO const char rb_vm_insn_op_base[] = -% a.each_slice 5 do |d| - <%= d.map {|i| sprintf("%-6s", cstr(i)) }.join(' "\0" ') %> "\0" +% insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" % end +#endif ; const unsigned short rb_vm_insn_op_offset[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%3d", i) }.join(', ') %>, +% insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, % end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_op_offset); diff --git a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb b/tool/ruby_vm/views/_insn_sp_pc_dependency.erb deleted file mode 100644 index 95528fbbf4..0000000000 --- a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb +++ /dev/null @@ -1,27 +0,0 @@ -%# -*- C -*- -%# Copyright (c) 2019 Takashi Kokubun. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -%# -PUREFUNC(MAYBE_UNUSED(static bool insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes))); - -static bool -insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes) -{ - switch (insn) { -% RubyVM::Instructions.each do |insn| -% # handles_sp?: If true, it requires to move sp in JIT -% # always_leaf?: If false, it may call an arbitrary method. pc should be moved -% # before the call, and the method may refer to caller's pc (lineno). -% unless !insn.is_a?(RubyVM::TraceInstructions) && !insn.handles_sp? && insn.always_leaf? - case <%= insn.bin %>: -% end -% end - return true; - default: - return false; - } -} diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb index f740107a0a..2756fa2dec 100644 --- a/tool/ruby_vm/views/_leaf_helpers.erb +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -10,10 +10,6 @@ #include "iseq.h" -// This is used to tell RJIT that this insn would be leaf if CHECK_INTS didn't exist. -// It should be used only when RUBY_VM_CHECK_INTS is directly written in insns.def. -static bool leafness_of_check_ints = false; - static bool leafness_of_defined(rb_num_t op_type) { @@ -25,7 +21,7 @@ leafness_of_defined(rb_num_t op_type) case DEFINED_YIELD: case DEFINED_REF: case DEFINED_ZSUPER: - return false; + return true; case DEFINED_CONST: case DEFINED_CONST_FROM: /* has rb_autoload_load(); */ diff --git a/tool/ruby_vm/views/_zjit_helpers.erb b/tool/ruby_vm/views/_zjit_helpers.erb new file mode 100644 index 0000000000..1185dbd9d8 --- /dev/null +++ b/tool/ruby_vm/views/_zjit_helpers.erb @@ -0,0 +1,31 @@ +#if USE_ZJIT + +MAYBE_UNUSED(static int vm_bare_insn_to_zjit_insn(int insn)); +static int +vm_bare_insn_to_zjit_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case BIN(<%= insn.jump_destination %>): + return <%= insn.bin %>; +% end + default: + return insn; + } +} + +MAYBE_UNUSED(static int vm_zjit_insn_to_bare_insn(int insn)); +static int +vm_zjit_insn_to_bare_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case <%= insn.bin %>: + return BIN(<%= insn.jump_destination %>); +% end + default: + return insn; + } +} + +#endif diff --git a/tool/ruby_vm/views/_zjit_instruction.erb b/tool/ruby_vm/views/_zjit_instruction.erb new file mode 100644 index 0000000000..7fd657697c --- /dev/null +++ b/tool/ruby_vm/views/_zjit_instruction.erb @@ -0,0 +1,12 @@ +#if USE_ZJIT + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + rb_zjit_profile_insn(BIN(<%= insn.jump_destination %>), ec); + DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} + +#endif diff --git a/tool/ruby_vm/views/insns.inc.erb b/tool/ruby_vm/views/insns.inc.erb index 29981a8a2d..6521a89b8a 100644 --- a/tool/ruby_vm/views/insns.inc.erb +++ b/tool/ruby_vm/views/insns.inc.erb @@ -6,21 +6,36 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'contains YARV instruction list', edit: __FILE__, } -%> +#ifndef INSNS_INC +#define INSNS_INC 1 + /* BIN : Basic Instruction Name */ #define BIN(n) YARVINSN_##n enum ruby_vminsn_type { -% RubyVM::Instructions.each do |i| +% insns.each do |i| + <%= i.bin %>, +% end +#if USE_ZJIT +% zjit_insns.each do |i| <%= i.bin %>, % end +#endif VM_INSTRUCTION_SIZE }; +#define VM_BARE_INSTRUCTION_SIZE <%= RubyVM::Instructions.count { |i| i.name !~ /\A(trace|zjit)_/ } %> + #define ASSERT_VM_INSTRUCTION_SIZE(array) \ STATIC_ASSERT(numberof_##array, numberof(array) == VM_INSTRUCTION_SIZE) + +#endif diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb index 2ca5aca7cf..48dd0e8832 100644 --- a/tool/ruby_vm/views/insns_info.inc.erb +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -11,12 +11,16 @@ this_file: 'contains instruction information for yarv instruction sequence.', edit: __FILE__, } %> +#ifndef INSNS_INFO_INC +#define INSNS_INFO_INC 1 <%= render 'insn_type_chars' %> <%= render 'insn_name_info' %> <%= render 'insn_len_info' %> <%= render 'insn_operand_info' %> <%= render 'leaf_helpers' %> <%= render 'sp_inc_helpers' %> +<%= render 'zjit_helpers' %> <%= render 'attributes' %> +<%= render 'insn_leaf_info' %> <%= render 'comptime_insn_stack_increase' %> -<%= render 'insn_sp_pc_dependency' %> +#endif diff --git a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb index 4f08524a77..793528af5d 100644 --- a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb +++ b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb @@ -7,7 +7,7 @@ module RubyVM::RJIT # :nodoc: all name: :<%= insn.name %>, bin: <%= i %>, # BIN(<%= insn.name %>) len: <%= insn.width %>, # insn_len - operands: <%= (insn.operands unless insn.name.start_with?('trace_')).inspect %>, + operands: <%= (insn.operands unless insn.name.start_with?(/trace_|zjit_/)).inspect %>, ), % end } diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb index de7bb210ea..9d9cf0a43a 100644 --- a/tool/ruby_vm/views/optinsn.inc.erb +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -23,7 +23,7 @@ insn_operands_unification(INSN *iobj) /* do nothing */; break; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| case <%= orig.bin %>: % unifs.each do |insn| @@ -56,7 +56,7 @@ rb_insn_unified_local_var_level(VALUE insn) switch (insn) { default: return -1; /* do nothing */; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| % unifs.each do|insn| case <%= insn.bin %>: % insn.spec.map{|(var,val)|val}.reject{|i| i == '*' }.each do |val| diff --git a/tool/ruby_vm/views/optunifs.inc.erb b/tool/ruby_vm/views/optunifs.inc.erb index e92a95beff..c096712936 100644 --- a/tool/ruby_vm/views/optunifs.inc.erb +++ b/tool/ruby_vm/views/optunifs.inc.erb @@ -7,7 +7,6 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % raise ':FIXME:TBW' if RubyVM::VmOptsH['INSTRUCTIONS_UNIFICATION'] -% n = RubyVM::Instructions.size <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -16,6 +15,4 @@ /* Let .bss section automatically initialize this variable */ /* cf. Section 6.7.8 of ISO/IEC 9899:1999 */ -static const int *const *const unified_insns_data[<%= n %>]; - -ASSERT_VM_INSTRUCTION_SIZE(unified_insns_data); +static const int *const *const unified_insns_data[VM_INSTRUCTION_SIZE]; diff --git a/tool/ruby_vm/views/vm.inc.erb b/tool/ruby_vm/views/vm.inc.erb index c1a3faf60a..38bf5f05ae 100644 --- a/tool/ruby_vm/views/vm.inc.erb +++ b/tool/ruby_vm/views/vm.inc.erb @@ -13,18 +13,22 @@ } -%> #include "vm_insnhelper.h" -% RubyVM::BareInstructions.to_a.each do |insn| +% RubyVM::BareInstruction.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::OperandsUnifications.to_a.each do |insn| +% RubyVM::OperandsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::InstructionsUnifications.to_a.each do |insn| +% RubyVM::InstructionsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::TraceInstructions.to_a.each do |insn| +% RubyVM::ZJITInstruction.all.each do |insn| +<%= render 'zjit_instruction', locals: { insn: insn } -%> +% end +% +% RubyVM::TraceInstruction.all.each do |insn| <%= render 'trace_instruction', locals: { insn: insn } -%> % end diff --git a/tool/ruby_vm/views/vmtc.inc.erb b/tool/ruby_vm/views/vmtc.inc.erb index 99cbd92614..39dc8bfa6b 100644 --- a/tool/ruby_vm/views/vmtc.inc.erb +++ b/tool/ruby_vm/views/vmtc.inc.erb @@ -6,6 +6,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' -%> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -13,9 +16,14 @@ } -%> static const void *const insns_address_table[] = { -% RubyVM::Instructions.each do |i| +% insns.each do |i| LABEL_PTR(<%= i.name %>), % end +#if USE_ZJIT +% zjit_insns.each do |i| + LABEL_PTR(<%= i.name %>), +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(insns_address_table); diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 9581922bfa..477cc75546 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -4,6 +4,8 @@ require 'fileutils' require "rbconfig" +require "find" +require "tempfile" module SyncDefaultGems include FileUtils @@ -11,90 +13,337 @@ module SyncDefaultGems module_function - REPOSITORIES = { - "io-console": 'ruby/io-console', - "io-nonblock": 'ruby/io-nonblock', - "io-wait": 'ruby/io-wait', - "net-http": "ruby/net-http", - "net-protocol": "ruby/net-protocol", - "open-uri": "ruby/open-uri", - "win32-registry": "ruby/win32-registry", - English: "ruby/English", - benchmark: "ruby/benchmark", - cgi: "ruby/cgi", - date: 'ruby/date', - delegate: "ruby/delegate", - did_you_mean: "ruby/did_you_mean", - digest: "ruby/digest", - erb: "ruby/erb", - error_highlight: "ruby/error_highlight", - etc: 'ruby/etc', - fcntl: 'ruby/fcntl', - fiddle: 'ruby/fiddle', - fileutils: 'ruby/fileutils', - find: "ruby/find", - forwardable: "ruby/forwardable", - ipaddr: 'ruby/ipaddr', - irb: 'ruby/irb', - json: 'ruby/json', - logger: 'ruby/logger', - mmtk: ['ruby/mmtk', "main"], - open3: "ruby/open3", - openssl: "ruby/openssl", - optparse: "ruby/optparse", - ostruct: 'ruby/ostruct', - pathname: "ruby/pathname", - pp: "ruby/pp", - prettyprint: "ruby/prettyprint", - prism: ["ruby/prism", "main"], - pstore: "ruby/pstore", - psych: 'ruby/psych', - rdoc: 'ruby/rdoc', - readline: "ruby/readline", - reline: 'ruby/reline', - resolv: "ruby/resolv", - rubygems: 'rubygems/rubygems', - securerandom: "ruby/securerandom", - set: "ruby/set", - shellwords: "ruby/shellwords", - singleton: "ruby/singleton", - stringio: 'ruby/stringio', - strscan: 'ruby/strscan', - syntax_suggest: ["ruby/syntax_suggest", "main"], - tempfile: "ruby/tempfile", - time: "ruby/time", - timeout: "ruby/timeout", - tmpdir: "ruby/tmpdir", - tsort: "ruby/tsort", - un: "ruby/un", - uri: "ruby/uri", - weakref: "ruby/weakref", - win32ole: "ruby/win32ole", - yaml: "ruby/yaml", - zlib: 'ruby/zlib', - }.transform_keys(&:to_s) + # upstream: "owner/repo" + # branch: "branch_name" + # mappings: [ ["path_in_upstream", "path_in_ruby"], ... ] + # NOTE: path_in_ruby is assumed to be "owned" by this gem, and the contents + # will be removed before sync + # exclude: [ "fnmatch_pattern_after_mapping", ... ] + Repository = Data.define(:upstream, :branch, :mappings, :exclude) do + def excluded?(newpath) + p = newpath + until p == "." + return true if exclude.any? {|pat| File.fnmatch?(pat, p, File::FNM_PATHNAME|File::FNM_EXTGLOB)} + p = File.dirname(p) + end + false + end + + def rewrite_for_ruby(path) + newpath = mappings.find do |src, dst| + if path == src || path.start_with?(src + "/") + break path.sub(src, dst) + end + end + return nil unless newpath + return nil if excluded?(newpath) + newpath + end + end CLASSICAL_DEFAULT_BRANCH = "master" - class << REPOSITORIES - def [](gem) - repo, branch = super(gem) - return repo, branch || CLASSICAL_DEFAULT_BRANCH - end + def repo((upstream, branch), mappings, exclude: []) + branch ||= CLASSICAL_DEFAULT_BRANCH + exclude += ["ext/**/depend"] + Repository.new(upstream:, branch:, mappings:, exclude:) + end + + def lib((upstream, branch), gemspec_in_subdir: false) + _org, name = upstream.split("/") + gemspec_dst = gemspec_in_subdir ? "lib/#{name}/#{name}.gemspec" : "lib/#{name}.gemspec" + repo([upstream, branch], [ + ["lib/#{name}.rb", "lib/#{name}.rb"], + ["lib/#{name}", "lib/#{name}"], + ["test/test_#{name}.rb", "test/test_#{name}.rb"], + ["test/#{name}", "test/#{name}"], + ["#{name}.gemspec", gemspec_dst], + ]) + end + + # Note: tool/auto_review_pr.rb also depends on these constants. + NO_UPSTREAM = [ + "lib/unicode_normalize", # not to match with "lib/un" + ] + REPOSITORIES = { + Onigmo: repo("k-takata/Onigmo", [ + ["regcomp.c", "regcomp.c"], + ["regenc.c", "regenc.c"], + ["regenc.h", "regenc.h"], + ["regerror.c", "regerror.c"], + ["regexec.c", "regexec.c"], + ["regint.h", "regint.h"], + ["regparse.c", "regparse.c"], + ["regparse.h", "regparse.h"], + ["regsyntax.c", "regsyntax.c"], + ["onigmo.h", "include/ruby/onigmo.h"], + ["enc", "enc"], + ]), + "io-console": repo("ruby/io-console", [ + ["ext/io/console", "ext/io/console"], + ["test/io/console", "test/io/console"], + ["lib/io/console", "ext/io/console/lib/console"], + ["io-console.gemspec", "ext/io/console/io-console.gemspec"], + ]), + "io-nonblock": repo("ruby/io-nonblock", [ + ["ext/io/nonblock", "ext/io/nonblock"], + ["test/io/nonblock", "test/io/nonblock"], + ["io-nonblock.gemspec", "ext/io/nonblock/io-nonblock.gemspec"], + ]), + "io-wait": repo("ruby/io-wait", [ + ["ext/io/wait", "ext/io/wait"], + ["test/io/wait", "test/io/wait"], + ["io-wait.gemspec", "ext/io/wait/io-wait.gemspec"], + ]), + "net-http": repo("ruby/net-http", [ + ["lib/net/http.rb", "lib/net/http.rb"], + ["lib/net/http", "lib/net/http"], + ["test/net/http", "test/net/http"], + ["net-http.gemspec", "lib/net/http/net-http.gemspec"], + ]), + "net-protocol": repo("ruby/net-protocol", [ + ["lib/net/protocol.rb", "lib/net/protocol.rb"], + ["test/net/protocol", "test/net/protocol"], + ["net-protocol.gemspec", "lib/net/net-protocol.gemspec"], + ]), + "open-uri": lib("ruby/open-uri"), + "win32-registry": repo("ruby/win32-registry", [ + ["lib/win32/registry.rb", "ext/win32/lib/win32/registry.rb"], + ["test/win32/test_registry.rb", "test/win32/test_registry.rb"], + ["win32-registry.gemspec", "ext/win32/win32-registry.gemspec"], + ]), + English: lib("ruby/English"), + cgi: repo("ruby/cgi", [ + ["ext/cgi", "ext/cgi"], + ["lib/cgi/escape.rb", "lib/cgi/escape.rb"], + ["test/cgi/test_cgi_escape.rb", "test/cgi/test_cgi_escape.rb"], + ["test/cgi/update_env.rb", "test/cgi/update_env.rb"], + ]), + date: repo("ruby/date", [ + ["doc/date", "doc/date"], + ["ext/date", "ext/date"], + ["lib", "ext/date/lib"], + ["test/date", "test/date"], + ["date.gemspec", "ext/date/date.gemspec"], + ], exclude: [ + "ext/date/lib/date_core.bundle", + ]), + delegate: lib("ruby/delegate"), + did_you_mean: repo("ruby/did_you_mean", [ + ["lib/did_you_mean.rb", "lib/did_you_mean.rb"], + ["lib/did_you_mean", "lib/did_you_mean"], + ["test", "test/did_you_mean"], + ["did_you_mean.gemspec", "lib/did_you_mean/did_you_mean.gemspec"], + ], exclude: [ + "test/did_you_mean/lib", + "test/did_you_mean/tree_spell/test_explore.rb", + ]), + digest: repo("ruby/digest", [ + ["ext/digest/lib/digest/sha2", "ext/digest/sha2/lib/sha2"], + ["ext/digest", "ext/digest"], + ["lib/digest.rb", "ext/digest/lib/digest.rb"], + ["lib/digest/version.rb", "ext/digest/lib/digest/version.rb"], + ["lib/digest/sha2.rb", "ext/digest/sha2/lib/sha2.rb"], + ["test/digest", "test/digest"], + ["digest.gemspec", "ext/digest/digest.gemspec"], + ]), + erb: repo("ruby/erb", [ + ["ext/erb", "ext/erb"], + ["lib/erb", "lib/erb"], + ["lib/erb.rb", "lib/erb.rb"], + ["test/erb", "test/erb"], + ["erb.gemspec", "lib/erb/erb.gemspec"], + ["libexec/erb", "libexec/erb"], + ]), + error_highlight: repo("ruby/error_highlight", [ + ["lib/error_highlight.rb", "lib/error_highlight.rb"], + ["lib/error_highlight", "lib/error_highlight"], + ["test", "test/error_highlight"], + ["error_highlight.gemspec", "lib/error_highlight/error_highlight.gemspec"], + ]), + etc: repo("ruby/etc", [ + ["ext/etc", "ext/etc"], + ["test/etc", "test/etc"], + ["etc.gemspec", "ext/etc/etc.gemspec"], + ]), + fcntl: repo("ruby/fcntl", [ + ["ext/fcntl", "ext/fcntl"], + ["fcntl.gemspec", "ext/fcntl/fcntl.gemspec"], + ]), + fileutils: lib("ruby/fileutils"), + find: lib("ruby/find"), + forwardable: lib("ruby/forwardable", gemspec_in_subdir: true), + ipaddr: lib("ruby/ipaddr"), + json: repo("ruby/json", [ + ["ext/json/ext", "ext/json"], + ["test/json", "test/json"], + ["lib", "ext/json/lib"], + ["json.gemspec", "ext/json/json.gemspec"], + ], exclude: [ + "ext/json/lib/json/ext/.keep", + "ext/json/lib/json/pure.rb", + "ext/json/lib/json/pure", + "ext/json/lib/json/truffle_ruby", + "test/json/lib", + "ext/json/extconf.rb", + ]), + mmtk: repo(["ruby/mmtk", "main"], [ + ["gc/mmtk", "gc/mmtk"], + ]), + open3: lib("ruby/open3", gemspec_in_subdir: true).tap { + it.exclude << "lib/open3/jruby_windows.rb" + }, + openssl: repo("ruby/openssl", [ + ["ext/openssl", "ext/openssl"], + ["lib", "ext/openssl/lib"], + ["test/openssl", "test/openssl"], + ["sample", "sample/openssl"], + ["openssl.gemspec", "ext/openssl/openssl.gemspec"], + ["History.md", "ext/openssl/History.md"], + ], exclude: [ + "test/openssl/envutil.rb", + "ext/openssl/depend", + ]), + optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap { + it.mappings << ["doc/optparse", "doc/optparse"] + }, + pathname: repo("ruby/pathname", [ + ["ext/pathname/pathname.c", "pathname.c"], + ["lib/pathname_builtin.rb", "pathname_builtin.rb"], + ["lib/pathname.rb", "lib/pathname.rb"], + ["test/pathname", "test/pathname"], + ]), + pp: lib("ruby/pp"), + prettyprint: lib("ruby/prettyprint"), + prism: repo(["ruby/prism", "main"], [ + ["ext/prism", "prism"], + ["lib/prism.rb", "lib/prism.rb"], + ["lib/prism", "lib/prism"], + ["test/prism", "test/prism"], + ["src", "prism"], + ["prism.gemspec", "lib/prism/prism.gemspec"], + ["include/prism", "prism"], + ["include/prism.h", "prism/prism.h"], + ["config.yml", "prism/config.yml"], + ["templates", "prism/templates"], + ], exclude: [ + "prism/templates/{javascript,java,rbi,sig}", + "test/prism/snapshots_test.rb", + "test/prism/snapshots", + "prism/extconf.rb", + "prism/srcs.mk*", + ]), + psych: repo("ruby/psych", [ + ["ext/psych", "ext/psych"], + ["lib", "ext/psych/lib"], + ["test/psych", "test/psych"], + ["psych.gemspec", "ext/psych/psych.gemspec"], + ], exclude: [ + "ext/psych/lib/org", + "ext/psych/lib/psych.jar", + "ext/psych/lib/psych_jars.rb", + "ext/psych/lib/psych.{bundle,so}", + "ext/psych/lib/2.*", + "ext/psych/yaml/LICENSE", + "ext/psych/.gitignore", + ]), + resolv: repo("ruby/resolv", [ + ["lib/resolv.rb", "lib/resolv.rb"], + ["test/resolv", "test/resolv"], + ["resolv.gemspec", "lib/resolv.gemspec"], + ["ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32/resolv.rb"], + ["ext/win32/resolv", "ext/win32/resolv"], + ]), + rubygems: repo("ruby/rubygems", [ + ["lib/rubygems.rb", "lib/rubygems.rb"], + ["lib/rubygems", "lib/rubygems"], + ["test/rubygems", "test/rubygems"], + ["bundler/lib/bundler.rb", "lib/bundler.rb"], + ["bundler/lib/bundler", "lib/bundler"], + ["bundler/exe/bundle", "libexec/bundle"], + ["bundler/exe/bundler", "libexec/bundler"], + ["bundler/bundler.gemspec", "lib/bundler/bundler.gemspec"], + ["bundler/spec", "spec/bundler"], + *["bundle", "parallel_rspec", "rspec"].map {|binstub| + ["bundler/bin/#{binstub}", "spec/bin/#{binstub}"] + }, + *%w[dev_gems test_gems rubocop_gems standard_gems].flat_map {|gemfile| + ["rb.lock", "rb"].map do |ext| + ["tool/bundler/#{gemfile}.#{ext}", "tool/bundler/#{gemfile}.#{ext}"] + end + }, + ], exclude: [ + "spec/bundler/bin", + "spec/bundler/support/artifice/vcr_cassettes", + "spec/bundler/support/artifice/used_cassettes.txt", + "lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}", + ]), + securerandom: lib("ruby/securerandom"), + shellwords: lib("ruby/shellwords"), + singleton: lib("ruby/singleton"), + stringio: repo("ruby/stringio", [ + ["ext/stringio", "ext/stringio"], + ["test/stringio", "test/stringio"], + ["stringio.gemspec", "ext/stringio/stringio.gemspec"], + ["doc/stringio", "doc/stringio"], + ], exclude: [ + "ext/stringio/README.md", + ]), + strscan: repo("ruby/strscan", [ + ["ext/strscan", "ext/strscan"], + ["lib", "ext/strscan/lib"], + ["test/strscan", "test/strscan"], + ["strscan.gemspec", "ext/strscan/strscan.gemspec"], + ["doc/strscan", "doc/strscan"], + ], exclude: [ + "ext/strscan/regenc.h", + "ext/strscan/regint.h", + ]), + syntax_suggest: lib(["ruby/syntax_suggest", "main"], gemspec_in_subdir: true), + tempfile: lib("ruby/tempfile"), + time: lib("ruby/time"), + timeout: lib("ruby/timeout"), + tmpdir: lib("ruby/tmpdir"), + un: lib("ruby/un"), + uri: lib("ruby/uri", gemspec_in_subdir: true), + weakref: lib("ruby/weakref"), + yaml: lib("ruby/yaml", gemspec_in_subdir: true), + zlib: repo("ruby/zlib", [ + ["ext/zlib", "ext/zlib"], + ["test/zlib", "test/zlib"], + ["zlib.gemspec", "ext/zlib/zlib.gemspec"], + ]), + }.transform_keys(&:to_s) - def each_pair - super do |gem, (repo, branch)| - yield gem, [repo, branch || CLASSICAL_DEFAULT_BRANCH] + class << Repository + def find_upstream(file) + return if NO_UPSTREAM.any? {|dst| file.start_with?(dst) } + REPOSITORIES.find do |repo_name, repository| + if repository.mappings.any? {|_src, dst| file.start_with?(dst) } + break repo_name + end end end + + def group(files) + files.group_by {|file| find_upstream(file)} + end end + # Allow synchronizing commits up to this FETCH_DEPTH. We've historically merged PRs + # with about 250 commits to ruby/ruby, so we use this depth for ruby/ruby in general. + FETCH_DEPTH = 500 + def pipe_readlines(args, rs: "\0", chomp: true) IO.popen(args) do |f| f.readlines(rs, chomp: chomp) end end + def porcelain_status(*pattern) + pipe_readlines(%W"git status --porcelain --no-renames -z --" + pattern) + end + def replace_rdoc_ref(file) src = File.binread(file) changed = false @@ -113,7 +362,7 @@ module SyncDefaultGems end def replace_rdoc_ref_all - result = pipe_readlines(%W"git status --porcelain -z -- *.c *.rb *.rdoc") + result = porcelain_status("*.c", "*.rb", "*.rdoc") result.map! {|line| line[/\A.M (.*)/, 1]} result.compact! return if result.empty? @@ -121,300 +370,69 @@ module SyncDefaultGems result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} end + def replace_rdoc_ref_all_full + Dir.glob("**/*.{c,rb,rdoc}").inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def rubygems_do_fixup + gemspec_content = File.readlines("lib/bundler/bundler.gemspec").map do |line| + next if line =~ /LICENSE\.md/ + + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") + end.compact.join + File.write("lib/bundler/bundler.gemspec", gemspec_content) + + ["bundle", "parallel_rspec", "rspec"].each do |binstub| + path = "spec/bin/#{binstub}" + next unless File.exist?(path) + content = File.read(path).gsub("../spec", "../bundler") + File.write(path, content) + chmod("+x", path) + end + end + # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. def sync_default_gems(gem) - repo, = REPOSITORIES[gem] - puts "Sync #{repo}" - - upstream = File.join("..", "..", repo) - - case gem - when "rubygems" - rm_rf(%w[lib/rubygems lib/rubygems.rb test/rubygems]) - cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") - cp_r("#{upstream}/test/rubygems", "test") - rm_rf(%w[lib/bundler lib/bundler.rb libexec/bundler libexec/bundle spec/bundler tool/bundler/*]) - cp_r(Dir.glob("#{upstream}/bundler/lib/bundler*"), "lib") - cp_r(Dir.glob("#{upstream}/bundler/exe/bundle*"), "libexec") - - gemspec_content = File.readlines("#{upstream}/bundler/bundler.gemspec").map do |line| - next if line =~ /LICENSE\.md/ - - line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") - end.compact.join - File.write("lib/bundler/bundler.gemspec", gemspec_content) - - cp_r("#{upstream}/bundler/spec", "spec/bundler") - %w[dev_gems test_gems rubocop_gems standard_gems].each do |gemfile| - cp_r("#{upstream}/tool/bundler/#{gemfile}.rb", "tool/bundler") - end - rm_rf Dir.glob("spec/bundler/support/artifice/{vcr_cassettes,used_cassettes.txt}") - rm_rf Dir.glob("lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}") - when "rdoc" - rm_rf(%w[lib/rdoc lib/rdoc.rb test/rdoc libexec/rdoc libexec/ri]) - cp_r(Dir.glob("#{upstream}/lib/rdoc*"), "lib") - cp_r("#{upstream}/doc/rdoc", "doc") - cp_r("#{upstream}/test/rdoc", "test") - cp_r("#{upstream}/rdoc.gemspec", "lib/rdoc") - cp_r("#{upstream}/Gemfile", "lib/rdoc") - cp_r("#{upstream}/Rakefile", "lib/rdoc") - cp_r("#{upstream}/exe/rdoc", "libexec") - cp_r("#{upstream}/exe/ri", "libexec") - parser_files = { - 'lib/rdoc/markdown.kpeg' => 'lib/rdoc/markdown.rb', - 'lib/rdoc/markdown/literals.kpeg' => 'lib/rdoc/markdown/literals.rb', - 'lib/rdoc/rd/block_parser.ry' => 'lib/rdoc/rd/block_parser.rb', - 'lib/rdoc/rd/inline_parser.ry' => 'lib/rdoc/rd/inline_parser.rb' - } - Dir.chdir(upstream) do - `bundle install` - parser_files.each_value do |dst| - `bundle exec rake #{dst}` + config = REPOSITORIES[gem] + puts "Sync #{config.upstream}" + + upstream = File.join("..", "..", config.upstream) + + unless File.exist?(upstream) + abort %[Expected '#{upstream}' (#{File.expand_path("#{upstream}")}) to be a directory, but it didn't exist.] + end + + config.mappings.each do |src, dst| + rm_rf(dst) + end + + copied = Set.new + config.mappings.each do |src, dst| + prefix = File.join(upstream, src) + # Maybe mapping needs to be updated? + next unless File.exist?(prefix) + Find.find(prefix) do |path| + next if File.directory?(path) + if copied.add?(path) + newpath = config.rewrite_for_ruby(path.sub(%r{\A#{Regexp.escape(upstream)}/}, "")) + next unless newpath + mkdir_p(File.dirname(newpath)) + cp(path, newpath) end end - parser_files.each_pair do |src, dst| - rm_rf(src) - cp_r("#{upstream}/#{dst}", dst) - end - `git checkout lib/rdoc/.document` - rm_rf(%w[lib/rdoc/Gemfile lib/rdoc/Rakefile]) - when "reline" - rm_rf(%w[lib/reline lib/reline.rb test/reline]) - cp_r(Dir.glob("#{upstream}/lib/reline*"), "lib") - cp_r("#{upstream}/test/reline", "test") - cp_r("#{upstream}/reline.gemspec", "lib/reline") - when "irb" - rm_rf(%w[lib/irb lib/irb.rb test/irb]) - cp_r(Dir.glob("#{upstream}/lib/irb*"), "lib") - rm_rf(%w[lib/irb/.document]) - cp_r("#{upstream}/test/irb", "test") - cp_r("#{upstream}/irb.gemspec", "lib/irb") - cp_r("#{upstream}/man/irb.1", "man/irb.1") - cp_r("#{upstream}/doc/irb", "doc") - when "json" - rm_rf(%w[ext/json lib/json test/json]) - cp_r("#{upstream}/ext/json/ext", "ext/json") - cp_r("#{upstream}/test/json", "test/json") - rm_rf("test/json/lib") - cp_r("#{upstream}/lib", "ext/json") - cp_r("#{upstream}/json.gemspec", "ext/json") - rm_rf(%w[ext/json/lib/json/pure.rb ext/json/lib/json/pure ext/json/lib/json/truffle_ruby/]) - json_files = Dir.glob("ext/json/lib/json/ext/**/*", File::FNM_DOTMATCH).select { |f| File.file?(f) } - rm_rf(json_files - Dir.glob("ext/json/lib/json/ext/**/*.rb") - Dir.glob("ext/json/lib/json/ext/**/depend")) - `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend ext/json/depend benchmark/` - when "psych" - rm_rf(%w[ext/psych test/psych]) - cp_r("#{upstream}/ext/psych", "ext") - cp_r("#{upstream}/lib", "ext/psych") - cp_r("#{upstream}/test/psych", "test") - rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) - rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) - rm_rf(["ext/psych/yaml/LICENSE"]) - cp_r("#{upstream}/psych.gemspec", "ext/psych") - `git checkout ext/psych/depend ext/psych/.gitignore` - when "fiddle" - rm_rf(%w[ext/fiddle test/fiddle]) - cp_r("#{upstream}/ext/fiddle", "ext") - cp_r("#{upstream}/lib", "ext/fiddle") - cp_r("#{upstream}/test/fiddle", "test") - cp_r("#{upstream}/fiddle.gemspec", "ext/fiddle") - `git checkout ext/fiddle/depend` - rm_rf(%w[ext/fiddle/lib/fiddle.{bundle,so}]) - when "stringio" - rm_rf(%w[ext/stringio test/stringio]) - cp_r("#{upstream}/ext/stringio", "ext") - cp_r("#{upstream}/test/stringio", "test") - cp_r("#{upstream}/stringio.gemspec", "ext/stringio") - `git checkout ext/stringio/depend ext/stringio/README.md` - when "io-console" - rm_rf(%w[ext/io/console test/io/console]) - cp_r("#{upstream}/ext/io/console", "ext/io") - cp_r("#{upstream}/test/io/console", "test/io") - mkdir_p("ext/io/console/lib") - cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") - rm_rf("ext/io/console/lib/console/ffi") - cp_r("#{upstream}/io-console.gemspec", "ext/io/console") - `git checkout ext/io/console/depend` - when "io-nonblock" - rm_rf(%w[ext/io/nonblock test/io/nonblock]) - cp_r("#{upstream}/ext/io/nonblock", "ext/io") - cp_r("#{upstream}/test/io/nonblock", "test/io") - cp_r("#{upstream}/io-nonblock.gemspec", "ext/io/nonblock") - `git checkout ext/io/nonblock/depend` - when "io-wait" - rm_rf(%w[ext/io/wait test/io/wait]) - cp_r("#{upstream}/ext/io/wait", "ext/io") - cp_r("#{upstream}/test/io/wait", "test/io") - cp_r("#{upstream}/io-wait.gemspec", "ext/io/wait") - `git checkout ext/io/wait/depend` - when "etc" - rm_rf(%w[ext/etc test/etc]) - cp_r("#{upstream}/ext/etc", "ext") - cp_r("#{upstream}/test/etc", "test") - cp_r("#{upstream}/etc.gemspec", "ext/etc") - `git checkout ext/etc/depend` - when "date" - rm_rf(%w[ext/date test/date]) - cp_r("#{upstream}/doc/date", "doc") - cp_r("#{upstream}/ext/date", "ext") - cp_r("#{upstream}/lib", "ext/date") - cp_r("#{upstream}/test/date", "test") - cp_r("#{upstream}/date.gemspec", "ext/date") - `git checkout ext/date/depend` - rm_rf(["ext/date/lib/date_core.bundle"]) - when "zlib" - rm_rf(%w[ext/zlib test/zlib]) - cp_r("#{upstream}/ext/zlib", "ext") - cp_r("#{upstream}/test/zlib", "test") - cp_r("#{upstream}/zlib.gemspec", "ext/zlib") - `git checkout ext/zlib/depend` - when "fcntl" - rm_rf(%w[ext/fcntl]) - cp_r("#{upstream}/ext/fcntl", "ext") - cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") - `git checkout ext/fcntl/depend` - when "strscan" - rm_rf(%w[ext/strscan test/strscan]) - cp_r("#{upstream}/ext/strscan", "ext") - cp_r("#{upstream}/lib", "ext/strscan") - cp_r("#{upstream}/test/strscan", "test") - cp_r("#{upstream}/strscan.gemspec", "ext/strscan") - begin - cp_r("#{upstream}/doc/strscan", "doc") - rescue Errno::ENOENT + end + + porcelain_status().each do |line| + /\A(?:.)(?:.) (?<path>.*)\z/ =~ line or raise + if config.excluded?(path) + puts "Restoring excluded file: #{path}" + IO.popen(%W"git checkout --" + [path], "rb", &:read) end - rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) - `git checkout ext/strscan/depend` - when "cgi" - rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) - cp_r("#{upstream}/ext/cgi", "ext") - cp_r("#{upstream}/lib/cgi", "lib") - cp_r("#{upstream}/lib/cgi.rb", "lib") - rm_rf("lib/cgi/escape.jar") - cp_r("#{upstream}/test/cgi", "test") - cp_r("#{upstream}/cgi.gemspec", "lib/cgi") - `git checkout ext/cgi/escape/depend` - when "openssl" - rm_rf(%w[ext/openssl test/openssl]) - cp_r("#{upstream}/ext/openssl", "ext") - cp_r("#{upstream}/lib", "ext/openssl") - cp_r("#{upstream}/test/openssl", "test") - rm_rf("test/openssl/envutil.rb") - cp_r("#{upstream}/openssl.gemspec", "ext/openssl") - cp_r("#{upstream}/History.md", "ext/openssl") - `git checkout ext/openssl/depend` - when "net-protocol" - rm_rf(%w[lib/net/protocol.rb lib/net/net-protocol.gemspec test/net/protocol]) - cp_r("#{upstream}/lib/net/protocol.rb", "lib/net") - cp_r("#{upstream}/test/net/protocol", "test/net") - cp_r("#{upstream}/net-protocol.gemspec", "lib/net") - when "net-http" - rm_rf(%w[lib/net/http.rb lib/net/http test/net/http]) - cp_r("#{upstream}/lib/net/http.rb", "lib/net") - cp_r("#{upstream}/lib/net/http", "lib/net") - cp_r("#{upstream}/test/net/http", "test/net") - cp_r("#{upstream}/net-http.gemspec", "lib/net/http") - when "did_you_mean" - rm_rf(%w[lib/did_you_mean lib/did_you_mean.rb test/did_you_mean]) - cp_r(Dir.glob("#{upstream}/lib/did_you_mean*"), "lib") - cp_r("#{upstream}/did_you_mean.gemspec", "lib/did_you_mean") - cp_r("#{upstream}/test", "test/did_you_mean") - rm_rf("test/did_you_mean/lib") - rm_rf(%w[test/did_you_mean/tree_spell/test_explore.rb]) - when "erb" - rm_rf(%w[lib/erb* test/erb libexec/erb]) - cp_r("#{upstream}/lib/erb.rb", "lib") - cp_r("#{upstream}/test/erb", "test") - cp_r("#{upstream}/erb.gemspec", "lib") - cp_r("#{upstream}/libexec/erb", "libexec") - when "pathname" - rm_rf(%w[ext/pathname test/pathname]) - cp_r("#{upstream}/ext/pathname", "ext") - cp_r("#{upstream}/test/pathname", "test") - cp_r("#{upstream}/lib", "ext/pathname") - cp_r("#{upstream}/pathname.gemspec", "ext/pathname") - `git checkout ext/pathname/depend` - when "digest" - rm_rf(%w[ext/digest test/digest]) - cp_r("#{upstream}/ext/digest", "ext") - mkdir_p("ext/digest/lib/digest") - cp_r("#{upstream}/lib/digest.rb", "ext/digest/lib/") - cp_r("#{upstream}/lib/digest/version.rb", "ext/digest/lib/digest/") - mkdir_p("ext/digest/sha2/lib") - cp_r("#{upstream}/lib/digest/sha2.rb", "ext/digest/sha2/lib") - move("ext/digest/lib/digest/sha2", "ext/digest/sha2/lib") - cp_r("#{upstream}/test/digest", "test") - cp_r("#{upstream}/digest.gemspec", "ext/digest") - `git checkout ext/digest/depend ext/digest/*/depend` - when "set" - sync_lib gem, upstream - cp_r(Dir.glob("#{upstream}/test/*"), "test/set") - when "optparse" - sync_lib gem, upstream - rm_rf(%w[doc/optparse]) - mkdir_p("doc/optparse") - cp_r("#{upstream}/doc/optparse", "doc") - when "error_highlight" - rm_rf(%w[lib/error_highlight lib/error_highlight.rb test/error_highlight]) - cp_r(Dir.glob("#{upstream}/lib/error_highlight*"), "lib") - cp_r("#{upstream}/error_highlight.gemspec", "lib/error_highlight") - cp_r("#{upstream}/test", "test/error_highlight") - when "win32ole" - sync_lib gem, upstream - rm_rf(%w[ext/win32ole/lib]) - Dir.mkdir(*%w[ext/win32ole/lib]) - move("lib/win32ole/win32ole.gemspec", "ext/win32ole") - move(Dir.glob("lib/win32ole*"), "ext/win32ole/lib") - when "open3" - sync_lib gem, upstream - rm_rf("lib/open3/jruby_windows.rb") - when "syntax_suggest" - sync_lib gem, upstream - rm_rf(%w[spec/syntax_suggest libexec/syntax_suggest]) - cp_r("#{upstream}/spec", "spec/syntax_suggest") - cp_r("#{upstream}/exe/syntax_suggest", "libexec/syntax_suggest") - when "prism" - rm_rf(%w[test/prism prism]) - - cp_r("#{upstream}/ext/prism", "prism") - cp_r("#{upstream}/lib/.", "lib") - cp_r("#{upstream}/test/prism", "test") - cp_r("#{upstream}/src/.", "prism") - - cp_r("#{upstream}/prism.gemspec", "lib/prism") - cp_r("#{upstream}/include/prism/.", "prism") - cp_r("#{upstream}/include/prism.h", "prism") - - cp_r("#{upstream}/config.yml", "prism/") - cp_r("#{upstream}/templates", "prism/") - rm_rf("prism/templates/javascript") - rm_rf("prism/templates/java") - rm_rf("prism/templates/rbi") - rm_rf("prism/templates/sig") - - rm("test/prism/snapshots_test.rb") - rm_rf("test/prism/snapshots") - - rm("prism/extconf.rb") - when "resolv" - rm_rf(%w[lib/resolv.* ext/win32/resolv test/resolv ext/win32/lib/win32/resolv.rb]) - cp_r("#{upstream}/lib/resolv.rb", "lib") - cp_r("#{upstream}/resolv.gemspec", "lib") - cp_r("#{upstream}/ext/win32/resolv", "ext/win32") - move("ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32") - rm_rf("ext/win32/resolv/lib") # Clean up empty directory - cp_r("#{upstream}/test/resolv", "test") - `git checkout ext/win32/resolv/depend` - when "win32-registry" - rm_rf(%w[ext/win32/lib/win32/registry.rb test/win32/test_registry.rb]) - cp_r("#{upstream}/lib/win32/registry.rb", "ext/win32/lib/win32") - cp_r("#{upstream}/test/win32/test_registry.rb", "test/win32") - cp_r("#{upstream}/win32-registry.gemspec", "ext/win32") - when "mmtk" - rm_rf("gc/mmtk") - cp_r("#{upstream}/gc/mmtk", "gc") - else - sync_lib gem, upstream + end + + # RubyGems/Bundler needs special care + if gem == "rubygems" + rubygems_do_fixup end check_prerelease_version(gem) @@ -425,16 +443,13 @@ module SyncDefaultGems end def check_prerelease_version(gem) - return if gem == "rubygems" - return if gem == "mmtk" - - gem = gem.downcase + return if ["rubygems", "mmtk", "cgi", "pathname", "Onigmo"].include?(gem) require "net/https" require "json" require "uri" - uri = URI("https://rubygems.org/api/v1/versions/#{gem}/latest.json") + uri = URI("https://rubygems.org/api/v1/versions/#{gem.downcase}/latest.json") response = Net::HTTP.get(uri) latest_version = JSON.parse(response)["version"] @@ -451,35 +466,18 @@ module SyncDefaultGems puts "#{gem}-#{spec.version} is not latest version of rubygems.org" if spec.version.to_s != latest_version end - def ignore_file_pattern_for(gem) - patterns = [] - - # Common patterns - patterns << %r[\A(?: - [^/]+ # top-level entries - |\.git.* - |bin/.* - |ext/.*\.java - |rakelib/.* - |test/(?:lib|fixtures)/.* - |tool/(?!bundler/).* - )\z]mx - - # Gem-specific patterns - case gem - when nil - end&.tap do |pattern| - patterns << pattern + def message_filter(repo, sha, log, context: nil) + unless repo.count("/") == 1 and /\A\S+\z/ =~ repo + raise ArgumentError, "invalid repository: #{repo}" end + unless /\A\h{10,40}\z/ =~ sha + raise ArgumentError, "invalid commit-hash: #{sha}" + end + repo_url = "https://github.com/#{repo}" - Regexp.union(*patterns) - end - - def message_filter(repo, sha, input: ARGF) - log = input.read - log.delete!("\r") + # Log messages generated by GitHub web UI have inconsistent line endings + log = log.delete("\r") log << "\n" if !log.end_with?("\n") - repo_url = "https://github.com/#{repo}" # Split the subject from the log message according to git conventions. # SPECIAL TREAT: when the first line ends with a dot `.` (which is not @@ -502,41 +500,69 @@ module SyncDefaultGems end end commit_url = "#{repo_url}/commit/#{sha[0,10]}\n" + sync_note = context ? "#{commit_url}\n#{context}" : commit_url if log and !log.empty? log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last conv[log] log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) { - ($~.begin(1) ? "" : "\n\n") + commit_url + ($~.begin(2) ? "\n" : "") + ($~.begin(1) ? "" : "\n\n") + sync_note + ($~.begin(2) ? "\n" : "") } else - log = commit_url + log = sync_note end - puts subject, "\n", log + "#{subject}\n\n#{log}" end - # Returns commit list as array of [commit_hash, subject]. - def commits_in_ranges(gem, repo, default_branch, ranges) - # If -a is given, discover all commits since the last picked commit - if ranges == true - # \r? needed in the regex in case the commit has windows-style line endings (because e.g. we're running - # tests on Windows) - pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)\r?$" - log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", "rb", &:read) - ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] - end + def log_format(format, args, &block) + IO.popen(%W[git -c core.autocrlf=false -c core.eol=lf + log --no-show-signature --format=#{format}] + args, "rb", &block) + end - # Parse a given range with git log - ranges.flat_map do |range| - unless range.include?("..") - range = "#{range}~1..#{range}" - end + def commits_in_range(upto, exclude, toplevel:) + args = [upto, *exclude.map {|s|"^#{s}"}] + log_format('%H,%P,%s', %W"--first-parent" + args) do |f| + f.read.split("\n").reverse.flat_map {|commit| + hash, parents, subject = commit.split(',', 3) + parents = parents.split + + # Non-merge commit + if parents.size <= 1 + puts "#{hash} #{subject}" + next [[hash, subject]] + end - IO.popen(%W"git log --format=%H,%s #{range} --", "rb") do |f| - f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} - end + # Clean 2-parent merge commit: follow the other parent as long as it + # contains no potentially-non-clean merges + if parents.size == 2 && + IO.popen(%W"git diff-tree --remerge-diff #{hash}", "rb", &:read).empty? + puts "\e[2mChecking the other parent of #{hash} #{subject}\e[0m" + ret = catch(:quit) { + commits_in_range(parents[1], exclude + [parents[0]], toplevel: false) + } + next ret if ret + end + + unless toplevel + puts "\e[1mMerge commit with possible conflict resolution #{hash} #{subject}\e[0m" + throw :quit + end + + puts "#{hash} #{subject} " \ + "\e[1m[merge commit with possible conflicts, will do a squash merge]\e[0m" + [[hash, subject]] + } end end + # Returns commit list as array of [commit_hash, subject, sync_note]. + def commits_in_ranges(ranges) + ranges.flat_map do |range| + exclude, upto = range.include?("..") ? range.split("..", 2) : ["#{range}~1", range] + puts "Looking for commits in range #{exclude}..#{upto}" + commits_in_range(upto, exclude.empty? ? [] : [exclude], toplevel: true) + end.uniq + end + #-- # Following methods used by sync_default_gems_with_commits return # true: success @@ -545,27 +571,9 @@ module SyncDefaultGems #++ def resolve_conflicts(gem, sha, edit) - # Skip this commit if everything has been removed as `ignored_paths`. - changes = pipe_readlines(%W"git status --porcelain -z") - if changes.empty? - puts "Skip empty commit #{sha}" - return false - end - - # We want to skip DD: deleted by both. - deleted = changes.grep(/^DD /) {$'} - system(*%W"git rm -f --", *deleted) unless deleted.empty? - - # Import UA: added by them - added = changes.grep(/^UA /) {$'} - system(*%W"git add --", *added) unless added.empty? - - # Discover unmerged files - # AU: unmerged, added by us - # DU: unmerged, deleted by us - # UU: unmerged, both modified - # AA: unmerged, both added - conflict = changes.grep(/\A(?:.U|AA) /) {$'} + # Discover unmerged files: any unstaged changes + changes = porcelain_status() + conflict = changes.grep(/\A(?:.[^ ?]) /) {$'} # If -e option is given, open each conflicted file with an editor unless conflict.empty? if edit @@ -586,123 +594,170 @@ module SyncDefaultGems return true end - def preexisting?(base, file) - system(*%w"git cat-file -e", "#{base}:#{file}", err: File::NULL) + def collect_cacheinfo(tree) + pipe_readlines(%W"git ls-tree -r -t -z #{tree}").filter_map do |line| + fields, path = line.split("\t", 2) + mode, type, object = fields.split(" ", 3) + next unless type == "blob" + [mode, type, object, path] + end end - def filter_pickup_files(changed, ignore_file_pattern, base) - toplevels = {} - remove = [] - ignore = [] - changed = changed.reject do |f| - case - when toplevels.fetch(top = f[%r[\A[^/]+(?=/|\z)]m]) { - remove << top if toplevels[top] = !preexisting?(base, top) - } - # Remove any new top-level directories. - true - when ignore_file_pattern.match?(f) - # Forcibly reset any changes matching ignore_file_pattern. - (preexisting?(base, f) ? ignore : remove) << f - end + def rewrite_cacheinfo(gem, blobs) + config = REPOSITORIES[gem] + rewritten = [] + ignored = blobs.dup + ignored.delete_if do |mode, type, object, path| + newpath = config.rewrite_for_ruby(path) + next unless newpath + rewritten << [mode, type, object, newpath] end - return changed, remove, ignore + [rewritten, ignored] end - def pickup_files(gem, changed, picked) - # Forcibly remove any files that we don't want to copy to this - # repository. - - ignore_file_pattern = ignore_file_pattern_for(gem) + def make_commit_info(gem, sha) + config = REPOSITORIES[gem] + headers, orig = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + /^author (?<author_name>.+?) <(?<author_email>.*?)> (?<author_date>.+?)$/ =~ headers or + raise "unable to parse author info for commit #{sha}" + author = { + "GIT_AUTHOR_NAME" => author_name, + "GIT_AUTHOR_EMAIL" => author_email, + "GIT_AUTHOR_DATE" => author_date, + } + context = nil + if /^parent (?<first_parent>.{40})\nparent .{40}$/ =~ headers + # Squashing a merge commit: keep authorship information + context = IO.popen(%W"git shortlog #{first_parent}..#{sha} --", "rb", &:read) + end + message = message_filter(config.upstream, sha, orig, context: context) + [author, message] + end - base = picked ? "HEAD~" : "HEAD" - changed, remove, ignore = filter_pickup_files(changed, ignore_file_pattern, base) + def fixup_commit(gem, commit) + wt = File.join("tmp", "sync_default_gems-fixup-worktree") + if File.directory?(wt) + IO.popen(%W"git -C #{wt} clean -xdf", "rb", &:read) + IO.popen(%W"git -C #{wt} reset --hard #{commit}", "rb", &:read) + else + IO.popen(%W"git worktree remove --force #{wt}", "rb", err: File::NULL, &:read) + IO.popen(%W"git worktree add --detach #{wt} #{commit}", "rb", &:read) + end + raise "git worktree prepare failed for commit #{commit}" unless $?.success? - unless remove.empty? - puts "Remove added files: #{remove.join(', ')}" - system(*%w"git rm -fr --", *remove) - if picked - system(*%w"git commit --amend --no-edit --", *remove, %i[out err] => File::NULL) + Dir.chdir(wt) do + if gem == "rubygems" + rubygems_do_fixup end + replace_rdoc_ref_all_full end - unless ignore.empty? - puts "Reset ignored files: #{ignore.join(', ')}" - system(*%W"git rm -r --", *ignore) - ignore.each {|f| system(*%W"git checkout -f", base, "--", f)} - end + IO.popen(%W"git -C #{wt} add -u", "rb", &:read) + IO.popen(%W"git -C #{wt} commit --amend --no-edit", "rb", &:read) + IO.popen(%W"git -C #{wt} rev-parse HEAD", "rb", &:read).chomp + end + + def make_and_fixup_commit(gem, original_commit, cacheinfo, parent: nil, message: nil, author: nil) + tree = Tempfile.create("sync_default_gems-#{gem}-index") do |f| + File.unlink(f.path) + IO.popen({"GIT_INDEX_FILE" => f.path}, + %W"git update-index --index-info", "wb", out: IO::NULL) do |io| + cacheinfo.each do |mode, type, object, path| + io.puts("#{mode} #{type} #{object}\t#{path}") + end + end + raise "git update-index failed" unless $?.success? - if changed.empty? - return nil + IO.popen({"GIT_INDEX_FILE" => f.path}, %W"git write-tree --missing-ok", "rb", &:read).chomp end - return changed + args = ["-m", message || "Rewriten commit for #{original_commit}"] + args += ["-p", parent] if parent + commit = IO.popen({**author}, %W"git commit-tree #{tree}" + args, "rb", &:read).chomp + + # Apply changes that require a working tree + commit = fixup_commit(gem, commit) + + commit end - def pickup_commit(gem, sha, edit) - # Attempt to cherry-pick a commit - result = IO.popen(%W"git cherry-pick #{sha}", "rb", &:read) - picked = $?.success? - if result =~ /nothing\ to\ commit/ - `git reset` - puts "Skip empty commit #{sha}" + def rewrite_commit(gem, sha) + author, message = make_commit_info(gem, sha) + new_blobs = collect_cacheinfo("#{sha}") + new_rewritten, new_ignored = rewrite_cacheinfo(gem, new_blobs) + + headers, _ = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + first_parent = headers[/^parent (.{40})$/, 1] + unless first_parent + # Root commit, first time to sync this repo + return make_and_fixup_commit(gem, sha, new_rewritten, message: message, author: author) + end + + old_blobs = collect_cacheinfo(first_parent) + old_rewritten, old_ignored = rewrite_cacheinfo(gem, old_blobs) + if old_ignored != new_ignored + paths = (old_ignored + new_ignored - (old_ignored & new_ignored)) + .map {|*_, path| path}.uniq + puts "\e\[1mIgnoring file changes not in mappings: #{paths.join(" ")}\e\[0m" + end + changed_paths = (old_rewritten + new_rewritten - (old_rewritten & new_rewritten)) + .map {|*_, path| path}.uniq + if changed_paths.empty? + puts "Skip commit only for tools or toplevel" return false end - # Skip empty commits - if result.empty? - return false - end + # Build commit objects from "cacheinfo" + new_parent = make_and_fixup_commit(gem, first_parent, old_rewritten) + new_commit = make_and_fixup_commit(gem, sha, new_rewritten, parent: new_parent, message: message, author: author) + puts "Created a temporary commit for cherry-pick: #{new_commit}" + new_commit + end - if picked - changed = pipe_readlines(%w"git diff-tree --name-only -r -z HEAD~..HEAD --") - else - changed = pipe_readlines(%w"git diff --name-only -r -z HEAD --") - end + def pickup_commit(gem, sha, edit) + rewritten = rewrite_commit(gem, sha) - # Pick up files to merge. - unless changed = pickup_files(gem, changed, picked) - puts "Skip commit #{sha} only for tools or toplevel" - if picked - `git reset --hard HEAD~` - else - `git cherry-pick --abort` - end - return false - end + # No changes remaining after rewriting + return false unless rewritten - # If the cherry-pick attempt failed, try to resolve conflicts. - # Skip the commit, if it contains unresolved conflicts or no files to pick up. - unless picked or resolve_conflicts(gem, sha, edit) - `git reset` && `git checkout .` && `git clean -fd` - return picked || nil # Fail unless cherry-picked - end + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{rewritten}", "rb", err: [:child, :out], &:read) + unless $?.success? + if result =~ /The previous cherry-pick is now empty/ + system(*%w"git cherry-pick --skip") + puts "Skip empty commit #{sha}" + return false + end - # Commit cherry-picked commit - if picked - system(*%w"git commit --amend --no-edit") - else - system(*%w"git cherry-pick --continue --no-edit") - end or return nil + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless resolve_conflicts(gem, sha, edit) + system(*%w"git --no-pager diff") if !edit # If failed, show `git diff` unless editing + `git reset` && `git checkout .` && `git clean -fd` # Clean up un-committed diffs + return nil # Fail unless cherry-picked + end - # Amend the commit if RDoc references need to be replaced - head = `git log --format=%H -1 HEAD`.chomp - system(*%w"git reset --quiet HEAD~ --") - amend = replace_rdoc_ref_all - system(*%W"git reset --quiet #{head} --") - if amend - `git commit --amend --no-edit --all` + # Commit cherry-picked commit + if porcelain_status().empty? + system(*%w"git cherry-pick --skip") + return false + else + system(*%w"git cherry-pick --continue --no-edit") + return nil unless $?.success? + end end + new_head = IO.popen(%W"git rev-parse HEAD", "rb", &:read).chomp + puts "Committed cherry-pick as #{new_head}" return true end - # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. - # @param ranges [Array<String>] "before..after". Note that it will NOT sync "before" (but commits after that). + # @param ranges [Array<String>, true] "commit", "before..after", or true. Note that it will NOT sync "before" (but commits after that). # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo, default_branch = REPOSITORIES[gem] + config = REPOSITORIES[gem] + repo, default_branch = config.upstream, config.branch puts "Sync #{repo} with commit history." # Fetch the repository to be synchronized @@ -711,86 +766,46 @@ module SyncDefaultGems `git remote add #{gem} https://github.com/#{repo}.git` end end - system(*%W"git fetch --no-tags #{gem}") - - commits = commits_in_ranges(gem, repo, default_branch, ranges) + system(*%W"git fetch --no-tags --depth=#{FETCH_DEPTH} #{gem} #{default_branch}") - # Ignore Merge commits and already-merged commits. - commits.delete_if do |sha, subject| - subject.start_with?("Merge", "Auto Merge") + # If -a is given, discover all commits since the last picked commit + if ranges == true + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" + log = log_format('%B', %W"-E --grep=#{pattern} -n1 --", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] end - + commits = commits_in_ranges(ranges) if commits.empty? puts "No commits to pick" return true end - puts "Try to pick these commits:" - puts commits.map{|commit| commit.join(": ")} - puts "----" - failed_commits = [] - - require 'shellwords' - filter = [ - ENV.fetch('RUBY', 'ruby').shellescape, - File.realpath(__FILE__).shellescape, - "--message-filter", - ] commits.each do |sha, subject| - puts "Pick #{sha} from #{repo}." + puts "----" + puts "Pick #{sha} #{subject}" case pickup_commit(gem, sha, edit) when false - next + # skipped when nil - failed_commits << sha - next - end - - puts "Update commit message: #{sha}" - - # Run this script itself (tool/sync_default_gems.rb --message-filter) as a message filter - IO.popen({"FILTER_BRANCH_SQUELCH_WARNING" => "1"}, - %W[git filter-branch -f --msg-filter #{[filter, repo, sha].join(' ')} -- HEAD~1..HEAD], - &:read) - unless $?.success? - puts "Failed to modify commit message of #{sha}" - break + failed_commits << [sha, subject] end end unless failed_commits.empty? puts "---- failed commits ----" - puts failed_commits + failed_commits.each do |sha, subject| + puts "#{sha} #{subject}" + end return false end return true end - def sync_lib(repo, upstream = nil) - unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") - abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] - end - rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) - cp_r(Dir.glob("#{upstream}/lib/*"), "lib") - tests = if File.directory?("test/#{repo}") - "test/#{repo}" - else - "test/test_#{repo}.rb" - end - cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") - gemspec = if File.directory?("lib/#{repo}") - "lib/#{repo}/#{repo}.gemspec" - else - "lib/#{repo}.gemspec" - end - cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") - end - def update_default_gems(gem, release: false) - - repository, default_branch = REPOSITORIES[gem] - author, repository = repository.split('/') + config = REPOSITORIES[gem] + author, repository = config.upstream.split('/') + default_branch = config.branch puts "Update #{author}/#{repository}" @@ -830,28 +845,18 @@ module SyncDefaultGems REPOSITORIES.each_key {|gem| update_default_gems(gem)} end when "all" - if ARGV[1] == "release" - REPOSITORIES.each_key do |gem| - update_default_gems(gem, release: true) - sync_default_gems(gem) - end - else - REPOSITORIES.each_key {|gem| sync_default_gems(gem)} + REPOSITORIES.each_key do |gem| + next if ["Onigmo"].include?(gem) + update_default_gems(gem, release: true) if ARGV[1] == "release" + sync_default_gems(gem) end when "list" ARGV.shift pattern = Regexp.new(ARGV.join('|')) - REPOSITORIES.each_pair do |name, (gem)| - next unless pattern =~ name or pattern =~ gem - printf "%-15s https://github.com/%s\n", name, gem - end - when "--message-filter" - ARGV.shift - if ARGV.size < 2 - abort "usage: #{$0} --message-filter repository commit-hash [input...]" + REPOSITORIES.each do |gem, config| + next unless pattern =~ gem or pattern =~ config.upstream + printf "%-15s https://github.com/%s\n", gem, config.upstream end - message_filter(*ARGV.shift(2)) - exit when "rdoc-ref" ARGV.shift pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV @@ -867,7 +872,13 @@ module SyncDefaultGems puts <<-HELP \e[1mSync with upstream code of default libraries\e[0m -\e[1mImport a default library through `git clone` and `cp -rf` (git commits are lost)\e[0m +\e[1mImport all default gems through `git clone` and `cp -rf` (git commits are lost)\e[0m + ruby #$0 all + +\e[1mImport all released version of default gems\e[0m + ruby #$0 all release + +\e[1mImport a default gem with specific gem same as all command\e[0m ruby #$0 rubygems \e[1mPick a single commit from the upstream repository\e[0m @@ -879,6 +890,9 @@ module SyncDefaultGems \e[1mPick all commits since the last picked commit\e[0m ruby #$0 -a rubygems +\e[1mUpdate repositories of default gems\e[0m + ruby #$0 up + \e[1mList known libraries\e[0m ruby #$0 list diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb index 78d2ed002e..778fe3311a 100644 --- a/tool/test-bundled-gems.rb +++ b/tool/test-bundled-gems.rb @@ -1,6 +1,7 @@ require 'rbconfig' require 'timeout' require 'fileutils' +require 'shellwords' require_relative 'lib/colorize' require_relative 'lib/gem_env' @@ -8,29 +9,31 @@ ENV.delete("GNUMAKEFLAGS") github_actions = ENV["GITHUB_ACTIONS"] == "true" +DEFAULT_ALLOWED_FAILURES = RUBY_PLATFORM =~ /mswin|mingw/ ? [ + 'debug', + 'irb', + 'csv', +] : [] allowed_failures = ENV['TEST_BUNDLED_GEMS_ALLOW_FAILURES'] || '' -if RUBY_PLATFORM =~ /mswin|mingw/ - allowed_failures = [allowed_failures, "net-smtp,rbs,debug"].join(',') -end -allowed_failures = allowed_failures.split(',').uniq.reject(&:empty?) +allowed_failures = allowed_failures.split(',').concat(DEFAULT_ALLOWED_FAILURES).uniq.reject(&:empty?) # make test-bundled-gems BUNDLED_GEMS=gem1,gem2,gem3 -bundled_gems = ARGV.first || '' +bundled_gems = nil if (bundled_gems = ARGV.first&.split(","))&.empty? colorize = Colorize.new rake = File.realpath("../../.bundle/bin/rake", __FILE__) gem_dir = File.realpath('../../gems', __FILE__) rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +run_opts = ENV["RUN_OPTS"]&.shellsplit exit_code = 0 ruby = ENV['RUBY'] || RbConfig.ruby failed = [] File.foreach("#{gem_dir}/bundled_gems") do |line| - next if /^\s*(?:#|$)/ =~ line - gem = line.split.first - next unless bundled_gems.empty? || bundled_gems.split(",").include?(gem) + next unless gem = line[/^[^\s\#]+/] + next if bundled_gems&.none? {|pat| File.fnmatch?(pat, gem)} next unless File.directory?("#{gem_dir}/src/#{gem}/test") - test_command = "#{ruby} -C #{gem_dir}/src/#{gem} #{rake} test" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", rake, "test"] first_timeout = 600 # 10min toplib = gem @@ -50,7 +53,15 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| File.unlink(path) if File.exist?(path) end - test_command << " stdlib_test validate RBS_SKIP_TESTS=#{__dir__}/rbs_skip_tests SKIP_RBS_VALIDATION=true" + rbs_skip_tests = [ + File.join(__dir__, "/rbs_skip_tests") + ] + + if /mswin|mingw/ =~ RUBY_PLATFORM + rbs_skip_tests << File.join(__dir__, "/rbs_skip_tests_windows") + end + + test_command.concat %W[stdlib_test validate RBS_SKIP_TESTS=#{rbs_skip_tests.join(File::PATH_SEPARATOR)} SKIP_RBS_VALIDATION=true] first_timeout *= 3 when "debug" @@ -60,7 +71,13 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| load_path = true when "test-unit" - test_command = "#{ruby} -C #{gem_dir}/src/#{gem} test/run-test.rb" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", "test/run.rb"] + + when "csv" + first_timeout = 30 + + when "win32ole" + next unless /mswin|mingw/ =~ RUBY_PLATFORM end @@ -75,12 +92,21 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| # 93(bright yellow) is copied from .github/workflows/mingw.yml puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" print "[command]" if github_actions - puts test_command - pid = Process.spawn(test_command, "#{/mingw|mswin/ =~ RUBY_PLATFORM ? 'new_' : ''}pgroup": true) - {nil => first_timeout, INT: 30, TERM: 10, KILL: nil}.each do |sig, sec| + p test_command + timeouts = {nil => first_timeout, INT: 30, TERM: 10, KILL: nil} + if /mingw|mswin/ =~ RUBY_PLATFORM + timeouts.delete(:TERM) # Inner process signal on Windows + group = :new_pgroup + pg = "" + else + group = :pgroup + pg = "-" + end + pid = Process.spawn(*test_command, group => true) + timeouts.each do |sig, sec| if sig puts "Sending #{sig} signal" - Process.kill("-#{sig}", pid) + Process.kill("#{pg}#{sig}", pid) end begin break Timeout.timeout(sec) {Process.wait(pid)} @@ -88,7 +114,7 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| end rescue Interrupt exit_code = Signal.list["INT"] - Process.kill("-KILL", pid) + Process.kill("#{pg}KILL", pid) Process.wait(pid) break end @@ -101,11 +127,11 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| "with exit code #{$?.exitstatus}") puts colorize.decorate(mesg, "fail") if allowed_failures.include?(gem) - mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES" + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" puts colorize.decorate(mesg, "skip") else failed << gem - exit_code = $?.exitstatus if $?.exitstatus + exit_code = 1 end end end diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb index 055577feea..28ef0bf7f8 100644 --- a/tool/test-coverage.rb +++ b/tool/test-coverage.rb @@ -114,6 +114,10 @@ pid = $$ pwd = Dir.pwd at_exit do + # Some tests leave GC.stress enabled, causing slow coverage processing. + # Reset it here to avoid performance issues. + GC.stress = false + exit_exc = $! Dir.chdir(pwd) do diff --git a/tool/test/init.rb b/tool/test/init.rb index 3a1143d01d..3fd1419a9c 100644 --- a/tool/test/init.rb +++ b/tool/test/init.rb @@ -1,7 +1,15 @@ -# This file includes the settings for "make test-all". +# This file includes the settings for "make test-all" and "make test-tool". # Note that this file is loaded not only by test/runner.rb but also by tool/lib/test/unit/parallel.rb. -ENV["GEM_SKIP"] = ENV["GEM_HOME"] = ENV["GEM_PATH"] = "".freeze +# Prevent test-all from using bundled gems +["GEM_HOME", "GEM_PATH"].each do |gem_env| + # Preserve the gem environment prepared by tool/runruby.rb for test-tool, which uses bundled gems. + ENV["BUNDLED_#{gem_env}"] = ENV[gem_env] + + ENV[gem_env] = "".freeze +end +ENV["GEM_SKIP"] = "".freeze + ENV.delete("RUBY_CODESIGN") Warning[:experimental] = false diff --git a/tool/test/test_commit_email.rb b/tool/test/test_commit_email.rb new file mode 100644 index 0000000000..db441584fd --- /dev/null +++ b/tool/test/test_commit_email.rb @@ -0,0 +1,102 @@ +require 'test/unit' +require 'shellwords' +require 'tmpdir' +require 'fileutils' +require 'open3' + +class TestCommitEmail < Test::Unit::TestCase + STDIN_DELIMITER = "---\n" + + def setup + omit 'git command is not available' unless system('git', '--version', out: File::NULL, err: File::NULL) + + @ruby = Dir.mktmpdir + Dir.chdir(@ruby) do + git('init', '--initial-branch=master') + git('config', 'user.name', 'Jóhän Grübél') + git('config', 'user.email', 'johan@example.com') + env = { + 'GIT_AUTHOR_DATE' => '2025-10-08T12:00:00Z', + 'GIT_CONFIG_GLOBAL' => @ruby + "/gitconfig", + 'TZ' => 'UTC', + } + git('commit', '--allow-empty', '-m', 'New repository initialized by cvs2svn.', env:) + git('commit', '--allow-empty', '-m', 'Initial revision', env:) + git('commit', '--allow-empty', '-m', 'version 1.0.0', env:) + end + + @sendmail = File.join(Dir.mktmpdir, 'sendmail') + File.write(@sendmail, <<~SENDMAIL, mode: "wx", perm: 0755) + #!/bin/sh + echo #{STDIN_DELIMITER.chomp.dump} + exec cat + SENDMAIL + + @commit_email = File.expand_path('../../tool/commit-email.rb', __dir__) + end + + def teardown + # Clean up temporary files if #setup was not omitted + if @sendmail + File.unlink(@sendmail) + Dir.rmdir(File.dirname(@sendmail)) + end + if @ruby + FileUtils.rm_rf(@ruby) + end + end + + def test_sendmail_encoding + omit 'the sendmail script does not work on windows' if windows? + + Dir.chdir(@ruby) do + before_rev = git('rev-parse', 'HEAD^').chomp + after_rev = git('rev-parse', 'HEAD').chomp + short_rev = after_rev[0...10] + + out, _, status = EnvUtil.invoke_ruby([ + { 'SENDMAIL' => @sendmail, 'TZ' => 'UTC' }.merge!(gem_env), + @commit_email, './', 'cvs-admin@ruby-lang.org', + before_rev, after_rev, 'refs/heads/master', + '--viewer-uri', 'https://github.com/ruby/ruby/commit/', + '--error-to', 'cvs-admin@ruby-lang.org', + ], '', true) + stdin = out.b.split(STDIN_DELIMITER.b, 2).last.force_encoding('UTF-8') + + assert_true(status.success?) + assert_equal(stdin, <<~EOS) + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: =?UTF-8?B?SsOzaMOkbiBHcsO8YsOpbA==?= <noreply@ruby-lang.org> + To: cvs-admin@ruby-lang.org + Subject: #{short_rev} (master): =?UTF-8?B?dmVyc2lvbuOAgDEuMC4w?= + J=C3=B3h=C3=A4n Gr=C3=BCb=C3=A9l\t2025-10-08 12:00:00 +0000 (Wed, 08 Oct 2= + 025) + + New Revision: #{short_rev} + + https://github.com/ruby/ruby/commit/#{short_rev} + + Log: + version=E3=80=801.0.0= + EOS + end + end + + private + + # Resurrect the gem environment preserved by tool/test/init.rb. + # This should work as long as you have run `make up` or `make install`. + def gem_env + { 'GEM_PATH' => ENV['BUNDLED_GEM_PATH'], 'GEM_HOME' => ENV['BUNDLED_GEM_HOME'] } + end + + def git(*cmd, env: {}) + out, status = Open3.capture2(env, 'git', *cmd) + unless status.success? + raise "git #{cmd.shelljoin}\n#{out}" + end + out + end +end diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb index e64c6c6fda..252687f3f3 100755 --- a/tool/test/test_sync_default_gems.rb +++ b/tool/test/test_sync_default_gems.rb @@ -2,6 +2,7 @@ require 'test/unit' require 'stringio' require 'tmpdir' +require 'rubygems/version' require_relative '../sync_default_gems' module Test_SyncDefaultGems @@ -19,14 +20,8 @@ module Test_SyncDefaultGems expected.concat(trailers.map {_1+"\n"}) end - out, err = capture_output do - SyncDefaultGems.message_filter(repo, sha, input: StringIO.new(input, "r")) - end - - all_assertions do |a| - a.for("error") {assert_empty err} - a.for("result") {assert_pattern_list(expected, out)} - end + out = SyncDefaultGems.message_filter(repo, sha, input) + assert_pattern_list(expected, out) end def test_subject_only @@ -90,14 +85,41 @@ module Test_SyncDefaultGems @target = nil pend "No git" unless system("git --version", out: IO::NULL) @testdir = Dir.mktmpdir("sync") - @git_config = %W"HOME GIT_CONFIG_GLOBAL".each_with_object({}) {|k, c| c[k] = ENV[k]} + user, email = "Ruby", "test@ruby-lang.org" + @git_config = %W"HOME USER GIT_CONFIG_GLOBAL GNUPGHOME".each_with_object({}) {|k, c| c[k] = ENV[k]} ENV["HOME"] = @testdir + ENV["USER"] = user + ENV["GNUPGHOME"] = @testdir + '/.gnupg' + expire = EnvUtil.apply_timeout_scale(30).to_i + # Generate a new unprotected key with default parameters that + # expires after 30 seconds. + if @gpgsign = system(*%w"gpg --quiet --batch --passphrase", "", + "--quick-generate-key", email, *%W"default default seconds=#{expire}", + err: IO::NULL) + # Fetch the generated public key. + signingkey = IO.popen(%W"gpg --quiet --list-public-key #{email}", &:read)[/^pub .*\n +\K\h+/] + end ENV["GIT_CONFIG_GLOBAL"] = @testdir + "/gitconfig" - git(*%W"config --global user.email test@ruby-lang.org") - git(*%W"config --global user.name", "Ruby") + git(*%W"config --global user.email", email) + git(*%W"config --global user.name", user) git(*%W"config --global init.defaultBranch default") + if signingkey + git(*%W"config --global user.signingkey", signingkey) + git(*%W"config --global commit.gpgsign true") + git(*%W"config --global gpg.program gpg") + git(*%W"config --global log.showSignature true") + end @target = "sync-test" - SyncDefaultGems::REPOSITORIES[@target] = ["ruby/#{@target}", "default"] + SyncDefaultGems::REPOSITORIES[@target] = SyncDefaultGems.repo( + ["ruby/#{@target}", "default"], + [ + ["lib", "lib"], + ["test", "test"], + ], + exclude: [ + "test/fixtures/*", + ], + ) @sha = {} @origdir = Dir.pwd Dir.chdir(@testdir) @@ -129,6 +151,9 @@ module Test_SyncDefaultGems def teardown if @target + if @gpgsign + system(*%W"gpgconf --kill all") + end Dir.chdir(@origdir) SyncDefaultGems::REPOSITORIES.delete(@target) ENV.update(@git_config) @@ -168,7 +193,7 @@ module Test_SyncDefaultGems end def top_commit(dir, format: "%H") - IO.popen(%W[git log --format=#{format} -1], chdir: dir, &:read)&.chomp + IO.popen(%W[git log --no-show-signature --format=#{format} -1], chdir: dir, &:read)&.chomp end def assert_sync(commits = true, success: true, editor: nil) @@ -293,5 +318,56 @@ module Test_SyncDefaultGems assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) assert_not_operator(File, :exist?, "src/lib/bad.rb", out) end - end + + def test_squash_merge + # This test is known to fail with git 2.43.0, which is used by Ubuntu 24.04. + # We don't know which exact version fixed it, but we know git 2.52.0 works. + stdout, status = Open3.capture2('git', '--version', err: File::NULL) + omit 'git version check failed' unless status.success? + git_version = stdout[/\Agit version \K\S+/] + omit "git #{git_version} is too old" if Gem::Version.new(git_version) < Gem::Version.new('2.44.0') + + # 2---. <- branch + # / \ + # 1---3---3'<- merge commit with conflict resolution + File.write("#@target/lib/conflict.rb", "# 1\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict.rb", chdir: @target) + + git(*%W"checkout -q -b branch", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 2\n") + File.write("#@target/lib/new.rb", "# new\n") + git(*%W"add lib/conflict.rb lib/new.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in branch", chdir: @target) + + git(*%W"checkout -q default", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 3\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in default", chdir: @target) + + # How can I suppress "Auto-merging ..." message from git merge? + git(*%W"merge -X ours -m", "Merge commit", "branch", chdir: @target, out: IO::NULL) + + out = assert_sync() + assert_equal("# 3\n", File.read("src/lib/conflict.rb"), out) + subject, body = top_commit("src", format: "%B").split("\n\n", 2) + assert_equal("[ruby/#@target] Merge commit", subject, out) + assert_includes(body, "Commit in branch", out) + end + + def test_no_upstream_file + group = SyncDefaultGems::Repository.group(%w[ + lib/un.rb + lib/unicode_normalize/normalize.rb + lib/unicode_normalize/tables.rb + lib/net/https.rb + ]) + expected = { + "un" => %w[lib/un.rb], + "net-http" => %w[lib/net/https.rb], + nil => %w[lib/unicode_normalize/normalize.rb lib/unicode_normalize/tables.rb], + } + assert_equal(expected, group) + end + end if /darwin|linux/ =~ RUBY_PLATFORM end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index 65c11fda42..d9bdc8f3c5 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -8,6 +8,8 @@ class TestAssertion < Test::Unit::TestCase end def test_timeout_separately + pend "hang-up" if /mswin|mingw/ =~ RUBY_PLATFORM + assert_raise(Timeout::Error) do assert_separately([], <<~"end;", timeout: 0.1) sleep @@ -15,6 +17,29 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assertion_count_separately + beginning = self._assertions + + assert_separately([], "") + assertions_at_nothing = self._assertions - beginning + + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "assert true") + assert_equal(1, self._assertions - prev_assertions) + + omit unless Process.respond_to?(:fork) + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "Process.fork {assert true}; assert true") + assert_equal(2, self._assertions - prev_assertions) + + prev_assertions = self._assertions + assertions_at_nothing + # TODO: assertions before `fork` are counted twice; it is possible + # to reset `_assertions` at `Process._fork`, but the hook can + # interfere in other tests. + assert_separately([], "assert true; Process.fork {assert true}") + assert_equal(3, self._assertions - prev_assertions) + end + def return_in_assert_raise assert_raise(RuntimeError) do return @@ -27,6 +52,38 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assert_raise_with_message + my_error = Class.new(StandardError) + + assert_raise_with_message(my_error, "with message") do + raise my_error, "with message" + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(RuntimeError, "with message") do + raise my_error, "with message" + end + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(my_error, "without message") do + raise my_error, "with message" + end + end + end + + def test_assert_raise_kind_of + my_error = Class.new(StandardError) + + assert_raise_kind_of(my_error) do + raise my_error + end + + assert_raise_kind_of(StandardError) do + raise my_error + end + end + def test_assert_pattern_list assert_pattern_list([/foo?/], "foo") assert_not_pattern_list([/foo?/], "afoo") diff --git a/tool/test/testunit/test_hideskip.rb b/tool/test/testunit/test_hideskip.rb index 0c4c9b40f2..a470368bca 100644 --- a/tool/test/testunit/test_hideskip.rb +++ b/tool/test/testunit/test_hideskip.rb @@ -6,7 +6,6 @@ class TestHideSkip < Test::Unit::TestCase assert_not_match(/^ *1\) Skipped/, hideskip) assert_match(/^ *1\) Skipped.*^ *2\) Skipped/m, hideskip("--show-skip")) output = hideskip("--hide-skip") - output.gsub!(/Successful RJIT finish\n/, '') if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? assert_match(/assertions\/s.\n+2 tests, 0 assertions, 0 failures, 0 errors, 2 skips/, output) end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb index 371e8524c1..adf7d62ecd 100644 --- a/tool/test/testunit/test_parallel.rb +++ b/tool/test/testunit/test_parallel.rb @@ -6,7 +6,15 @@ module TestParallel PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" TESTS = "#{__dir__}/tests_for_parallel" # use large timeout for --jit-wait - TIMEOUT = EnvUtil.apply_timeout_scale(defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? ? 100 : 30) + TIMEOUT = EnvUtil.apply_timeout_scale(100) + + def self.timeout(n, &blk) + start_time = Time.now + Timeout.timeout(n, &blk) + rescue Timeout::Error + end_time = Time.now + raise Timeout::Error, "execution expired (start: #{ start_time }, end: #{ end_time })" + end class TestParallelWorker < Test::Unit::TestCase def setup @@ -25,7 +33,7 @@ module TestParallel @worker_in.puts "quit normal" rescue IOError, Errno::EPIPE end - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@worker_pid) end rescue Timeout::Error @@ -45,7 +53,7 @@ module TestParallel end def test_run - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) @@ -58,7 +66,7 @@ module TestParallel end def test_run_multiple_testcase_in_one_file - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_second.rb test" assert_match(/^okay/,@worker_out.gets) @@ -75,7 +83,7 @@ module TestParallel end def test_accept_run_command_multiple_times - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) @@ -99,7 +107,7 @@ module TestParallel end def test_p - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_first.rb test" while buf = @worker_out.gets break if /^p (.+?)$/ =~ buf @@ -110,7 +118,7 @@ module TestParallel end def test_done - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_forth.rb test" while buf = @worker_out.gets break if /^done (.+?)$/ =~ buf @@ -118,24 +126,24 @@ module TestParallel assert_not_nil($1, "'done' was not found") result = Marshal.load($1.chomp.unpack1("m")) - assert_equal(5, result[0]) - pend "TODO: result[1] returns 17. We should investigate it" do # TODO: misusage of pend (pend doens't use given block) - assert_equal(12, result[1]) - end - assert_kind_of(Array,result[2]) - assert_kind_of(Array,result[3]) - assert_kind_of(Array,result[4]) - assert_kind_of(Array,result[2][1]) - assert_kind_of(Test::Unit::AssertionFailedError,result[2][0][2]) - assert_kind_of(Test::Unit::PendedError,result[2][1][2]) - assert_kind_of(Test::Unit::PendedError,result[2][2][2]) - assert_kind_of(Exception, result[2][3][2]) - assert_equal(result[5], "TestE") + tests, asserts, reports, failures, loadpaths, suite = result + assert_equal(5, tests) + assert_equal(12, asserts) + assert_kind_of(Array, reports) + assert_kind_of(Array, failures) + assert_kind_of(Array, loadpaths) + reports.sort_by! {|_, t| t} + assert_kind_of(Array, reports[1]) + assert_kind_of(Test::Unit::AssertionFailedError, reports[0][2]) + assert_kind_of(Test::Unit::PendedError, reports[1][2]) + assert_kind_of(Test::Unit::PendedError, reports[2][2]) + assert_kind_of(Exception, reports[3][2]) + assert_equal("TestE", suite) end end def test_quit - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "quit normal" assert_match(/^bye$/m,@worker_out.read) end @@ -143,9 +151,9 @@ module TestParallel end class TestParallel < Test::Unit::TestCase - def spawn_runner(*opt_args, jobs: "t1") + def spawn_runner(*opt_args, jobs: "t1", env: {}) @test_out, o = IO.pipe - @test_pid = spawn(*@__runner_options__[:ruby], TESTS+"/runner.rb", + @test_pid = spawn(env, *@__runner_options__[:ruby], TESTS+"/runner.rb", "--ruby", @__runner_options__[:ruby].join(" "), "-j", jobs, *opt_args, out: o, err: o) o.close @@ -154,7 +162,7 @@ module TestParallel def teardown begin if @test_pid - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@test_pid) end end @@ -167,54 +175,47 @@ module TestParallel def test_ignore_jzero spawn_runner(jobs: "0") - Timeout.timeout(TIMEOUT) { + ::TestParallel.timeout(TIMEOUT) { assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) } end def test_should_run_all_without_any_leaks spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^9 tests/,buf) end def test_should_retry_failed_on_workers - spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner "--retry" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying\.+$/,buf) end def test_no_retry_option spawn_runner "--no-retry" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} refute_match(/^Retrying\.+$/,buf) assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf) end def test_jobs_status spawn_runner "--jobs-status" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/\d+=ptest_(first|second|third|forth) */,buf) end def test_separate # this test depends to --jobs-status spawn_runner "--jobs-status", "--separate" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1, message("retried tests should run in different processes") {buf}) end def test_hungup - spawn_runner "--worker-timeout=1", "test4test_hungup.rb" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} - assert_match(/^Retrying hung up testcases\.+$/, buf) - assert_match(/^2 tests,.* 0 failures,/, buf) - end - - def test_retry_workers - spawn_runner "--worker-timeout=1", "test4test_slow_0.rb", "test4test_slow_1.rb", jobs: "2" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner("--worker-timeout=1", "--retry", "test4test_hungup.rb", env: {"RUBY_CRASH_REPORT"=>nil}) + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying hung up testcases\.+$/, buf) assert_match(/^2 tests,.* 0 failures,/, buf) end diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb index 8831676e19..54474c828d 100644 --- a/tool/test/testunit/tests_for_parallel/ptest_forth.rb +++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb @@ -8,19 +8,19 @@ class TestE < Test::Unit::TestCase assert_equal(1,1) end - def test_always_skip - skip "always" + def test_always_omit + omit "always" end def test_always_fail assert_equal(0,1) end - def test_skip_after_unknown_error + def test_pend_after_unknown_error begin raise UnknownError, "unknown error" rescue - skip "after raise" + pend "after raise" end end diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb deleted file mode 100644 index a749b0e1d3..0000000000 --- a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative 'slow_helper' - -class TestSlowV0 < Test::Unit::TestCase - include TestSlowTimeout -end diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb deleted file mode 100644 index 924a3b11fa..0000000000 --- a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative 'slow_helper' - -class TestSlowV1 < Test::Unit::TestCase - include TestSlowTimeout -end diff --git a/tool/update-NEWS-gemlist.rb b/tool/update-NEWS-gemlist.rb index e1535eb400..0b5503580d 100755 --- a/tool/update-NEWS-gemlist.rb +++ b/tool/update-NEWS-gemlist.rb @@ -6,10 +6,10 @@ prevs = [prev, prev.sub(/\.\d+\z/, '')] update = ->(list, type, desc = "updated") do item = ->(mark = "* ") do - "The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + + "### The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + list.map {|g, v|"#{mark}#{g} #{v}\n"}.join("") + "\n" end - news.sub!(/^(?:\*( +))?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do + news.sub!(/^(?:\*( +)|#+ *)?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do item["#{$1&.<< " "}*#{$2 || ' '}"] end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+( *\* *).*)*)*\n+\K/) do item[$1 || "* "] diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb index 2842516cac..dec6b49cee 100755 --- a/tool/update-bundled_gems.rb +++ b/tool/update-bundled_gems.rb @@ -1,4 +1,4 @@ -#!ruby -pla +#!ruby -alpF\s+|#.* BEGIN { require 'rubygems' date = nil @@ -9,7 +9,7 @@ output = STDERR if ARGF.file == STDIN END { output.print date.strftime("latest_date=%F") if date } -unless /^[^#]/ !~ (gem = $F[0]) +if gem = $F[0] ver = Gem::Version.new($F[1]) (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| s.platform == "ruby" && s.name == gem @@ -22,7 +22,10 @@ unless /^[^#]/ !~ (gem = $F[0]) else uri = $F[2] end - date = gem.date if !date or gem.date && gem.date > date + if (!date or gem.date && gem.date > date) and gem.date.to_i != 315_619_200 + # DEFAULT_SOURCE_DATE_EPOCH is meaningless + date = gem.date + end if $F[3] if $F[3].include?($F[1]) $F[3][$F[1]] = gem.version.to_s diff --git a/tool/update-deps b/tool/update-deps index 0b90876cd2..2d4a5674be 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -17,6 +17,14 @@ # 3. Use --fix to fix makefiles. # Ex. ./ruby tool/update-deps --fix # +# Usage to create a depend file initially: +# 1. Copy the dependency section from the Makefile generated by extconf.rb. +# Ex. ext/cgi/escape/Makefile +# 2. Add `# AUTOGENERATED DEPENDENCIES START` and `# AUTOGENERATED DEPENDENCIES END` +# sections to top and end of the depend file. +# 3. Run tool/update-deps --fix to fix the depend file. +# 4. Commit the depend file. +# # Other usages: # * Fix makefiles using previously detected dependency problems # Ex. ruby tool/update-deps --actual-fix [file] @@ -88,6 +96,15 @@ result.each {|k,v| # They can be referenced as $(top_srcdir)/filename. # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' FILES_IN_SOURCE_DIRECTORY = %w[ + prism/api_node.c + prism/ast.h + prism/diagnostic.c + prism/diagnostic.h + prism/node.c + prism/prettyprint.c + prism/serialize.c + prism/token_type.c + prism/version.h ] # Files built in the build directory (except extconf.h). @@ -149,16 +166,6 @@ FILES_NEED_VPATH = %w[ enc/trans/single_byte.c enc/trans/utf8_mac.c enc/trans/utf_16_32.c - - prism/api_node.c - prism/ast.h - prism/diagnostic.c - prism/diagnostic.h - prism/node.c - prism/prettyprint.c - prism/serialize.c - prism/token_type.c - prism/version.h ] # Multiple files with same filename. @@ -206,7 +213,7 @@ def in_makefile(target, source) when %r{\Acoroutine/} then source2 = "{$(VPATH)}$(COROUTINE_H)" else source2 = "$(top_srcdir)/#{source}" end - ["common.mk", target2, source2] + ["depend", target2, source2] when %r{\Aenc/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source @@ -319,6 +326,8 @@ def read_make_deps(cwd) deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp next if /\.o\z/ !~ target.to_s next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source) + next if /libzjit.o\z/ =~ target.to_s # skip ZJIT Rust object (no corresponding C source) + next if /target\/release\/libruby.o\z/ =~ target.to_s # skip YJIT+ZJIT Rust object (no corresponding C source) next if /\.bundle\// =~ target.to_s next if /\A\./ =~ target.to_s # skip rules such as ".c.o" #p [curdir, target, deps] diff --git a/tool/zjit_bisect.rb b/tool/zjit_bisect.rb new file mode 100755 index 0000000000..997c572f51 --- /dev/null +++ b/tool/zjit_bisect.rb @@ -0,0 +1,158 @@ +#!/usr/bin/env ruby +require 'logger' +require 'optparse' +require 'shellwords' +require 'tempfile' +require 'timeout' + +ARGS = {timeout: 5} +OptionParser.new do |opts| + opts.banner += " <path_to_ruby> -- <options>" + opts.on("--timeout=TIMEOUT_SEC", "Seconds until child process is killed") do |timeout| + ARGS[:timeout] = Integer(timeout) + end + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end +end.parse! + +usage = "Usage: zjit_bisect.rb <path_to_ruby> -- <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +raise(usage) if OPTIONS.empty? +LOGGER = Logger.new($stdout) + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +# Attempt to reduce the `items` argument as much as possible, returning the +# shorter version. `fixed` will always be used as part of the items when +# running `command`. +# `command` should return True if the command succeeded (the failure did not +# reproduce) and False if the command failed (the failure reproduced). +def bisect_impl(command, fixed, items, indent="") + LOGGER.info("#{indent}step fixed[#{fixed.length}] and items[#{items.length}]") + while items.length > 1 + LOGGER.info("#{indent}#{fixed.length + items.length} candidates") + # Return two halves of the given list. For odd-length lists, the second + # half will be larger. + half = items.length / 2 + left = items[0...half] + right = items[half..] + if !command.call(fixed + left) + items = left + next + end + if !command.call(fixed + right) + items = right + next + end + # We need something from both halves to trigger the failure. Try + # holding each half fixed and bisecting the other half to reduce the + # candidates. + new_right = bisect_impl(command, fixed + left, right, indent + "< ") + new_left = bisect_impl(command, fixed + new_right, left, indent + "> ") + return new_left + new_right + end + items +end + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +def run_bisect(command, items) + LOGGER.info("Verifying items") + if command.call(items) + raise StandardError.new("Command succeeded with full items") + end + if !command.call([]) + raise StandardError.new("Command failed with empty items") + end + bisect_impl(command, [], items) +end + +def add_zjit_options cmd + if RUBY == "make" + # Automatically detect that we're running a make command instead of a Ruby + # one. Pass the bisection options via RUN_OPTS/SPECOPTS instead. + zjit_opts = cmd.select { |arg| arg.start_with?("--zjit") } + run_opts_index = cmd.find_index { |arg| arg.start_with?("RUN_OPTS=") } + specopts_index = cmd.find_index { |arg| arg.start_with?("SPECOPTS=") } + if run_opts_index + run_opts = Shellwords.split(cmd[run_opts_index].delete_prefix("RUN_OPTS=")) + run_opts.concat(zjit_opts) + cmd[run_opts_index] = "RUN_OPTS=#{run_opts.shelljoin}" + elsif specopts_index + specopts = Shellwords.split(cmd[specopts_index].delete_prefix("SPECOPTS=")) + specopts.concat(zjit_opts) + cmd[specopts_index] = "SPECOPTS=#{specopts.shelljoin}" + else + raise "Expected RUN_OPTS or SPECOPTS to be present in make command" + end + cmd = cmd - zjit_opts + end + cmd +end + +def run_ruby *cmd + cmd = add_zjit_options(cmd) + pid = Process.spawn(*cmd, { + in: :close, + out: [File::NULL, File::RDWR], + err: [File::NULL, File::RDWR], + }) + begin + status = Timeout.timeout(ARGS[:timeout]) do + Process::Status.wait(pid) + end + rescue Timeout::Error + Process.kill("KILL", pid) + LOGGER.warn("Timed out after #{ARGS[:timeout]} seconds") + status = Process::Status.wait(pid) + end + + status +end + +def run_with_jit_list(ruby, options, jit_list) + # Make a new temporary file containing the JIT list + Tempfile.create("jit_list") do |temp_file| + temp_file.write(jit_list.join("\n")) + temp_file.flush + temp_file.close + # Run the JIT with the temporary file + run_ruby ruby, "--zjit-allowed-iseqs=#{temp_file.path}", *options + end +end + +# Try running with no JIT list to get a stable baseline +unless run_with_jit_list(RUBY, OPTIONS, []).success? + cmd = [RUBY, "--zjit-allowed-iseqs=/dev/null", *OPTIONS].shelljoin + raise "The command failed unexpectedly with an empty JIT list. To reproduce, try running the following: `#{cmd}`" +end +# Collect the JIT list from the failing Ruby process +jit_list = nil +Tempfile.create "jit_list" do |temp_file| + run_ruby RUBY, "--zjit-log-compiled-iseqs=#{temp_file.path}", *OPTIONS + jit_list = File.readlines(temp_file.path).map(&:strip).reject(&:empty?) +end +LOGGER.info("Starting with JIT list of #{jit_list.length} items.") +# Try running without the optimizer +status = run_with_jit_list(RUBY, ["--zjit-disable-hir-opt", *OPTIONS], jit_list) +if status.success? + LOGGER.warn "*** Command suceeded with HIR optimizer disabled. HIR optimizer is probably at fault. ***" +end +# Now narrow it down +command = lambda do |items| + run_with_jit_list(RUBY, OPTIONS, items).success? +end +result = run_bisect(command, jit_list) +File.open("jitlist.txt", "w") do |file| + file.puts(result) +end +puts "Run:" +jitlist_path = File.expand_path("jitlist.txt") +puts add_zjit_options([RUBY, "--zjit-allowed-iseqs=#{jitlist_path}", *OPTIONS]).shelljoin +puts "Reduced JIT list (available in jitlist.txt):" +puts result diff --git a/tool/zjit_iongraph.html b/tool/zjit_iongraph.html new file mode 100644 index 0000000000..993cce9045 --- /dev/null +++ b/tool/zjit_iongraph.html @@ -0,0 +1,551 @@ +<!-- Copyright Mozilla and licensed under Mozilla Public License Version 2.0. + Source can be found at https://github.com/mozilla-spidermonkey/iongraph --> +<!-- Generated by `npm run build-www` on + 39b04fa18f23cbf3fd2ca7339a45341ff3351ba1 in tekknolagi/iongraph --> +<!DOCTYPE html> + +<head> + <title>iongraph</title> + <style>/* iongraph-specific styles inspired by Tachyons */ + +:root { + --ig-size-1: 1rem; + --ig-size-2: 2rem; + --ig-size-3: 4rem; + --ig-size-4: 8rem; + --ig-size-5: 16rem; + + --ig-spacing-1: .25rem; + --ig-spacing-2: .5rem; + --ig-spacing-3: 1rem; + --ig-spacing-4: 2rem; + --ig-spacing-5: 4rem; + --ig-spacing-6: 8rem; + --ig-spacing-7: 16rem; + + --ig-text-color: black; + --ig-text-color-dim: #777; + --ig-background-primary: #ffb54e; + --ig-background-light: white; + --ig-border-color: #0c0c0d; + + --ig-block-header-color: #0c0c0d; + --ig-loop-header-color: #1fa411; + --ig-movable-color: #1048af; + --ig-rob-color: #444; + --ig-in-worklist-color: red; + + --ig-block-selected: #ffc863; + --ig-block-last-selected: #ffb54e; + + --ig-highlight-0: #ffb54e; + --ig-highlight-1: #ffb5c5; + --ig-highlight-2: #a4cbff; + --ig-highlight-3: #8be182; + --ig-highlight-4: #d9a4fd; + + --ig-flash-color: #ffb54e; + + /* + * The heatmap of sample counts will effectively be sampled from a gradient: + * + * |----------|---------------------------------------| + * cold "cool" hot + * + * The "cold" color will simply be transparent. Therefore, the "cool" + * threshold indicates where the instruction will be fully colored and + * noticeable to the user. + */ + --ig-hot-color: #ff849e; + --ig-cool-color: #ffe546; + --ig-cool-threshold: 0.2; +} + +a.ig-link-normal { + color: inherit; + text-decoration: inherit; +} + +.ig-flex { + display: flex; +} + +.ig-flex-column { + flex-direction: column; +} + +.ig-flex-basis-0 { + flex-basis: 0; +} + +.ig-flex-grow-1 { + flex-grow: 1; +} + +.ig-flex-shrink-0 { + flex-shrink: 0; +} + +.ig-flex-shrink-1 { + flex-shrink: 1; +} + +.ig-items-center { + align-items: center; +} + +.ig-relative { + position: relative; +} + +.ig-absolute { + position: absolute; +} + +.ig-absolute-fill { + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.ig-g1 { + gap: var(--ig-spacing-1); +} + +.ig-g2 { + gap: var(--ig-spacing-2); +} + +.ig-g3 { + gap: var(--ig-spacing-3); +} + +.ig-w1 { + width: var(--ig-size-1); +} + +.ig-w2 { + width: var(--ig-size-2); +} + +.ig-w3 { + width: var(--ig-size-3); +} + +.ig-w4 { + width: var(--ig-size-4); +} + +.ig-w5 { + width: var(--ig-size-5); +} + +.ig-w-100 { + width: 100%; +} + +.ig-ba { + border-style: solid; + border-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bt { + border-top-style: solid; + border-top-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-br { + border-right-style: solid; + border-right-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bb { + border-bottom-style: solid; + border-bottom-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bl { + border-left-style: solid; + border-left-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-pa1 { + padding: var(--ig-spacing-1); +} + +.ig-pa2 { + padding: var(--ig-spacing-2); +} + +.ig-pa3 { + padding: var(--ig-spacing-3); +} + +.ig-ph1 { + padding-left: var(--ig-spacing-1); + padding-right: var(--ig-spacing-1); +} + +.ig-ph2 { + padding-left: var(--ig-spacing-2); + padding-right: var(--ig-spacing-2); +} + +.ig-ph3 { + padding-left: var(--ig-spacing-3); + padding-right: var(--ig-spacing-3); +} + +.ig-pv1 { + padding-top: var(--ig-spacing-1); + padding-bottom: var(--ig-spacing-1); +} + +.ig-pv2 { + padding-top: var(--ig-spacing-2); + padding-bottom: var(--ig-spacing-2); +} + +.ig-pv3 { + padding-top: var(--ig-spacing-3); + padding-bottom: var(--ig-spacing-3); +} + +.ig-pt1 { + padding-top: var(--ig-spacing-1); +} + +.ig-pt2 { + padding-top: var(--ig-spacing-2); +} + +.ig-pt3 { + padding-top: var(--ig-spacing-3); +} + +.ig-pr1 { + padding-right: var(--ig-spacing-1); +} + +.ig-pr2 { + padding-right: var(--ig-spacing-2); +} + +.ig-pr3 { + padding-right: var(--ig-spacing-3); +} + +.ig-pb1 { + padding-bottom: var(--ig-spacing-1); +} + +.ig-pb2 { + padding-bottom: var(--ig-spacing-2); +} + +.ig-pb3 { + padding-bottom: var(--ig-spacing-3); +} + +.ig-pl1 { + padding-left: var(--ig-spacing-1); +} + +.ig-pl2 { + padding-left: var(--ig-spacing-2); +} + +.ig-pl3 { + padding-left: var(--ig-spacing-3); +} + +.ig-f1 { + font-size: 3rem; +} + +.ig-f2 { + font-size: 2.25rem; +} + +.ig-f3 { + font-size: 1.5rem; +} + +.ig-f4 { + font-size: 1.25rem; +} + +.ig-f5 { + font-size: 1rem; +} + +.ig-f6 { + font-size: .875rem; +} + +.ig-f7 { + font-size: .75rem; +} + +.ig-text-normal { + color: var(--ig-text-color); +} + +.ig-text-dim { + color: var(--ig-text-color-dim); +} + +.ig-tl { + text-align: left; +} + +.ig-tr { + text-align: right; +} + +.ig-tc { + text-align: center; +} + +.ig-bg-white { + background-color: var(--ig-background-light); +} + +.ig-bg-primary { + background-color: var(--ig-background-primary); +} + +.ig-overflow-hidden { + overflow: hidden; +} + +.ig-overflow-auto { + overflow: auto; +} + +.ig-overflow-x-auto { + overflow-x: auto; +} + +.ig-overflow-y-auto { + overflow-y: auto; +} + +.ig-hide-if-empty:empty { + display: none; +} + +/* Non-utility styles */ + +.ig-graph { + color: var(--ig-text-color); + position: absolute; + left: 0; + top: 0; + isolation: isolate; +} + +.ig-block { + position: absolute; + + .ig-block-header { + font-weight: bold; + text-align: center; + background-color: var(--ig-block-header-color); + color: white; + padding: 0 1em; + border: 1px solid var(--ig-border-color); + border-width: 1px 1px 0; + } + + .ig-instructions { + padding: 0.5em; + border: 1px solid var(--ig-border-color); + border-width: 0 1px 1px; + + table { + border-collapse: collapse; + } + + td, + th { + white-space: nowrap; + padding: 0.1em 0.5em; + } + + th { + font-weight: normal; + } + } + + &.ig-selected { + outline: 4px solid var(--ig-block-selected); + } + + &.ig-last-selected { + outline-color: var(--ig-block-last-selected); + } +} + +.ig-block-att-loopheader { + .ig-block-header { + background-color: var(--ig-loop-header-color); + } +} + +.ig-block-att-splitedge { + .ig-instructions { + border-style: dotted; + border-width: 0 2px 2px; + } +} + +.ig-ins-num { + text-align: right; + cursor: pointer; +} + +.ig-ins-type { + text-align: right; +} + +.ig-ins-samples { + font-size: 0.875em; + text-align: right; + cursor: pointer; +} + +.ig-use { + padding: 0 0.25em; + border-radius: 2px; + cursor: pointer; +} + +.ig-edge-label { + position: absolute; + font-size: 0.8em; + line-height: 1; + bottom: -1em; + padding-left: 4px; +} + +.ig-ins-att-RecoveredOnBailout { + color: var(--ig-rob-color); +} + +.ig-ins-att-Movable { + color: var(--ig-movable-color); +} + +.ig-ins-att-Guard { + text-decoration: underline; +} + +.ig-ins-att-InWorklist { + color: var(--ig-in-worklist-color); +} + +.ig-can-flash { + transition: outline-color 1s ease-out; + outline: 3px solid color-mix(in srgb, var(--ig-flash-color) 0%, transparent); +} + +.ig-flash { + transition: outline-color 0s; + outline-color: var(--ig-flash-color); +} + +.ig-hotness { + --ig-hotness: 0; + + --ig-cold-color: color-mix(in srgb, var(--ig-cool-color) 20%, transparent); + background-color: + /* cool <-> hot */ + color-mix(in oklab, + /* cold <-> cool */ + color-mix(in oklab, + /* dead or cold */ + color-mix(in srgb, transparent, var(--ig-cold-color) clamp(0%, calc(var(--ig-hotness) * 100000000%), 100%)), + var(--ig-cool-color) clamp(0%, calc((var(--ig-hotness) / var(--ig-cool-threshold)) * 100%), 100%)), + var(--ig-hot-color) clamp(0%, calc(((var(--ig-hotness) - var(--ig-cool-threshold)) / (1 - var(--ig-cool-threshold))) * 100%), 100%)); +} + +.ig-highlight { + --ig-highlight-color: transparent; + background-color: var(--ig-highlight-color); +}</style> + <style>* { + box-sizing: border-box; +} + +:root { + font-size: 0.875rem; +} + +body { + margin: 0; + background-color: #e5e8ea; + /* Font, and many other styles, taken from the Firefox Profiler/ */ + font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen-Sans, Ubuntu, Noto Sans, Liberation Sans, Cantarell, Helvetica Neue, sans-serif; +} + +#container { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +.tweaks-panel { + position: absolute; + bottom: 0; + right: 0; + padding: 1rem; + border: 1px solid black; + border-width: 1px 0 0 1px; + background-color: white; +}</style> +</head> + +<body> + <script>"use strict";var iongraph=(()=>{var J=Object.defineProperty;var mt=Object.getOwnPropertyDescriptor;var gt=Object.getOwnPropertyNames;var ft=Object.prototype.hasOwnProperty;var bt=(a,t)=>{for(var e in t)J(a,e,{get:t[e],enumerable:!0})},kt=(a,t,e,s)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of gt(t))!ft.call(a,i)&&i!==e&&J(a,i,{get:()=>t[i],enumerable:!(s=mt(t,i))||s.enumerable});return a};var yt=a=>kt(J({},"__esModule",{value:!0}),a);var Yt={};bt(Yt,{StandaloneUI:()=>et,WebUI:()=>tt});function X(a){a.version===void 0&&(a.version=0);for(let t of a.functions)xt(t,a.version);return a.version=1,a}function xt(a,t){for(let e of a.passes){for(let s of e.mir.blocks)wt(s,t);for(let s of e.lir.blocks)It(s,t)}return a}function wt(a,t){t===0&&(a.ptr=(a.id??a.number)+1,a.id=a.number);for(let e of a.instructions)vt(e,t);return a}function vt(a,t){return t===0&&(a.ptr=a.id),a}function It(a,t){t===0&&(a.ptr=a.id??a.number,a.id=a.number);for(let e of a.instructions)Lt(e,t);return a}function Lt(a,t){return t===0&&(a.ptr=a.id,a.mirPtr=null),a}function q(a,t,e){return Math.max(t,Math.min(e,a))}function R(a,t,e,s){return(a-t)*Math.pow(e,s)+t}function k(a,t,e=!1){if(!a)if(e)console.error(t??"Assertion failed");else throw new Error(t??"Assertion failed")}function L(a,t){return k(a,t),a}function Pt(a){return typeof a=="string"?document.createTextNode(a):a}function Bt(a,t){for(let e of t)e&&a.appendChild(Pt(e))}function x(a,t,e,s){let i=document.createElement(a);if(t&&t.length>0){let c=t.filter(l=>!!l);i.classList.add(...c)}return e?.(i),s&&Bt(i,s),i}var st=Object.prototype.hasOwnProperty;function E(a,t){var e,s;if(a===t)return!0;if(a&&t&&(e=a.constructor)===t.constructor){if(e===Date)return a.getTime()===t.getTime();if(e===RegExp)return a.toString()===t.toString();if(e===Array){if((s=a.length)===t.length)for(;s--&&E(a[s],t[s]););return s===-1}if(!e||typeof a=="object"){s=0;for(e in a)if(st.call(a,e)&&++s&&!st.call(t,e)||!(e in t)||!E(a[e],t[e]))return!1;return Object.keys(t).length===s}}return a!==a&&t!==t}function N(a,t,e={}){let s=t,i=[],c={get(){return s},set(l){s=l;for(let o of i)o(s)},valueOf(){return s},toString(){return String(s)},[Symbol.toPrimitive](l){return l==="string"?String(s):s},onChange(l){i.push(l)},initial:t,name:a,min:e.min??0,max:e.max??100,step:e.step??1};return(e.tweaksObject??K).add(c)}var F=class{constructor(t){this.container=t.container,this.tweaks=[],this.callbacks=[]}add(t){let e=this.tweaks.find(r=>r.name===t.name);if(e)return e;this.tweaks.push(t);let s=document.createElement("div");this.container.appendChild(s),s.style.display="flex",s.style.alignItems="center",s.style.justifyContent="end",s.style.gap="0.5rem";let i=t.name.replace(/[a-zA-Z0-9]/g,"_"),c=document.createElement("label");s.appendChild(c),c.innerText=t.name,c.htmlFor=`tweak-${i}-input`;let l=document.createElement("input");s.appendChild(l),l.type="number",l.value=String(t),l.id=`tweak-${i}-input`,l.style.width="4rem",l.addEventListener("input",()=>{t.set(l.valueAsNumber)});let o=document.createElement("input");s.appendChild(o),o.type="range",o.value=String(t),o.min=String(t.min),o.max=String(t.max),o.step=t.step===0?"any":String(t.step),o.addEventListener("input",()=>{t.set(o.valueAsNumber)});let n=document.createElement("button");return s.appendChild(n),n.innerText="Reset",n.disabled=t.get()===t.initial,n.addEventListener("click",()=>{t.set(t.initial)}),t.onChange(r=>{l.value=String(r),o.value=String(r),n.disabled=t.get()===t.initial;for(let d of this.callbacks)d(t)}),t}onTweak(t){this.callbacks.push(t)}},nt=document.createElement("div");nt.classList.add("tweaks-panel");var K=new F({container:nt});K.onTweak(a=>{window.dispatchEvent(new CustomEvent("tweak",{detail:a}))});window.tweaks=K;var Nt=document.createElement("div"),St=new F({container:Nt}),ot=N("Test Value",3,{tweaksObject:St});ot.set(4);ot=4;var dt=N("Debug?",0,{min:0,max:1}),P=20,C=44,I=16,S=60,T=12,H=36,it=16,D=16,Tt=N("Layout Iterations",2,{min:0,max:6}),rt=N("Nearly Straight Threshold",30,{min:0,max:200}),$t=N("Nearly Straight Iterations",8,{min:0,max:10}),at=N("Stop At Pass",30,{min:0,max:30}),Mt=1.5,Et=.01,Ct=1,Ht=.1,z=40;function _(a){return a.mir.attributes.includes("loopheader")}function Dt(a){return a.loopHeight!==void 0}function $(a){if(k(a),Dt(a))return a;throw new Error(`Block ${a.id} is not a pseudo LoopHeader`)}var j=1,lt=2,U=4,O=0,W=1,w=new Proxy(console,{get(a,t){let e=a[t];return typeof e!="function"?e:+dt?e.bind(a):()=>{}}}),V=class{constructor(t,e,s={}){this.viewport=t;let i=t.getBoundingClientRect();this.viewportSize={x:i.width,y:i.height},this.graphContainer=document.createElement("div"),this.graphContainer.classList.add("ig-graph"),this.graphContainer.style.transformOrigin="top left",this.viewport.appendChild(this.graphContainer),this.sampleCounts=s.sampleCounts,this.maxSampleCounts=[0,0],this.heatmapMode=O;for(let[n,r]of this.sampleCounts?.totalLineHits??[])this.maxSampleCounts[O]=Math.max(this.maxSampleCounts[O],r);for(let[n,r]of this.sampleCounts?.selfLineHits??[])this.maxSampleCounts[W]=Math.max(this.maxSampleCounts[W],r);this.size={x:0,y:0},this.numLayers=0,this.zoom=1,this.translation={x:0,y:0},this.animating=!1,this.targetZoom=1,this.targetTranslation={x:0,y:0},this.startMousePos={x:0,y:0},this.lastMousePos={x:0,y:0},this.selectedBlockPtrs=new Set,this.lastSelectedBlockPtr=0,this.nav={visited:[],currentIndex:-1,siblings:[]},this.highlightedInstructions=[],this.instructionPalette=s.instructionPalette??[0,1,2,3,4].map(n=>`var(--ig-highlight-${n})`),this.blocks=e.mir.blocks.map(n=>{let r={ptr:n.ptr,id:n.id,mir:n,lir:e.lir.blocks.find(d=>d.id===n.id)??null,preds:[],succs:[],el:void 0,size:{x:0,y:0},layer:-1,loopID:-1,layoutNode:void 0};if(r.mir.attributes.includes("loopheader")){let d=r;d.loopHeight=0,d.parentLoop=null,d.outgoingEdges=[]}return k(r.ptr,"blocks must always have non-null ptrs"),r}),this.blocksByID=new Map,this.blocksByPtr=new Map,this.insPtrsByID=new Map,this.insIDsByPtr=new Map,this.loops=[];for(let n of this.blocks){this.blocksByID.set(n.id,n),this.blocksByPtr.set(n.ptr,n);for(let r of n.mir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id);if(n.lir)for(let r of n.lir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id)}for(let n of this.blocks)if(n.preds=n.mir.predecessors.map(r=>L(this.blocksByID.get(r))),n.succs=n.mir.successors.map(r=>L(this.blocksByID.get(r))),_(n)){let r=n.preds.filter(d=>d.mir.attributes.includes("backedge"));k(r.length===1),n.backedge=r[0]}for(let n of this.blocks)n.el=this.renderBlock(n);for(let n of this.blocks)n.size={x:n.el.clientWidth,y:n.el.clientHeight};let[c,l,o]=this.layout();this.render(c,l,o),this.addEventListeners()}layout(){let[t,e]=this.findLayoutRoots();w.log("Layout roots:",t.map(l=>l.id));for(let l of[...t,...e]){let o=l;o.loopHeight=0,o.parentLoop=null,o.outgoingEdges=[],Object.defineProperty(o,"backedge",{get(){throw new Error("Accessed .backedge on a pseudo loop header! Don't do that.")},configurable:!0})}for(let l of t)w.group("findLoops"),this.findLoops(l),w.groupEnd();for(let l of t)w.group("layer"),this.layer(l),w.groupEnd();for(let l of e)l.layer=0,l.loopID=l.id;let s=this.makeLayoutNodes();this.straightenEdges(s);let i=this.finagleJoints(s),c=this.verticalize(s,i);return[s,c,i]}findLayoutRoots(){let t=[],e=[],s=this.blocks.filter(i=>i.preds.length===0);for(let i of s){let c=i;if(i.mir.attributes.includes("osr")){k(i.succs.length>0);let l=i.succs[0];c=l;for(let o=0;;o++){if(o>=1e7)throw new Error("likely infinite loop");let n=c.preds.filter(r=>!r.mir.attributes.includes("osr")&&!r.mir.attributes.includes("backedge"));if(n.length===0)break;c=n[0]}c!==l?e.push(i):c=i}t.includes(c)||t.push(c)}return[t,e]}findLoops(t,e=null){if(e===null&&(e=[t.id]),!(t.loopID>=0)){if(w.log("block:",t.id,t.mir.loopDepth,"loopIDsByDepth:",e),w.log(t.mir.attributes),_(t)){let s=e[e.length-1],i=$(this.blocksByID.get(s));t.parentLoop=i,e=[...e,t.id],w.log("Block",t.id,"is true loop header, loopIDsByDepth is now",e)}if(t.mir.loopDepth>e.length-1&&(t.mir.loopDepth=e.length-1,w.log("Block",t.id,"has been forced back to loop depth",t.mir.loopDepth)),t.mir.loopDepth<e.length-1&&(e=e.slice(0,t.mir.loopDepth+1),w.log("Block",t.id,"has low loop depth, therefore we exited a loop. loopIDsByDepth:",e)),t.loopID=e[t.mir.loopDepth],!t.mir.attributes.includes("backedge"))for(let s of t.succs)this.findLoops(s,e)}}layer(t,e=0){if(w.log("block",t.id,"layer",e),t.mir.attributes.includes("backedge")){t.layer=t.succs[0].layer;return}if(e<=t.layer)return;t.layer=Math.max(t.layer,e),this.numLayers=Math.max(t.layer+1,this.numLayers);let s=$(this.blocksByID.get(t.loopID));for(;s;)s.loopHeight=Math.max(s.loopHeight,t.layer-s.layer+1),s=s.parentLoop;for(let i of t.succs)i.mir.loopDepth<t.mir.loopDepth?$(this.blocksByID.get(t.loopID)).outgoingEdges.push(i):this.layer(i,e+1);if(_(t))for(let i of t.outgoingEdges)this.layer(i,e+t.loopHeight)}makeLayoutNodes(){w.group("makeLayoutNodes");function t(o,n,r){o.dstNodes[n]=r,r.srcNodes.includes(o)||r.srcNodes.push(o),w.log("connected",o.id,"to",r.id)}let e;{let o={};for(let n of this.blocks)o[n.layer]||(o[n.layer]=[]),o[n.layer].push(n);e=Object.entries(o).map(([n,r])=>[Number(n),r]).sort((n,r)=>n[0]-r[0]).map(([n,r])=>r)}let s=0,i=e.map(()=>[]),c=[],l=new Map;for(let[o,n]of e.entries()){w.group("layer",o,"blocks",n.map(u=>u.id));let r=[];for(let u of n)for(let h=c.length-1;h>=0;h--){let p=c[h];p.dstBlock===u&&(r.unshift(p),c.splice(h,1))}let d=new Map;for(let u of c){let h,p=d.get(u.dstBlock.id);if(p)t(u.src,u.srcPort,p),h=p;else{let m={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:u.dstBlock,jointOffsets:[],flags:0};t(u.src,u.srcPort,m),i[o].push(m),d.set(u.dstBlock.id,m),h=m,w.log("Created dummy",h.id,"on the way to block",u.dstBlock.id)}u.src=h,u.srcPort=0}let b=[];for(let u of n){let h=$(this.blocksByID.get(u.loopID));for(;_(h);){let p=b.find(f=>f.loopID===h.id);p?p.block=u:b.push({loopID:h.id,block:u});let m=h.parentLoop;if(!m)break;h=m}}let g=[];for(let u of n){let h={id:s++,pos:{x:P,y:P},size:u.size,block:u,srcNodes:[],dstNodes:[],jointOffsets:[],flags:0};for(let p of r)p.dstBlock===u&&t(p.src,p.srcPort,h);i[o].push(h),u.layoutNode=h;for(let p of b.filter(m=>m.block===u)){let m=$(this.blocksByID.get(p.loopID)).backedge,f={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:m,jointOffsets:[],flags:0},y=l.get(m);y?t(f,0,y):(f.flags|=U,t(f,0,m.layoutNode)),i[o].push(f),l.set(m,f)}if(u.mir.attributes.includes("backedge"))t(u.layoutNode,0,u.succs[0].layoutNode);else for(let[p,m]of u.succs.entries())m.mir.attributes.includes("backedge")?g.push({src:h,srcPort:p,dstBlock:m}):c.push({src:h,srcPort:p,dstBlock:m})}for(let u of g){let h=L(l.get(u.dstBlock));t(u.src,u.srcPort,h)}w.groupEnd()}w.log("Pruning backedge dummies");{let o=[];for(let r of At(i))r.srcNodes.length===0&&o.push(r);let n=new Set;for(let r of o){let d=r;for(;d.block===null&&d.srcNodes.length===0;)Ot(d),n.add(d),k(d.dstNodes.length===1),d=d.dstNodes[0]}for(let r of i)for(let d=r.length-1;d>=0;d--)n.has(r[d])&&r.splice(d,1)}w.log("Marking leftmost and rightmost dummies");for(let o of i){for(let n=0;n<o.length&&o[n].block===null;n++)o[n].flags|=j;for(let n=o.length-1;n>=0&&o[n].block===null;n--)o[n].flags|=lt}w.log("Verifying integrity of all nodes");for(let o of i)for(let n of o){n.block?k(n.dstNodes.length===n.block.succs.length,`expected node ${n.id} for block ${n.block.id} to have ${n.block.succs.length} destination nodes, but got ${n.dstNodes.length} instead`):k(n.dstNodes.length===1,`expected dummy node ${n.id} to have only one destination node, but got ${n.dstNodes.length} instead`);for(let r=0;r<n.dstNodes.length;r++)k(n.dstNodes[r]!==void 0,`dst slot ${r} of node ${n.id} was undefined`)}return w.groupEnd(),i}straightenEdges(t){let e=g=>{for(let u=0;u<g.length-1;u++){let h=g[u],p=g[u+1],m=h.block===null&&p.block!==null,f=h.pos.x+h.size.x+(m?I:0)+C;p.pos.x=Math.max(p.pos.x,f)}},s=()=>{for(let g of t)for(let u of g){if(u.block===null)continue;let h=u.block.loopID!==null?$(this.blocksByID.get(u.block.loopID)):null;if(h){let p=h.layoutNode;u.pos.x=Math.max(u.pos.x,p.pos.x)}}},i=()=>{let g=new Map;for(let u of Q(t)){let h=u.dstBlock,p=u.pos.x;g.set(h,Math.max(g.get(h)??0,p))}for(let u of Q(t)){let h=u.dstBlock,p=g.get(h);k(p,`no position for backedge ${h.id}`),u.pos.x=p}for(let u of t)e(u)},c=()=>{let g=new Map;for(let u of t){let h=0,p=0;for(;h<u.length;h++)if(!(u[h].flags&j)){p=u[h].pos.x;break}for(h-=1,p-=C+I;h>=0;h--){let m=u[h];k(m.block===null&&m.flags&j);let f=p;for(let y of m.srcNodes){let v=y.pos.x+y.dstNodes.indexOf(m)*S;v<f&&(f=v)}m.pos.x=f,p=m.pos.x-C,g.set(m.dstBlock,Math.min(g.get(m.dstBlock)??1/0,f))}}for(let u of Q(t)){if(!(u.flags&j))continue;let h=g.get(u.dstBlock);k(h,`no position for run to block ${u.dstBlock.id}`),u.pos.x=h}},l=()=>{for(let g=0;g<t.length-1;g++){let u=t[g];e(u);let h=-1;for(let p of u)for(let[m,f]of p.dstNodes.entries()){let y=t[g+1].indexOf(f);if(y>h&&f.srcNodes[0]===p){let v=I+S*m,B=I,Z=f.pos.x;f.pos.x=Math.max(f.pos.x,p.pos.x+v-B),f.pos.x!==Z&&(h=y)}}}},o=()=>{for(let g of t){for(let u=g.length-1;u>=0;u--){let h=g[u];if(!h.block||h.block.mir.attributes.includes("backedge"))continue;let p=[];for(let m of h.srcNodes){let f=I+m.dstNodes.indexOf(h)*S,y=I;p.push(m.pos.x+f-(h.pos.x+y))}for(let[m,f]of h.dstNodes.entries()){if(f.block===null&&f.dstBlock.mir.attributes.includes("backedge"))continue;let y=I+m*S,v=I;p.push(f.pos.x+v-(h.pos.x+y))}if(!p.includes(0)){p=p.filter(m=>m>0).sort((m,f)=>m-f);for(let m of p){let f=!1;for(let y=u+1;y<g.length;y++){let v=g[y];if(v.flags<)continue;let B=h.pos.x+m,Z=h.pos.x+m+h.size.x,ht=v.pos.x-C,pt=v.pos.x+v.size.x+C;Z>=ht&&B<=pt&&(f=!0)}if(!f){h.pos.x+=m;break}}}}e(g)}},n=()=>{for(let g=t.length-1;g>=0;g--){let u=t[g];e(u);for(let h of u)for(let p of h.srcNodes){if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}},r=()=>{for(let g=0;g<t.length;g++){let u=t[g];e(u);for(let h of u){if(h.dstNodes.length===0)continue;let p=h.dstNodes[0];if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}};function d(g,u){let h=[];for(let p=0;p<u;p++)for(let m of g)h.push(m);return h}let b=[...d([l,s,i],Tt),i,...d([n,r],$t),o,i,c];k(b.length<=(at.initial??1/0),`STOP_AT_PASS was too small - should be at least ${b.length}`),w.group("Running passes");for(let[g,u]of b.entries())g<at&&(w.log(u.name??u.toString()),u());w.groupEnd()}finagleJoints(t){let e=[];for(let s of t){let i=[];for(let r of s)if(r.jointOffsets=new Array(r.dstNodes.length).fill(0),!r.block?.mir.attributes.includes("backedge"))for(let[d,b]of r.dstNodes.entries()){let g=r.pos.x+I+S*d,u=b.pos.x+I;Math.abs(u-g)<2*T||i.push({x1:g,x2:u,src:r,srcPort:d,dst:b})}i.sort((r,d)=>r.x1-d.x1);let c=[],l=[];t:for(let r of i){let d=r.x2-r.x1>=0?c:l,b=null;for(let g=d.length-1;g>=0;g--){let u=d[g],h=!1;for(let p of u){if(r.dst===p.dst){u.push(r);continue t}let m=Math.min(r.x1,r.x2),f=Math.max(r.x1,r.x2),y=Math.min(p.x1,p.x2),v=Math.max(p.x1,p.x2);if(f>=y&&m<=v){h=!0;break}}if(h)break;b=u}b?b.push(r):d.push([r])}let o=Math.max(0,c.length+l.length-1)*it,n=-o/2;for(let r of[...c.reverse(),...l]){for(let d of r)d.src.jointOffsets[d.srcPort]=n;n+=it}e.push(o)}return k(e.length===t.length),e}verticalize(t,e){let s=new Array(t.length),i=P;for(let c=0;c<t.length;c++){let l=t[c],o=0;for(let n of l)n.pos.y=i,o=Math.max(o,n.size.y);s[c]=o,i+=o+H+e[c]+H}return s}renderBlock(t){let e=document.createElement("div");this.graphContainer.appendChild(e),e.classList.add("ig-block","ig-bg-white");for(let o of t.mir.attributes)e.classList.add(`ig-block-att-${o}`);e.setAttribute("data-ig-block-ptr",`${t.ptr}`),e.setAttribute("data-ig-block-id",`${t.id}`);let s="";t.mir.attributes.includes("loopheader")?s=" (loop header)":t.mir.attributes.includes("backedge")?s=" (backedge)":t.mir.attributes.includes("splitedge")&&(s=" (split edge)");let i=document.createElement("div");i.classList.add("ig-block-header"),i.innerText=`Block ${t.id}${s}`,e.appendChild(i);let c=document.createElement("div");c.classList.add("ig-instructions"),e.appendChild(c);let l=document.createElement("table");if(t.lir){l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + ${this.sampleCounts?` + <col style="width: 1px"> + <col style="width: 1px"> + `:""} + </colgroup> + ${this.sampleCounts?` + <thead> + <tr> + <th></th> + <th></th> + <th class="ig-f6">Total</th> + <th class="ig-f6">Self</th> + </tr> + </thead> + `:""} + `;for(let o of t.lir.instructions)l.appendChild(this.renderLIRInstruction(o))}else{l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + <col style="width: 1px"> + </colgroup> + `;for(let o of t.mir.instructions)l.appendChild(this.renderMIRInstruction(o))}if(c.appendChild(l),t.succs.length===2)for(let[o,n]of[1,0].entries()){let r=document.createElement("div");r.innerText=`${n}`,r.classList.add("ig-edge-label"),r.style.left=`${I+S*o}px`,e.appendChild(r)}return i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",o=>{o.stopPropagation(),o.shiftKey||this.selectedBlockPtrs.clear(),this.setSelection([],t.ptr)}),e}render(t,e,s){for(let n of t)for(let r of n)if(r.block!==null){let d=r.block;d.el.style.left=`${r.pos.x}px`,d.el.style.top=`${r.pos.y}px`}let i=0,c=0;for(let n of t)for(let r of n)i=Math.max(i,r.pos.x+r.size.x+P),c=Math.max(c,r.pos.y+r.size.y+P);let l=document.createElementNS("http://www.w3.org/2000/svg","svg");this.graphContainer.appendChild(l);let o=(n,r)=>{for(let d of n)i=Math.max(i,d+P);for(let d of r)c=Math.max(c,d+P)};for(let n=0;n<t.length;n++){let r=t[n];for(let d of r){d.block||k(d.dstNodes.length===1,`dummy nodes must have exactly one destination, but dummy ${d.id} had ${d.dstNodes.length}`),k(d.dstNodes.length===d.jointOffsets.length,"must have a joint offset for each destination");for(let[b,g]of d.dstNodes.entries()){let u=d.pos.x+I+S*b,h=d.pos.y+d.size.y;if(d.block?.mir.attributes.includes("backedge")){let p=d.block.succs[0],m=d.pos.x,f=d.pos.y+D,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=jt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(d.flags&U){let p=L(g.block),m=d.pos.x+I,f=d.pos.y+D+T,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=zt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(g.block===null&&g.dstBlock.mir.attributes.includes("backedge")){let p=g.pos.x+I,m=g.pos.y+(g.flags&U?D+T:0);if(d.block===null){let f=h-H,y=Ft(u,h,p,m,f,!1);l.appendChild(y),o([u,p],[h,m,f])}else{let f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=_t(u,h,p,m,f);l.appendChild(y),o([u,p],[h,m,f])}}else{let p=g.pos.x+I,m=g.pos.y,f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=Rt(u,h,p,m,f,g.block!==null);l.appendChild(y),o([u,p],[h,m,f])}}}}if(l.setAttribute("width",`${i}`),l.setAttribute("height",`${c}`),this.size={x:i,y:c},+dt)for(let n of t)for(let r of n){let d=document.createElement("div");d.innerHTML=`${r.id}<br><- ${r.srcNodes.map(b=>b.id)}<br>-> ${r.dstNodes.map(b=>b.id)}<br>${r.flags}`,d.style.position="absolute",d.style.border="1px solid black",d.style.backgroundColor="white",d.style.left=`${r.pos.x}px`,d.style.top=`${r.pos.y}px`,d.style.whiteSpace="nowrap",this.graphContainer.appendChild(d)}this.updateHighlightedInstructions(),this.updateHotness()}renderMIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-mir","ig-can-flash",...t.attributes.map(o=>`ig-ins-att-${o}`)),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=`v${t.id}`,s.appendChild(i);let c=document.createElement("td");c.innerHTML=e.replace(/(v)(\d+)/g,(o,n,r)=>`<span class="ig-use ig-highlightable" data-ig-use="${r}">${n}${r}</span>`),s.appendChild(c);let l=document.createElement("td");return l.classList.add("ig-ins-type"),l.innerText=t.type==="None"?"":t.type,s.appendChild(l),i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),c.querySelectorAll(".ig-use").forEach(o=>{o.addEventListener("pointerdown",n=>{n.preventDefault(),n.stopPropagation()}),o.addEventListener("click",n=>{let r=parseInt(L(o.getAttribute("data-ig-use")),10);this.jumpToInstruction(r,{zoom:1})})}),s}renderLIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-lir","ig-hotness"),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=String(t.id),s.appendChild(i);let c=document.createElement("td");if(c.innerText=e,s.appendChild(c),this.sampleCounts){let l=this.sampleCounts?.totalLineHits.get(t.id)??0,o=this.sampleCounts?.selfLineHits.get(t.id)??0,n=document.createElement("td");n.classList.add("ig-ins-samples"),n.classList.toggle("ig-text-dim",l===0),n.innerText=`${l}`,n.title="Color by total count",s.appendChild(n);let r=document.createElement("td");r.classList.add("ig-ins-samples"),r.classList.toggle("ig-text-dim",o===0),r.innerText=`${o}`,r.title="Color by self count",s.appendChild(r);for(let[d,b]of[n,r].entries())b.addEventListener("pointerdown",g=>{g.preventDefault(),g.stopPropagation()}),b.addEventListener("click",()=>{k(d===O||d===W),this.heatmapMode=d,this.updateHotness()})}return i.addEventListener("pointerdown",l=>{l.preventDefault(),l.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),s}renderSelection(){this.graphContainer.querySelectorAll(".ig-block").forEach(t=>{let e=parseInt(L(t.getAttribute("data-ig-block-ptr")),10);t.classList.toggle("ig-selected",this.selectedBlockPtrs.has(e)),t.classList.toggle("ig-last-selected",this.lastSelectedBlockPtr===e)})}removeNonexistentHighlights(){this.highlightedInstructions=this.highlightedInstructions.filter(t=>this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`))}updateHighlightedInstructions(){for(let t of this.highlightedInstructions)k(this.highlightedInstructions.filter(e=>e.ptr===t.ptr).length===1,`instruction ${t.ptr} was highlighted more than once`);this.graphContainer.querySelectorAll(".ig-ins, .ig-use").forEach(t=>{Vt(t)});for(let t of this.highlightedInstructions){let e=this.instructionPalette[t.paletteColor%this.instructionPalette.length],s=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`);if(s){ct(s,e);let i=this.insIDsByPtr.get(t.ptr);this.graphContainer.querySelectorAll(`.ig-use[data-ig-use="${i}"]`).forEach(c=>{ct(c,e)})}}}updateHotness(){this.graphContainer.querySelectorAll(".ig-ins-lir").forEach(t=>{k(t.classList.contains("ig-hotness"));let e=parseInt(L(t.getAttribute("data-ig-ins-id")),10),s=0;this.sampleCounts&&(s=((this.heatmapMode===O?this.sampleCounts.totalLineHits:this.sampleCounts.selfLineHits).get(e)??0)/this.maxSampleCounts[this.heatmapMode]),t.style.setProperty("--ig-hotness",`${s}`)})}addEventListeners(){this.viewport.addEventListener("wheel",e=>{e.preventDefault();let s=this.zoom;if(e.ctrlKey){s=Math.max(Ht,Math.min(Ct,this.zoom*Math.pow(Mt,-e.deltaY*Et)));let c=s/this.zoom-1;this.zoom=s;let{x:l,y:o}=this.viewport.getBoundingClientRect(),n=e.clientX-l-this.translation.x,r=e.clientY-o-this.translation.y;this.translation.x-=n*c,this.translation.y-=r*c}else this.translation.x-=e.deltaX,this.translation.y-=e.deltaY;let i=this.clampTranslation(this.translation,s);this.translation.x=i.x,this.translation.y=i.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerdown",e=>{e.pointerType==="mouse"&&!(e.button===0||e.button===1)||(e.preventDefault(),this.viewport.setPointerCapture(e.pointerId),this.startMousePos={x:e.clientX,y:e.clientY},this.lastMousePos={x:e.clientX,y:e.clientY},this.animating=!1)}),this.viewport.addEventListener("pointermove",e=>{if(!this.viewport.hasPointerCapture(e.pointerId))return;let s=e.clientX-this.lastMousePos.x,i=e.clientY-this.lastMousePos.y;this.translation.x+=s,this.translation.y+=i,this.lastMousePos={x:e.clientX,y:e.clientY};let c=this.clampTranslation(this.translation,this.zoom);this.translation.x=c.x,this.translation.y=c.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerup",e=>{this.viewport.releasePointerCapture(e.pointerId);let s=2,i=this.startMousePos.x-e.clientX,c=this.startMousePos.y-e.clientY;Math.abs(i)<=s&&Math.abs(c)<=s&&this.setSelection([]),this.animating=!1}),new ResizeObserver(e=>{k(e.length===1);let s=e[0].contentRect;this.viewportSize.x=s.width,this.viewportSize.y=s.height}).observe(this.viewport)}setSelection(t,e=0){this.setSelectionRaw(t,e),e?this.nav={visited:[e],currentIndex:0,siblings:[e]}:this.nav={visited:[],currentIndex:-1,siblings:[]}}setSelectionRaw(t,e){this.selectedBlockPtrs.clear();for(let s of[...t,e])this.blocksByPtr.has(s)&&this.selectedBlockPtrs.add(s);this.lastSelectedBlockPtr=this.blocksByPtr.has(e)?e:0,this.renderSelection()}navigate(t){let e=this.lastSelectedBlockPtr;if(t==="down"||t==="up")if(e){let s=L(this.blocksByPtr.get(e)),i=(t==="down"?s.succs:s.preds).map(l=>l.ptr);s.ptr!==this.nav.visited[this.nav.currentIndex]&&(this.nav.visited=[s.ptr],this.nav.currentIndex=0);let c=this.nav.currentIndex+(t==="down"?1:-1);if(0<=c&&c<this.nav.visited.length)this.nav.currentIndex=c,this.nav.siblings=i;else{let l=i[0];l!==void 0&&(t==="down"?(this.nav.visited.push(l),this.nav.currentIndex+=1,k(this.nav.currentIndex===this.nav.visited.length-1)):(this.nav.visited.unshift(l),k(this.nav.currentIndex===0)),this.nav.siblings=i)}this.setSelectionRaw([],this.nav.visited[this.nav.currentIndex])}else{let s=[...this.blocks].sort((n,r)=>n.id-r.id),i=s.filter(n=>n.preds.length===0),c=s.filter(n=>n.succs.length===0),l=t==="down"?i:c,o=l[0];k(o),this.setSelectionRaw([],o.ptr),this.nav={visited:[o.ptr],currentIndex:0,siblings:l.map(n=>n.ptr)}}else if(e!==void 0){let s=this.nav.siblings.indexOf(e);k(s>=0,"currently selected node should be in siblings array");let i=s+(t==="right"?1:-1);0<=i&&i<this.nav.siblings.length&&this.setSelectionRaw([],this.nav.siblings[i])}k(this.nav.visited.length===0||this.nav.siblings.includes(this.nav.visited[this.nav.currentIndex]),"expected currently visited node to be in the siblings array"),k(this.lastSelectedBlockPtr===0||this.nav.siblings.includes(this.lastSelectedBlockPtr),"expected currently selected block to be in siblings array")}toggleInstructionHighlight(t,e){this.removeNonexistentHighlights();let s=this.highlightedInstructions.findIndex(c=>c.ptr===t),i=s>=0;if(e!==void 0&&(i=!e),i)s>=0&&this.highlightedInstructions.splice(s,1);else if(s<0){let c=0;for(;;){if(this.highlightedInstructions.find(l=>l.paletteColor===c)){c+=1;continue}break}this.highlightedInstructions.push({ptr:t,paletteColor:c})}this.updateHighlightedInstructions()}clampTranslation(t,e){let s=z-this.size.x*e,i=this.viewportSize.x-z,c=z-this.size.y*e,l=this.viewportSize.y-z,o=q(t.x,s,i),n=q(t.y,c,l);return{x:o,y:n}}updatePanAndZoom(){let t=this.clampTranslation(this.translation,this.zoom);this.graphContainer.style.transform=`translate(${t.x}px, ${t.y}px) scale(${this.zoom})`}graph2viewport(t,e=this.translation,s=this.zoom){return{x:t.x*s+e.x,y:t.y*s+e.y}}viewport2graph(t,e=this.translation,s=this.zoom){return{x:(t.x-e.x)/s,y:(t.y-e.y)/s}}async goToGraphCoordinates(t,{zoom:e=this.zoom,animate:s=!0}){let i={x:-t.x*e,y:-t.y*e};if(!s){this.animating=!1,this.translation.x=i.x,this.translation.y=i.y,this.zoom=e,this.updatePanAndZoom(),await new Promise(l=>setTimeout(l,0));return}if(this.targetTranslation=i,this.targetZoom=e,this.animating)return;this.animating=!0;let c=performance.now();for(;this.animating;){let l=await new Promise(h=>requestAnimationFrame(h)),o=(l-c)/1e3;c=l;let n=1,r=.01,d=1e-6,b=this.targetTranslation.x-this.translation.x,g=this.targetTranslation.y-this.translation.y,u=this.targetZoom-this.zoom;if(this.translation.x=R(this.translation.x,this.targetTranslation.x,d,o),this.translation.y=R(this.translation.y,this.targetTranslation.y,d,o),this.zoom=R(this.zoom,this.targetZoom,d,o),this.updatePanAndZoom(),Math.abs(b)<=n&&Math.abs(g)<=n&&Math.abs(u)<=r){this.translation.x=this.targetTranslation.x,this.translation.y=this.targetTranslation.y,this.zoom=this.targetZoom,this.animating=!1,this.updatePanAndZoom();break}}await new Promise(l=>setTimeout(l,0))}jumpToBlock(t,{zoom:e=this.zoom,animate:s=!0,viewportPos:i}={}){let c=this.blocksByPtr.get(t);if(!c)return Promise.resolve();let l;return i?l={x:c.layoutNode.pos.x-i.x/e,y:c.layoutNode.pos.y-i.y/e}:l=this.graphPosToCenterRect(c.layoutNode.pos,c.layoutNode.size,e),this.goToGraphCoordinates(l,{zoom:e,animate:s})}async jumpToInstruction(t,{zoom:e=this.zoom,animate:s=!0}){let i=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-id="${t}"]`);if(!i)return;let c=i.getBoundingClientRect(),l=this.graphContainer.getBoundingClientRect(),o=(c.x-l.x)/this.zoom,n=(c.y-l.y)/this.zoom,r=c.width/this.zoom,d=c.height/this.zoom,b=this.graphPosToCenterRect({x:o,y:n},{x:r,y:d},e);i.classList.add("ig-flash"),await this.goToGraphCoordinates(b,{zoom:e,animate:s}),i.classList.remove("ig-flash")}graphPosToCenterRect(t,e,s){let i=this.viewportSize.x/s,c=this.viewportSize.y/s,l=Math.max(20/s,(i-e.x)/2),o=Math.max(20/s,(c-e.y)/2),n=t.x-l,r=t.y-o;return{x:n,y:r}}exportState(){let t={translation:this.translation,zoom:this.zoom,heatmapMode:this.heatmapMode,highlightedInstructions:this.highlightedInstructions,selectedBlockPtrs:this.selectedBlockPtrs,lastSelectedBlockPtr:this.lastSelectedBlockPtr,viewportPosOfSelectedBlock:void 0};return this.lastSelectedBlockPtr&&(t.viewportPosOfSelectedBlock=this.graph2viewport(L(this.blocksByPtr.get(this.lastSelectedBlockPtr)).layoutNode.pos)),t}restoreState(t,e){this.translation.x=t.translation.x,this.translation.y=t.translation.y,this.zoom=t.zoom,this.heatmapMode=t.heatmapMode,this.highlightedInstructions=t.highlightedInstructions,this.setSelection(Array.from(t.selectedBlockPtrs),t.lastSelectedBlockPtr),this.updatePanAndZoom(),this.updateHotness(),this.updateHighlightedInstructions(),e.preserveSelectedBlockPosition&&this.jumpToBlock(this.lastSelectedBlockPtr,{zoom:this.zoom,animate:!1,viewportPos:t.viewportPosOfSelectedBlock})}};function Ot(a){for(let t of a.dstNodes){let e=t.srcNodes.indexOf(a);k(e!==-1),t.srcNodes.splice(e,1)}}function*Q(a){for(let t of a)for(let e of t)e.block===null&&(yield e)}function*At(a){for(let t of a)for(let e of t)e.block===null&&e.dstBlock.mir.attributes.includes("backedge")&&(yield e)}function Rt(a,t,e,s,i,c,l=1){let o=T;k(t+o<=i&&i<s-o,`downward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i-o} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${e} ${i+o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,180);r.appendChild(b)}return r}function Ft(a,t,e,s,i,c,l=1){let o=T;k(s+o<=i&&i<=t-o,`upward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i+o} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${e} ${i-o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,0);r.appendChild(b)}return r}function zt(a,t,e,s,i=1){let c=T;k(t-c>=s&&a-c>=e,`to backedge: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, r = ${c} `,!0),i%2===1&&(a+=.5,s+=.5);let l="";l+=`M ${a} ${t} `,l+=`A ${c} ${c} 0 0 0 ${a-c} ${s} `,l+=`L ${e} ${s} `;let o=document.createElementNS("http://www.w3.org/2000/svg","g"),n=document.createElementNS("http://www.w3.org/2000/svg","path");n.setAttribute("d",l),n.setAttribute("fill","none"),n.setAttribute("stroke","black"),n.setAttribute("stroke-width",`${i} `),o.appendChild(n);let r=G(e,s,270);return o.appendChild(r),o}function _t(a,t,e,s,i,c=1){let l=T;k(t+l<=i&&a<=e&&s<=t,`block to backedge dummy: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${l} `,!0),c%2===1&&(a+=.5,e+=.5,i+=.5);let o="";o+=`M ${a} ${t} `,o+=`L ${a} ${i-l} `,o+=`A ${l} ${l} 0 0 0 ${a+l} ${i} `,o+=`L ${e-l} ${i} `,o+=`A ${l} ${l} 0 0 0 ${e} ${i-l} `,o+=`L ${e} ${s} `;let n=document.createElementNS("http://www.w3.org/2000/svg","g"),r=document.createElementNS("http://www.w3.org/2000/svg","path");return r.setAttribute("d",o),r.setAttribute("fill","none"),r.setAttribute("stroke","black"),r.setAttribute("stroke-width",`${c} `),n.appendChild(r),n}function jt(a,t,e,s,i=1){k(e<a&&s===t,`x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s} `,!0),i%2===1&&(t+=.5,s+=.5);let c="";c+=`M ${a} ${t} `,c+=`L ${e} ${s} `;let l=document.createElementNS("http://www.w3.org/2000/svg","g"),o=document.createElementNS("http://www.w3.org/2000/svg","path");o.setAttribute("d",c),o.setAttribute("fill","none"),o.setAttribute("stroke","black"),o.setAttribute("stroke-width",`${i} `),l.appendChild(o);let n=G(e,s,270);return l.appendChild(n),l}function G(a,t,e,s=5){let i=document.createElementNS("http://www.w3.org/2000/svg","path");return i.setAttribute("d",`M 0 0 L ${-s} ${s*1.5} L ${s} ${s*1.5} Z`),i.setAttribute("transform",`translate(${a}, ${t}) rotate(${e})`),i}function ct(a,t){a.classList.add("ig-highlight"),a.style.setProperty("--ig-highlight-color",t)}function Vt(a){a.classList.remove("ig-highlight"),a.style.setProperty("--ig-highlight-color","transparent")}var A=class{constructor(t,{func:e,pass:s=0,sampleCounts:i}){this.graph=null,this.func=e,this.passNumber=s,this.sampleCounts=i,this.keyPasses=[null,null,null,null];{let c=null;for(let[l,o]of e.passes.entries())o.mir.blocks.length>0&&(this.keyPasses[0]===null&&(this.keyPasses[0]=l),o.lir.blocks.length===0&&(this.keyPasses[1]=l)),o.lir.blocks.length>0&&(c?.lir.blocks.length===0&&(this.keyPasses[2]=l),this.keyPasses[3]=l),c=o}this.redundantPasses=[];{let c=null;for(let[l,o]of e.passes.entries()){if(c===null){c=o;continue}E(c.mir,o.mir)&&E(c.lir,o.lir)&&this.redundantPasses.push(l),c=o}}this.viewport=x("div",["ig-flex-grow-1","ig-overflow-hidden"],c=>{c.style.position="relative"}),this.sidebarLinks=e.passes.map((c,l)=>x("a",["ig-link-normal","ig-pv1","ig-ph2","ig-flex","ig-g2"],o=>{o.href="#",o.addEventListener("click",n=>{n.preventDefault(),this.switchPass(l)})},[x("div",["ig-w1","ig-tr","ig-f6","ig-text-dim"],o=>{o.style.paddingTop="0.08rem"},[`${l}`]),x("div",[this.redundantPasses.includes(l)&&"ig-text-dim"],()=>{},[c.name])])),this.container=x("div",["ig-absolute","ig-absolute-fill","ig-flex"],()=>{},[x("div",["ig-w5","ig-br","ig-flex-shrink-0","ig-overflow-y-auto","ig-bg-white"],()=>{},[...this.sidebarLinks]),this.viewport]),t.appendChild(this.container),this.keydownHandler=this.keydownHandler.bind(this),this.tweakHandler=this.tweakHandler.bind(this),window.addEventListener("keydown",this.keydownHandler),window.addEventListener("tweak",this.tweakHandler),this.update()}destroy(){this.container.remove(),window.removeEventListener("keydown",this.keydownHandler),window.removeEventListener("tweak",this.tweakHandler)}update(){for(let[s,i]of this.sidebarLinks.entries())i.classList.toggle("ig-bg-primary",this.passNumber===s);let t=this.graph?.exportState();this.viewport.innerHTML="",this.graph=null;let e=this.func.passes[this.passNumber];if(e)try{this.graph=new V(this.viewport,e,{sampleCounts:this.sampleCounts}),t&&this.graph.restoreState(t,{preserveSelectedBlockPosition:!0})}catch(s){this.viewport.innerHTML="An error occurred while laying out the graph. See console.",console.error(s)}}switchPass(t){this.passNumber=t,this.update()}keydownHandler(t){switch(t.key){case"w":case"s":this.graph?.navigate(t.key==="s"?"down":"up"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"a":case"d":this.graph?.navigate(t.key==="d"?"right":"left"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"f":for(let e=this.passNumber+1;e<this.func.passes.length;e++)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"r":for(let e=this.passNumber-1;e>=0;e--)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"1":case"2":case"3":case"4":{let e=["1","2","3","4"].indexOf(t.key),s=this.keyPasses[e];typeof s=="number"&&this.switchPass(s)}break;case"c":{let e=this.graph?.blocksByPtr.get(this.graph?.lastSelectedBlockPtr??-1);e&&this.graph?.jumpToBlock(e.ptr,{zoom:1})}break}}tweakHandler(){this.update()}};var M=new URL(window.location.toString()).searchParams,Gt=M.has("func")?parseInt(M.get("func"),10):void 0,ut=M.has("pass")?parseInt(M.get("pass"),10):void 0,Y=class{constructor(t){this.exportButton=null,this.ionjson=null,this.funcIndex=Gt??0,this.funcSelected=t.funcSelected,this.funcSelector=x("div",[],()=>{},["Function",x("input",["ig-w3"],e=>{e.type="number",e.min="1",e.addEventListener("input",()=>{this.switchFunc(parseInt(e.value,10)-1)})},[])," / ",x("span",["num-functions"])]),this.funcSelectorNone=x("div",[],()=>{},["No functions to display."]),this.funcName=x("div"),this.root=x("div",["ig-bb","ig-flex","ig-bg-white"],()=>{},[x("div",["ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center","ig-br","ig-hide-if-empty"],()=>{},[t.browse&&x("div",[],()=>{},[x("input",[],e=>{e.type="file",e.addEventListener("change",s=>{let i=s.target;i.files?.length&&this.fileSelected(i.files[0])})})]),this.funcSelector,this.funcSelectorNone]),x("div",["ig-flex-grow-1","ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center"],()=>{},[this.funcName,x("div",["ig-flex-grow-1"]),t.export&&x("div",[],()=>{},[x("button",[],e=>{this.exportButton=e,e.addEventListener("click",()=>{this.exportStandalone()})},["Export"])])])]),this.update()}async fileSelected(t){let e=JSON.parse(await t.text());this.ionjson=X(e),this.switchFunc(0),this.update()}switchIonJSON(t){this.ionjson=t,this.switchFunc(this.funcIndex)}switchFunc(t){t=Math.max(0,Math.min(this.numFunctions()-1,t)),this.funcIndex=isNaN(t)?0:t,this.funcSelected(this.ionjson?.functions[this.funcIndex]??null),this.update()}numFunctions(){return this.ionjson?.functions.length??0}update(){let t=0<=this.funcIndex&&this.funcIndex<this.numFunctions();this.funcSelector.hidden=this.numFunctions()<=1,this.funcSelectorNone.hidden=!(this.ionjson&&this.numFunctions()===0);let e=this.funcSelector.querySelector("input");e.max=`${this.numFunctions()}`,e.value=`${this.funcIndex+1}`,this.funcSelector.querySelector(".num-functions").innerHTML=`${this.numFunctions()}`,this.funcName.hidden=!t,this.funcName.innerText=`${this.ionjson?.functions[this.funcIndex].name??""}`,this.exportButton&&(this.exportButton.disabled=!this.ionjson||!t)}async exportStandalone(){let t=L(this.ionjson),e=t.functions[this.funcIndex].name,s={version:1,functions:[t.functions[this.funcIndex]]},c=(await(await fetch("./standalone.html")).text()).replace(/\{\{\s*IONJSON\s*\}\}/,JSON.stringify(s)),l=URL.createObjectURL(new Blob([c],{type:"text/html;charset=utf-8"})),o=document.createElement("a");o.href=l,o.download=`iongraph-${e}.html`,document.body.appendChild(o),o.click(),o.remove(),URL.revokeObjectURL(l)}},tt=class{constructor(){this.menuBar=new Y({browse:!0,export:!0,funcSelected:t=>this.switchFunc(t)}),this.func=null,this.sampleCountsFromFile=void 0,this.graph=null,this.loadStuffFromQueryParams(),this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer]),this.update()}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut,sampleCounts:this.sampleCountsFromFile}))}loadStuffFromQueryParams(){(async()=>{let t=M.get("file");if(t){let s=await(await fetch(t)).json(),i=X(s);this.menuBar.switchIonJSON(i)}})(),(async()=>{let t=M.get("sampleCounts");if(t){let s=await(await fetch(t)).json();this.sampleCountsFromFile={selfLineHits:new Map(s.selfLineHits),totalLineHits:new Map(s.totalLineHits)},this.update()}})()}switchFunc(t){this.func=t,this.update()}},et=class{constructor(){this.menuBar=new Y({funcSelected:t=>this.switchFunc(t)}),this.func=null,this.graph=null,this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer])}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut}))}setIonJSON(t){this.menuBar.switchIonJSON(t)}switchFunc(t){this.func=t,this.update()}};return yt(Yt);})(); +</script> + <script>window.__exportedIonJSON = {{ IONJSON }}</script> + <script> + const ui = new iongraph.StandaloneUI(); + document.body.appendChild(ui.root); + ui.setIonJSON(window.__exportedIonJSON); + </script> +</body> diff --git a/tool/zjit_iongraph.rb b/tool/zjit_iongraph.rb new file mode 100755 index 0000000000..0cb7701614 --- /dev/null +++ b/tool/zjit_iongraph.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +require 'json' +require 'logger' + +LOGGER = Logger.new($stderr) + +def run_ruby *cmd + # Find the first --zjit* option and add --zjit-dump-hir-iongraph after it + zjit_index = cmd.find_index { |arg| arg.start_with?("--zjit") } + raise "No --zjit option found in command" unless zjit_index + cmd.insert(zjit_index + 1, "--zjit-dump-hir-iongraph") + pid = Process.spawn(*cmd) + _, status = Process.wait2(pid) + if status.exitstatus != 0 + LOGGER.warn("Command failed with exit status #{status.exitstatus}") + end + pid +end + +usage = "Usage: zjit_iongraph.rb <path_to_ruby> <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +pid = run_ruby(RUBY, *OPTIONS) +functions = Dir["/tmp/zjit-iongraph-#{pid}/fun*.json"].map do |path| + JSON.parse(File.read(path)) +end + +if functions.empty? + LOGGER.warn("No iongraph functions found for PID #{pid}") +end + +json = JSON.dump({version: 1, functions: functions}) +# Get zjit_iongraph.html from the sibling file next to this script +html = File.read(File.join(File.dirname(__FILE__), "zjit_iongraph.html")) +html.sub!("{{ IONJSON }}", json) +output_path = "zjit_iongraph_#{pid}.html" +File.write(output_path, html) +puts "Wrote iongraph to #{output_path}" |