diff options
Diffstat (limited to 'tool')
338 files changed, 28769 insertions, 8226 deletions
diff --git a/tool/annocheck/Dockerfile b/tool/annocheck/Dockerfile new file mode 100644 index 0000000000..d1fb1839c9 --- /dev/null +++ b/tool/annocheck/Dockerfile @@ -0,0 +1,4 @@ +FROM ghcr.io/ruby/fedora:latest + +RUN dnf -y install annobin-annocheck +WORKDIR /work diff --git a/tool/annocheck/Dockerfile-copy b/tool/annocheck/Dockerfile-copy new file mode 100644 index 0000000000..d437f27387 --- /dev/null +++ b/tool/annocheck/Dockerfile-copy @@ -0,0 +1,6 @@ +FROM ghcr.io/ruby/fedora:latest +ARG IN_DIR + +RUN dnf -y install annobin-annocheck +COPY ${IN_DIR} /work +WORKDIR /work diff --git a/tool/auto-style.rb b/tool/auto-style.rb new file mode 100755 index 0000000000..3b93c8c317 --- /dev/null +++ b/tool/auto-style.rb @@ -0,0 +1,284 @@ +#!/usr/bin/env ruby +# Usage: +# auto-style.rb oldrev newrev [pushref] + +require 'shellwords' +require 'tmpdir' +ENV['LC_ALL'] = 'C' + +class Git + attr_reader :depth + + def initialize(oldrev, newrev, branch = nil) + @oldrev = oldrev + @newrev = !newrev || newrev.empty? ? 'HEAD' : newrev + @branch = branch + + return unless oldrev + + # GitHub may not fetch github.event.pull_request.base.sha at checkout + git('log', '--format=%H', '-1', @oldrev, out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=1', 'origin', @oldrev) + git('log', '--format=%H', '-1', "#@newrev~99", out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=100', 'origin', @newrev) + + with_clean_env do + @revs = {} + IO.popen(['git', 'log', '--format=%H %s', "#{@oldrev}..#{@newrev}"]) do |f| + f.each do |line| + line.chomp! + rev, subj = line.split(' ', 2) + @revs[rev] = subj + end + end + @depth = @revs.size + end + end + + # ["foo/bar.c", "baz.h", ...] + def updated_paths + with_clean_env do + IO.popen(['git', 'diff', '--name-only', @oldrev, @newrev], &:readlines).each(&:chomp!) + end + end + + # [0, 1, 4, ...] + def updated_lines(file) # NOTE: This doesn't work well on pull requests, so not used anymore + lines = [] + revs = @revs.map {|rev, subj| rev unless subj.start_with?("Revert ")}.compact + revs_pattern = /\A(?:#{revs.join('|')}) / + with_clean_env { IO.popen(['git', 'blame', '-l', '--', file], &:readlines) }.each_with_index do |line, index| + if revs_pattern =~ line + lines << index + end + end + lines + end + + def commit(log, *files) + git('add', *files) + git('commit', '-m', log) + end + + def push + git('push', 'origin', @branch) if @branch + end + + def diff + git('--no-pager', 'diff') + end + + private + + def git(*args, **opts) + cmd = ['git', *args] + puts "+ #{cmd.shelljoin}" + ret = with_clean_env { system(*cmd, **opts) } + unless ret or opts[:err] + abort "Failed to run: #{cmd}" + end + ret + end + + def with_clean_env + git_dir = ENV.delete('GIT_DIR') # this overcomes '-C' or pwd + yield + ensure + ENV['GIT_DIR'] = git_dir if git_dir + end +end + +DEFAULT_GEM_LIBS = %w[ + bundler + cmath + csv + e2mmap + fileutils + forwardable + ipaddr + irb + logger + matrix + mutex_m + ostruct + prime + rdoc + rexml + rss + scanf + shell + sync + thwait + tracer + webrick +] + +DEFAULT_GEM_EXTS = %w[ + bigdecimal + date + dbm + digest + etc + fcntl + fiddle + gdbm + io/console + io/nonblock + json + openssl + psych + racc + sdbm + stringio + strscan + zlib +] + +IGNORED_FILES = [ + # default gems whose master is GitHub + %r{\Abin/(?!erb)\w+\z}, + *(DEFAULT_GEM_LIBS + DEFAULT_GEM_EXTS).flat_map { |lib| + [ + %r{\Alib/#{lib}/}, + %r{\Alib/#{lib}\.gemspec\z}, + %r{\Alib/#{lib}\.rb\z}, + %r{\Atest/#{lib}/}, + ] + }, + *DEFAULT_GEM_EXTS.flat_map { |ext| + [ + %r{\Aext/#{ext}/}, + %r{\Atest/#{ext}/}, + ] + }, + + # vendoring (ccan) + %r{\Accan/}, + + # vendoring (io/) + %r{\Aext/io/}, + + # vendoring (nkf) + %r{\Aext/nkf/nkf-utf8/}, + + # vendoring (onigmo) + %r{\Aenc/}, + %r{\Ainclude/ruby/onigmo\.h\z}, + %r{\Areg.+\.(c|h)\z}, + + # explicit or implicit `c-file-style: "linux"` + %r{\Aaddr2line\.c\z}, + %r{\Amissing/}, + %r{\Astrftime\.c\z}, + %r{\Avsnprintf\.c\z}, + + # to respect the original statements of licenses + %r{\ALEGAL\z}, + + # trailing spaces could be intentional in TRICK code + %r{\Asample/trick[^/]*/}, +] + +DIFFERENT_STYLE_FILES = %w[ + addr2line.c io_buffer.c prism*.c scheduler.c +] + +def adjust_styles(files) + trailing = eofnewline = expandtab = indent = false + + edited_files = files.select do |f| + src = File.binread(f) rescue next + eofnewline = eofnewline0 = true if src.sub!(/(?<!\A|\n)\z/, "\n") + + trailing0 = false + expandtab0 = false + indent0 = false + + src.gsub!(/^.*$/).with_index do |line, lineno| + trailing = trailing0 = true if line.sub!(/[ \t]+$/, '') + line + end + + if f.end_with?('.c') || f.end_with?('.h') || f == 'insns.def' + # If and only if unedited lines did not have tab indentation, prevent introducing tab indentation to the file. + expandtab_allowed = src.each_line.with_index.all? do |line, lineno| + !line.start_with?("\t") + end + + if expandtab_allowed + src.gsub!(/^.*$/).with_index do |line, lineno| + if line.start_with?("\t") # last-committed line with hard tabs + expandtab = expandtab0 = true + line.sub(/\A\t+/) { |tabs| ' ' * (8 * tabs.length) } + else + line + end + end + end + end + + if File.fnmatch?("*.[chy]", f, File::FNM_PATHNAME) && + !DIFFERENT_STYLE_FILES.any? {|pat| File.fnmatch?(pat, f, File::FNM_PATHNAME)} + indent0 = true if src.gsub!(/^\w+\([^\n]*?\)\K[ \t]*(?=\{( *\\)?$)/, '\1' "\n") + indent0 = true if src.gsub!(/^([ \t]*)\}\K[ \t]*(?=else\b.*?( *\\)?$)/, '\2' "\n" '\1') + indent0 = true if src.gsub!(/^[ \t]*\}\n\K\n+(?=[ \t]*else\b)/, '') + indent ||= indent0 + end + + if trailing0 or eofnewline0 or expandtab0 or indent0 + File.binwrite(f, src) + true + end + end + if edited_files.empty? + return + else + msg = [('remove trailing spaces' if trailing), + ('append newline at EOF' if eofnewline), + ('expand tabs' if expandtab), + ('adjust indents' if indent), + ].compact + message = "* #{msg.join(', ')}. [ci skip]" + if expandtab + message += "\nPlease consider using misc/expand_tabs.rb as a pre-commit hook." + end + return message, edited_files + end +end + +oldrev, newrev, pushref = ARGV +if (dry_run = oldrev == '-n') or oldrev == '--' + _, *updated_files = ARGV + git = Git.new(nil, nil) +else + unless dry_run = pushref.nil? + branch = IO.popen(['git', 'rev-parse', '--symbolic', '--abbrev-ref', pushref], &:read).strip + end + git = Git.new(oldrev, newrev, branch) + + updated_files = git.updated_paths +end + +files = updated_files.select {|l| + /^\d/ !~ l and /\.bat\z/ !~ l and + (/\A(?:config|[Mm]akefile|GNUmakefile|README)/ =~ File.basename(l) or + /\A\z|\.(?:[chsy]|\d+|e?rb|tmpl|bas[eh]|z?sh|in|ma?k|def|src|trans|rdoc|ja|en|el|sed|awk|p[ly]|scm|mspec|html|rs)\z/ =~ File.extname(l)) +} +files.select! {|n| File.file?(n) } +files.reject! do |f| + IGNORED_FILES.any? { |re| f.match(re) } +end + +if files.empty? + puts "No files are an auto-style target:\n#{updated_files.join("\n")}" +elsif !(message, edited_files = adjust_styles(files)) + puts "All edited lines are formatted well:\n#{files.join("\n")}" +else + if dry_run + git.diff + abort message + else + git.commit(message, *edited_files) + git.push + end +end diff --git a/tool/auto_review_pr.rb b/tool/auto_review_pr.rb new file mode 100755 index 0000000000..07c98c7e0a --- /dev/null +++ b/tool/auto_review_pr.rb @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'uri' +require_relative './sync_default_gems' + +class GitHubAPIClient + def initialize(token) + @token = token + end + + def get(path) + response = Net::HTTP.get_response(URI("https://api.github.com#{path}"), { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end + + def post(path, body = {}) + body = JSON.dump(body) + response = Net::HTTP.post(URI("https://api.github.com#{path}"), body, { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + 'Content-Type' => 'application/json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end +end + +class AutoReviewPR + REPO = 'ruby/ruby' + + COMMENT_USER = 'github-actions[bot]' + COMMENT_PREFIX = 'The following files are maintained in the following upstream repositories:' + COMMENT_SUFFIX = 'Please file a pull request to the above instead. Thank you!' + + def initialize(client) + @client = client + end + + def review(pr_number) + # Fetch the list of files changed by the PR + changed_files = @client.get("/repos/#{REPO}/pulls/#{pr_number}/files").map { it.fetch(:filename) } + + # Build a Hash: { upstream_repo => files, ... } + upstream_repos = SyncDefaultGems::Repository.group(changed_files) + upstream_repos.delete(nil) # exclude no-upstream files + upstream_repos.delete('prism') if changed_files.include?('prism_compile.c') # allow prism changes in this case + if upstream_repos.empty? + puts "Skipped: The PR ##{pr_number} doesn't have upstream repositories." + return + end + + # Check if the PR is already reviewed + existing_comments = @client.get("/repos/#{REPO}/issues/#{pr_number}/comments") + existing_comments.map! { [it.fetch(:user).fetch(:login), it.fetch(:body)] } + if existing_comments.any? { |user, comment| user == COMMENT_USER && comment.start_with?(COMMENT_PREFIX) } + puts "Skipped: The PR ##{pr_number} already has an automated review comment." + return + end + + # Post a comment + comment = format_comment(upstream_repos) + result = @client.post("/repos/#{REPO}/issues/#{pr_number}/comments", { body: comment }) + puts "Success: #{JSON.pretty_generate(result)}" + end + + private + + # upstream_repos: { upstream_repo => files, ... } + def format_comment(upstream_repos) + comment = +'' + comment << "#{COMMENT_PREFIX}\n\n" + + upstream_repos.each do |upstream_repo, files| + comment << "* https://github.com/ruby/#{upstream_repo}\n" + files.each do |file| + comment << " * #{file}\n" + end + end + + comment << "\n#{COMMENT_SUFFIX}" + comment + end +end + +pr_number = ARGV[0] || abort("Usage: #{$0} <pr_number>") +client = GitHubAPIClient.new(ENV.fetch('GITHUB_TOKEN')) + +AutoReviewPR.new(client).review(pr_number) diff --git a/tool/bisect.sh b/tool/bisect.sh index 3b97707142..dfc3a64041 100755 --- a/tool/bisect.sh +++ b/tool/bisect.sh @@ -38,6 +38,18 @@ case $1 in ;; esac +# Apply $(srcdir)/bisect.patch to build if exists +# e.g., needs 5c2508060b~2..5c2508060b to use Bison 3.5.91. +if [ -f bisect.patch ]; then + if ! patch -p1 -N < bisect.patch || git diff --no-patch --exit-code; then + exit 125 + fi + git status + exec= +else + exec=exec +fi + case "$0" in */*) # assume a copy of this script is in builddir @@ -47,4 +59,7 @@ esac for target in srcs Makefile $prep; do $MAKE $target || exit 125 done -exec $MAKE $run +$exec $MAKE $run +status=$? +git checkout -f HEAD +exit $status diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb new file mode 100644 index 0000000000..91ac5628f1 --- /dev/null +++ b/tool/bundler/dev_gems.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "test-unit", "~> 3.0" +gem "test-unit-ruby-core" +gem "rake", "~> 13.1" +gem "rb_sys" + +gem "turbo_tests", "~> 2.2.3" +gem "parallel_tests", "~> 4.10.1" +gem "parallel", "~> 1.19" +gem "rspec-core", "~> 3.12" +gem "rspec-expectations", "~> 3.12" +gem "rspec-mocks", "~> 3.12" +gem "rubygems-generate_index", "~> 1.1" + +group :doc do + gem "ronn-ng", "~> 0.10.1", platform: :ruby +end diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock new file mode 100644 index 0000000000..832127fb4c --- /dev/null +++ b/tool/bundler/dev_gems.rb.lock @@ -0,0 +1,132 @@ +GEM + remote: https://rubygems.org/ + specs: + compact_index (0.15.0) + diff-lcs (1.6.2) + kramdown (2.5.1) + rexml (>= 3.3.9) + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + mini_portile2 (2.8.9) + mustache (1.1.1) + nokogiri (1.19.0) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.19.0-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-java) + racc (~> 1.4) + nokogiri (1.19.0-x64-mingw-ucrt) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-gnu) + racc (~> 1.4) + parallel (1.27.0) + parallel_tests (4.10.1) + parallel + power_assert (3.0.1) + racc (1.8.1) + racc (1.8.1-java) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rexml (3.4.4) + ronn-ng (0.10.1) + kramdown (~> 2, >= 2.1) + kramdown-parser-gfm (~> 1, >= 1.0.1) + mustache (~> 1) + nokogiri (~> 1, >= 1.14.3) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + test-unit (3.7.7) + power_assert + test-unit-ruby-core (1.0.14) + test-unit (>= 3.7.2) + turbo_tests (2.2.5) + parallel_tests (>= 3.3.0, < 5) + rspec (>= 3.10) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm-linux + arm64-darwin + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86-linux + x86_64-darwin + x86_64-linux + +DEPENDENCIES + parallel (~> 1.19) + parallel_tests (~> 4.10.1) + rake (~> 13.1) + rb_sys + ronn-ng (~> 0.10.1) + rspec-core (~> 3.12) + rspec-expectations (~> 3.12) + rspec-mocks (~> 3.12) + rubygems-generate_index (~> 1.1) + test-unit (~> 3.0) + test-unit-ruby-core + turbo_tests (~> 2.2.3) + +CHECKSUMS + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + kramdown (2.5.1) sha256=87bbb6abd9d3cebe4fc1f33e367c392b4500e6f8fa19dd61c0972cf4afe7368c + kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 + mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289 + mustache (1.1.1) sha256=90891fdd50b53919ca334c8c1031eada1215e78d226d5795e523d6123a2717d0 + nokogiri (1.19.0) sha256=e304d21865f62518e04f2bf59f93bd3a97ca7b07e7f03952946d8e1c05f45695 + nokogiri (1.19.0-aarch64-linux-gnu) sha256=11a97ecc3c0e7e5edcf395720b10860ef493b768f6aa80c539573530bc933767 + nokogiri (1.19.0-arm-linux-gnu) sha256=572a259026b2c8b7c161fdb6469fa2d0edd2b61cd599db4bbda93289abefbfe5 + nokogiri (1.19.0-arm64-darwin) sha256=0811dfd936d5f6dd3f6d32ef790568bf29b2b7bead9ba68866847b33c9cf5810 + nokogiri (1.19.0-java) sha256=5f3a70e252be641d8a4099f7fb4cc25c81c632cb594eec9b4b8f2ca8be4374f3 + nokogiri (1.19.0-x64-mingw-ucrt) sha256=05d7ed2d95731edc9bef2811522dc396df3e476ef0d9c76793a9fca81cab056b + nokogiri (1.19.0-x86_64-darwin) sha256=1dad56220b603a8edb9750cd95798bffa2b8dd9dd9aa47f664009ee5b43e3067 + nokogiri (1.19.0-x86_64-linux-gnu) sha256=f482b95c713d60031d48c44ce14562f8d2ce31e3a9e8dd0ccb131e9e5a68b58c + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parallel_tests (4.10.1) sha256=df05458c691462b210f7a41fc2651d4e4e8a881e8190e6d1e122c92c07735d70 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + ronn-ng (0.10.1) sha256=4eeb0185c0fbfa889efed923b5b50e949cd869e7d82ac74138acd0c9c7165ec0 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + test-unit-ruby-core (1.0.14) sha256=d2e997796c9c5c5e8e31ac014f83a473ff5c2523a67cfa491b08893e12d43d22 + turbo_tests (2.2.5) sha256=3fa31497d12976d11ccc298add29107b92bda94a90d8a0a5783f06f05102509f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb new file mode 100644 index 0000000000..a9b6fda11b --- /dev/null +++ b/tool/bundler/rubocop_gems.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rubocop", ">= 1.52.1", "< 2" + +gem "minitest", "~> 5.1" +gem "irb" +gem "rake" +gem "rake-compiler" +gem "rspec" +gem "test-unit" +gem "rb_sys" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock new file mode 100644 index 0000000000..d0c3120e11 --- /dev/null +++ b/tool/bundler/rubocop_gems.rb.lock @@ -0,0 +1,159 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) + ast (~> 2.4.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) + rake + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.82.1) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.48.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + stringio (3.2.0) + test-unit (3.7.7) + power_assert + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + irb + minitest (~> 5.1) + rake + rake-compiler + rb_sys + rspec + rubocop (>= 1.52.1, < 2) + test-unit + +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.82.1) sha256=09f1a6a654a960eda767aebea33e47603080f8e9c9a3f019bf9b94c9cab5e273 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb new file mode 100644 index 0000000000..f7bc34cf5e --- /dev/null +++ b/tool/bundler/standard_gems.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "standard", "~> 1.0" + +gem "minitest", "~> 5.1" +gem "irb" +gem "rake" +gem "rake-compiler" +gem "rspec" +gem "test-unit" +gem "rb_sys" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock new file mode 100644 index 0000000000..f3792f8611 --- /dev/null +++ b/tool/bundler/standard_gems.rb.lock @@ -0,0 +1,179 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) + ast (~> 2.4.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) + rake + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.81.7) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + rubocop-performance (1.26.1) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (1.13.0) + standard (1.52.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.81.7) + standard-custom (~> 1.0.0) + standard-performance (~> 1.8) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.9.0) + lint_roller (~> 1.1) + rubocop-performance (~> 1.26.0) + stringio (3.2.0) + test-unit (3.7.7) + power_assert + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + irb + minitest (~> 5.1) + rake + rake-compiler + rb_sys + rspec + standard (~> 1.0) + test-unit + +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.81.7) sha256=6fb5cc298c731691e2a414fe0041a13eb1beed7bab23aec131da1bcc527af094 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834 + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + standard (1.52.0) sha256=ec050e63228e31fabe40da3ef96da7edda476f7acdf3e7c2ad47b6e153f6a076 + standard-custom (1.0.2) sha256=424adc84179a074f1a2a309bb9cf7cd6bfdb2b6541f20c6bf9436c0ba22a652b + standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb new file mode 100644 index 0000000000..ddc19e2939 --- /dev/null +++ b/tool/bundler/test_gems.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rack", "~> 3.1" +gem "rack-test", "~> 2.1" +gem "compact_index", "~> 0.15.0" +gem "sinatra", "~> 4.1" +gem "rake", "~> 13.1" +gem "builder", "~> 3.2" +gem "rb_sys" +gem "fiddle" +gem "rubygems-generate_index", "~> 1.1" +gem "concurrent-ruby" +gem "psych" +gem "etc", platforms: [:ruby, :windows] +gem "open3" +gem "shellwords" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock new file mode 100644 index 0000000000..fdffc1f09d --- /dev/null +++ b/tool/bundler/test_gems.rb.lock @@ -0,0 +1,106 @@ +GEM + remote: https://rubygems.org/ + specs: + base64 (0.3.0) + builder (3.3.0) + compact_index (0.15.0) + concurrent-ruby (1.3.6) + date (3.5.1) + date (3.5.1-java) + etc (1.4.6) + fiddle (1.1.8) + jar-dependencies (0.5.5) + logger (1.7.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + open3 (0.2.1) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + rack (3.2.4) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + ruby2_keywords (0.0.5) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + shellwords (0.2.2) + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) + tilt (~> 2.0) + stringio (3.2.0) + tilt (2.6.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + builder (~> 3.2) + compact_index (~> 0.15.0) + concurrent-ruby + etc + fiddle + open3 + psych + rack (~> 3.1) + rack-test (~> 2.1) + rake (~> 13.1) + rb_sys + rubygems-generate_index (~> 1.1) + shellwords + sinatra (~> 4.1) + +CHECKSUMS + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + etc (1.4.6) sha256=0f7e9e7842ea5e3c3bd9bc81746ebb8c65ea29e4c42a93520a0d638129c7de01 + fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mustermann (3.0.4) sha256=85fadcb6b3c6493a8b511b42426f904b7f27b282835502233dd154daab13aa22 + open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + rack (3.2.4) sha256=5d74b6f75082a643f43c1e76b419c40f0e5527fcfee1e669ac1e6b73c0ccb6f6 + rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac + rack-session (2.1.1) sha256=0b6dc07dea7e4b583f58a48e8b806d4c9f1c6c9214ebc202ec94562cbea2e4e9 + rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94 + sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + tilt (2.6.1) sha256=35a99bba2adf7c1e362f5b48f9b581cce4edfba98117e34696dde6d308d84770 + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb new file mode 100644 index 0000000000..8d12c5adde --- /dev/null +++ b/tool/bundler/vendor_gems.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "fileutils", "1.8.0" +gem "molinillo", github: "cocoapods/molinillo", ref: "1d62d7d5f448e79418716dc779a4909509ccda2a" +gem "net-http", "0.7.0" # net-http-0.8.0 is broken with JRuby +gem "net-http-persistent", "4.0.6" +gem "net-protocol", "0.2.2" +gem "optparse", "0.8.0" +gem "pub_grub", github: "jhawthorn/pub_grub", ref: "df6add45d1b4d122daff2f959c9bd1ca93d14261" +gem "resolv", "0.6.2" +gem "securerandom", "0.4.1" +gem "timeout", "0.4.4" +gem "thor", "1.4.0" +gem "tsort", "0.2.0" +gem "uri", "1.1.1" diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock new file mode 100644 index 0000000000..cc7886e60b --- /dev/null +++ b/tool/bundler/vendor_gems.rb.lock @@ -0,0 +1,75 @@ +GIT + remote: https://github.com/cocoapods/molinillo.git + revision: 1d62d7d5f448e79418716dc779a4909509ccda2a + ref: 1d62d7d5f448e79418716dc779a4909509ccda2a + specs: + molinillo (0.8.0) + +GIT + remote: https://github.com/jhawthorn/pub_grub.git + revision: df6add45d1b4d122daff2f959c9bd1ca93d14261 + ref: df6add45d1b4d122daff2f959c9bd1ca93d14261 + specs: + pub_grub (0.5.0) + +GEM + remote: https://rubygems.org/ + specs: + connection_pool (2.5.4) + fileutils (1.8.0) + net-http (0.7.0) + uri + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + net-protocol (0.2.2) + timeout + optparse (0.8.0) + resolv (0.6.2) + securerandom (0.4.1) + thor (1.4.0) + timeout (0.4.4) + tsort (0.2.0) + uri (1.1.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + fileutils (= 1.8.0) + molinillo! + net-http (= 0.7.0) + net-http-persistent (= 4.0.6) + net-protocol (= 0.2.2) + optparse (= 0.8.0) + pub_grub! + resolv (= 0.6.2) + securerandom (= 0.4.1) + thor (= 1.4.0) + timeout (= 0.4.4) + tsort (= 0.2.0) + uri (= 1.1.1) + +CHECKSUMS + connection_pool (2.5.4) sha256=e9e1922327416091f3f6542f5f4446c2a20745276b9aa796dd0bb2fd0ea1e70a + fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02 + molinillo (0.8.0) + net-http (0.7.0) sha256=4db7d9f558f8ffd4dcf832d0aefd02320c569c7d4f857def49e585069673a425 + net-http-persistent (4.0.6) sha256=2abb3a04438edf6cb9e0e7e505969605f709eda3e3c5211beadd621a2c84dd5d + net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 + optparse (0.8.0) sha256=ef6b7fbaf7ec331474f325bc08dd5622e6e1e651007a5341330ee4b08ce734f0 + pub_grub (0.5.0) + resolv (0.6.2) sha256=61efe545cedddeb1b14f77e51f85c85ca66af5098fdbf567fadf32c34590fb14 + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + thor (1.4.0) sha256=8763e822ccb0f1d7bee88cde131b19a65606657b847cc7b7b4b82e772bcd8a3d + timeout (0.4.4) sha256=f0f6f970104b82427cd990680f539b6bbb8b1e55efa913a55c6492935e4e0edb + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + +BUNDLED WITH + 4.0.0.dev diff --git a/tool/checksum.rb b/tool/checksum.rb index bcc60ee14a..8f2d1d97d0 100755 --- a/tool/checksum.rb +++ b/tool/checksum.rb @@ -36,9 +36,7 @@ class Checksum end def update! - open(@checksum, "wb") {|f| - f.puts("src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") - } + File.binwrite(@checksum, "src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") end def update diff --git a/tool/commit-email.rb b/tool/commit-email.rb new file mode 100755 index 0000000000..c887f8783e --- /dev/null +++ b/tool/commit-email.rb @@ -0,0 +1,372 @@ +#!/usr/bin/env ruby + +require "optparse" +require "nkf" +require "shellwords" + +CommitEmailInfo = Struct.new( + :author, + :author_email, + :revision, + :entire_sha256, + :date, + :log, + :branch, + :diffs, + :added_files, :deleted_files, :updated_files, + :added_dirs, :deleted_dirs, :updated_dirs, +) + +class GitInfoBuilder + GitCommandFailure = Class.new(RuntimeError) + + def initialize(repo_path) + @repo_path = repo_path + end + + def build(oldrev, newrev, refname) + diffs = build_diffs(oldrev, newrev) + + info = CommitEmailInfo.new + info.author = git_show(newrev, format: '%an') + info.author_email = normalize_email(git_show(newrev, format: '%aE')) + info.revision = newrev[0...10] + info.entire_sha256 = newrev + info.date = Time.at(Integer(git_show(newrev, format: '%at'))) + info.log = git_show(newrev, format: '%B') + info.branch = git('rev-parse', '--symbolic', '--abbrev-ref', refname).strip + info.diffs = diffs + info.added_files = find_files(diffs, status: :added) + info.deleted_files = find_files(diffs, status: :deleted) + info.updated_files = find_files(diffs, status: :modified) + info.added_dirs = [] # git does not deal with directory + info.deleted_dirs = [] # git does not deal with directory + info.updated_dirs = [] # git does not deal with directory + info + end + + private + + # Force git-svn email address to @ruby-lang.org to avoid email bounce by invalid email address. + def normalize_email(email) + if email.match(/\A[^@]+@\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/) # git-svn + svn_user, _ = email.split('@', 2) + "#{svn_user}@ruby-lang.org" + else + email + end + end + + def find_files(diffs, status:) + files = [] + diffs.each do |path, values| + if values.keys.first == status + files << path + end + end + files + end + + # SVN version: + # { + # "filename" => { + # "[modified|added|deleted|copied|property_changed]" => { + # type: "[modified|added|deleted|copied|property_changed]", + # body: "diff body", # not implemented because not used + # added: Integer, + # deleted: Integer, + # } + # } + # } + def build_diffs(oldrev, newrev) + diffs = {} + + numstats = git('diff', '--numstat', oldrev, newrev).lines.map { |l| l.strip.split("\t", 3) } + git('diff', '--name-status', oldrev, newrev).each_line do |line| + status, path, _newpath = line.strip.split("\t", 3) + diff = build_diff(path, numstats) + + case status + when 'A' + diffs[path] = { added: { type: :added, **diff } } + when 'M' + diffs[path] = { modified: { type: :modified, **diff } } + when 'C' + diffs[path] = { copied: { type: :copied, **diff } } + when 'D' + diffs[path] = { deleted: { type: :deleted, **diff } } + when /\AR/ # R100 (which does not exist in git.ruby-lang.org's git 2.1.4) + # TODO: implement something + else + $stderr.puts "unexpected git diff status: #{status}" + end + end + + diffs + end + + def build_diff(path, numstats) + diff = { added: 0, deleted: 0 } # :body not implemented because not used + line = numstats.find { |(_added, _deleted, file, *)| file == path } + return diff if line.nil? + + added, deleted, _ = line + if added + diff[:added] = Integer(added) + end + if deleted + diff[:deleted] = Integer(deleted) + end + diff + end + + def git_show(revision, format:) + git('show', '--no-show-signature', "--pretty=#{format}", '--no-patch', revision).strip + end + + def git(*args) + command = ['git', '-C', @repo_path, *args] + output = with_gitenv { IO.popen(command, external_encoding: 'UTF-8', &:read) } + unless $?.success? + raise GitCommandFailure, "failed to execute '#{command.join(' ')}':\n#{output}" + end + output + end + + def with_gitenv + orig = ENV.to_h.dup + begin + ENV.delete('GIT_DIR') + yield + ensure + ENV.replace(orig) + end + end +end + +CommitEmailOptions = Struct.new(:error_to, :viewer_uri) + +CommitEmail = Module.new +class << CommitEmail + SENDMAIL = ENV.fetch('SENDMAIL', '/usr/sbin/sendmail') + private_constant :SENDMAIL + + def parse(args) + options = CommitEmailOptions.new + + opts = OptionParser.new do |opts| + opts.separator('') + + opts.on('-e', '--error-to [TO]', + 'Add [TO] to to address when error is occurred') do |to| + options.error_to = to + end + + opts.on('--viewer-uri [URI]', + 'Use [URI] as URI of revision viewer') do |uri| + options.viewer_uri = uri + end + + opts.on_tail('--help', 'Show this message') do + puts opts + exit + end + end + + return opts.parse(args), options + end + + def main(repo_path, to, rest) + args, options = parse(rest) + + infos = args.each_slice(3).flat_map do |oldrev, newrev, refname| + revisions = IO.popen(['git', 'log', '--no-show-signature', '--reverse', '--pretty=%H', "#{oldrev}^..#{newrev}"], &:read).lines.map(&:strip) + revisions[0..-2].zip(revisions[1..-1]).map do |old, new| + GitInfoBuilder.new(repo_path).build(old, new, refname) + end + end + + infos.each do |info| + next if info.branch.start_with?('notes/') + puts "#{info.branch}: #{info.revision} (#{info.author})" + + from = make_from(name: info.author, email: "noreply@ruby-lang.org") + sendmail(to, from, make_mail(to, from, info, viewer_uri: options.viewer_uri)) + end + end + + def sendmail(to, from, mail) + IO.popen([*SENDMAIL.shellsplit, to], 'w') do |f| + f.print(mail) + end + unless $?.success? + raise "Failed to run `#{SENDMAIL} #{to}` with: '#{mail}'" + end + end + + private + + def b_encode(str) + NKF.nkf('-WwM', str) + end + + def make_body(info, viewer_uri:) + body = +'' + body << "#{info.author}\t#{format_time(info.date)}\n" + body << "\n" + body << " New Revision: #{info.revision}\n" + body << "\n" + body << " #{viewer_uri}#{info.revision}\n" + body << "\n" + body << " Log:\n" + body << info.log.lstrip.gsub(/^\t*/, ' ').rstrip + body << "\n\n" + body << added_dirs(info) + body << added_files(info) + body << deleted_dirs(info) + body << deleted_files(info) + body << modified_dirs(info) + body << modified_files(info) + [body.rstrip].pack('M') + end + + def format_time(time) + time.strftime('%Y-%m-%d %X %z (%a, %d %b %Y)') + end + + def changed_items(title, type, items) + rv = '' + unless items.empty? + rv << " #{title} #{type}:\n" + rv << items.collect {|item| " #{item}\n"}.join('') + end + rv + end + + def changed_files(title, files) + changed_items(title, 'files', files) + end + + def added_files(info) + changed_files('Added', info.added_files) + end + + def deleted_files(info) + changed_files('Removed', info.deleted_files) + end + + def modified_files(info) + changed_files('Modified', info.updated_files) + end + + def changed_dirs(title, files) + changed_items(title, 'directories', files) + end + + def added_dirs(info) + changed_dirs('Added', info.added_dirs) + end + + def deleted_dirs(info) + changed_dirs('Removed', info.deleted_dirs) + end + + def modified_dirs(info) + changed_dirs('Modified', info.updated_dirs) + end + + def changed_dirs_info(info, uri) + (info.added_dirs.collect do |dir| + " Added: #{dir}\n" + end + info.deleted_dirs.collect do |dir| + " Deleted: #{dir}\n" + end + info.updated_dirs.collect do |dir| + " Modified: #{dir}\n" + end).join("\n") + end + + def diff_info(info, uri) + info.diffs.collect do |key, values| + [ + key, + values.collect do |type, value| + case type + when :added + rev = "?revision=#{info.revision}&view=markup" + when :modified, :property_changed + prev_revision = (info.revision.is_a?(Integer) ? info.revision - 1 : "#{info.revision}^") + rev = "?r1=#{info.revision}&r2=#{prev_revision}&diff_format=u" + when :deleted, :copied + rev = '' + else + raise "unknown diff type: #{value[:type]}" + end + + link = [uri, key.sub(/ .+/, '') || ''].join('/') + rev + + desc = '' + + [desc, link] + end + ] + end + end + + def make_header(to, from, info) + <<~EOS + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: #{from} + To: #{to} + Subject: #{make_subject(info)} + EOS + end + + def make_subject(info) + subject = +'' + subject << "#{info.revision}" + subject << " (#{info.branch})" + subject << ': ' + subject << info.log.lstrip.lines.first.to_s.strip + b_encode(subject) + end + + # https://tools.ietf.org/html/rfc822#section-4.1 + # https://tools.ietf.org/html/rfc822#section-6.1 + # https://tools.ietf.org/html/rfc822#appendix-D + # https://tools.ietf.org/html/rfc2047 + def make_from(name:, email:) + if name.ascii_only? + escaped_name = name.gsub(/["\\\n]/) { |c| "\\#{c}" } + %Q["#{escaped_name}" <#{email}>] + else + escaped_name = "=?UTF-8?B?#{NKF.nkf('-WwMB', name)}?=" + %Q[#{escaped_name} <#{email}>] + end + end + + def make_mail(to, from, info, viewer_uri:) + make_header(to, from, info) + make_body(info, viewer_uri: viewer_uri) + end +end + +repo_path, to, *rest = ARGV +begin + CommitEmail.main(repo_path, to, rest) +rescue StandardError => e + $stderr.puts "#{e.class}: #{e.message}" + $stderr.puts e.backtrace + + _, options = CommitEmail.parse(rest) + to = options.error_to + CommitEmail.sendmail(to, to, <<-MAIL) +From: #{to} +To: #{to} +Subject: Error + +#{$!.class}: #{$!.message} +#{$@.join("\n")} +MAIL + exit 1 +end diff --git a/tool/darwin-ar b/tool/darwin-ar new file mode 100755 index 0000000000..8b25425cfe --- /dev/null +++ b/tool/darwin-ar @@ -0,0 +1,6 @@ +#!/bin/bash +export LANG=C LC_ALL=C # Suppress localication +exec 2> >(exec grep -v \ + -e ' no symbols$' \ + >&2) +exec "$@" diff --git a/tool/darwin-cc b/tool/darwin-cc index 6eee96e435..42637022a4 100755 --- a/tool/darwin-cc +++ b/tool/darwin-cc @@ -2,5 +2,8 @@ exec 2> >(exec grep -v \ -e '^ld: warning: The [a-z0-9_][a-z0-9_]* architecture is deprecated for macOS' \ -e '^ld: warning: text-based stub file /System/Library/Frameworks/' \ + -e '^ld: warning: ignoring duplicate libraries:' \ + -e "warning: '\.debug_macinfo' is not currently supported:" \ + -e "note: while processing" \ >&2) exec "$@" diff --git a/tool/disable_ipv6.sh b/tool/disable_ipv6.sh new file mode 100755 index 0000000000..ce1cc0da68 --- /dev/null +++ b/tool/disable_ipv6.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex +sysctl -w net.ipv6.conf.all.disable_ipv6=1 +sysctl -w net.ipv6.conf.default.disable_ipv6=1 +sysctl -w net.ipv6.conf.lo.disable_ipv6=1 + +cat /etc/hosts +ruby -e "hosts = File.read('/etc/hosts').sub(/^::1\s*localhost.*$/, ''); File.write('/etc/hosts', hosts)" +cat /etc/hosts diff --git a/tool/downloader.rb b/tool/downloader.rb index f42f9a0ef1..39ebf44a83 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -1,69 +1,53 @@ # Used by configure and make to download or update mirrored Ruby and GCC -# files. This will use HTTPS if possible, falling back to HTTP. +# files. + +# -*- frozen-string-literal: true -*- require 'fileutils' require 'open-uri' require 'pathname' -begin - require 'net/https' -rescue LoadError - https = 'http' -else - https = 'https' - - # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old - # versions do not. so, patching OpenSSL::X509::Store#add_file instead. - class OpenSSL::X509::Store - alias orig_add_file add_file - def add_file(pems) - Array(pems).each do |pem| - if File.directory?(pem) - add_path pem - else - orig_add_file pem - end - end - end - end - # since open-uri internally checks ssl_ca_cert using File.directory?, - # allow to accept an array. - class <<File - alias orig_directory? directory? - def File.directory? files - files.is_a?(Array) ? false : orig_directory?(files) - end - end -end +verbose, $VERBOSE = $VERBOSE, nil +require 'net/https' +$VERBOSE = verbose class Downloader - def self.https=(https) - @@https = https - end - - def self.https? - @@https == 'https' + def self.find(dlname) + constants.find do |name| + return const_get(name) if dlname.casecmp(name.to_s) == 0 + end end - def self.https - @@https + def self.get_option(argv, options) + false end class GNU < self - def self.download(name, *rest) - if https? - super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) + Mirrors = %w[ + https://raw.githubusercontent.com/autotools-mirror/autoconf/refs/heads/master/build-aux/ + https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master + ] + + def self.download(name, *rest, **options) + Mirrors.each_with_index do |url, i| + super("#{url}/#{name}", name, *rest, **options) + rescue => e + raise if i + 1 == Mirrors.size # no more URLs + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" else - super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) + return end end end class RubyGems < self - def self.download(name, dir = nil, since = true, options = {}) + def self.download(name, dir = nil, since = true, **options) require 'rubygems' - options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) - super("https://rubygems.org/downloads/#{name}", name, dir, since, options) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true + end + super("https://rubygems.org/downloads/#{name}", name, dir, since, **options) end end @@ -71,18 +55,30 @@ class Downloader class Unicode < self INDEX = {} # cache index file information across files in the same directory - UNICODE_PUBLIC = "http://www.unicode.org/Public/" + UNICODE_PUBLIC = "https://www.unicode.org/Public/" + + def self.get_option(argv, options) + case argv[0] + when '--unicode-beta' + options[:unicode_beta] = argv[1] + argv.shift(2) + true + when /\A--unicode-beta=(.*)/m + options[:unicode_beta] = $1 + argv.shift + true + else + super + end + end - def self.download(name, dir = nil, since = true, options = {}) - options = options.dup - unicode_beta = options.delete(:unicode_beta) + def self.download(name, dir = nil, since = true, unicode_beta: nil, **options) name_dir_part = name.sub(/[^\/]+$/, '') if unicode_beta == 'YES' if INDEX.size == 0 - index_options = options.dup - index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file + cache_save = false # TODO: make sure caching really doesn't work for index file index_data = File.read(under(dir, "index.html")) rescue nil - index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options) + index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, cache_save: cache_save, **options) INDEX[:index] = File.read(index_file) since = true unless INDEX[:index] == index_data end @@ -91,7 +87,7 @@ class Downloader beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] # make sure we always check for new versions of files, # because they can easily change in the beta period - super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options) + super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, **options) else index_file = Pathname.new(under(dir, name_dir_part+'index.html')) if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ @@ -99,7 +95,7 @@ class Downloader "Remove all files in this directory and in .downloaded-cache/ " + "because they may be leftovers from the beta period." end - super(UNICODE_PUBLIC+name, name, dir, since, options) + super(UNICODE_PUBLIC+name, name, dir, since, **options) end end end @@ -125,6 +121,21 @@ class Downloader options end + def self.httpdate(date) + Time.httpdate(date) + rescue ArgumentError => e + # Some hosts (e.g., zlib.net) return similar to RFC 850 but 4 + # digit year, sometimes. + /\A\s* + (?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day,\x20 + (\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4})\x20 + (\d\d):(\d\d):(\d\d)\x20 + GMT + \s*\z/ix =~ date or raise + warn e.message + Time.utc($3, $2, $1, $4, $5, $6) + end + # Downloader.download(url, name, [dir, [since]]) # # Update a file from url if newer version is available. @@ -149,59 +160,53 @@ class Downloader # Example usage: # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', # 'UnicodeData.txt', 'enc/unicode/data' - def self.download(url, name, dir = nil, since = true, options = {}) - options = options.dup + def self.download(url, name, dir = nil, since = true, + cache_save: ENV["CACHE_SAVE"] != "no", cache_dir: nil, + ignore_http_client_errors: nil, + dryrun: nil, verbose: false, **options) url = URI(url) - dryrun = options.delete(:dryrun) - options.delete(:unicode_beta) # just to be on the safe side for gems and gcc - if name file = Pathname.new(under(dir, name)) else name = File.basename(url.path) end - cache_save = options.delete(:cache_save) { - ENV["CACHE_SAVE"] != "no" - } - cache = cache_file(url, name, options.delete(:cache_dir)) + cache = cache_file(url, name, cache_dir) file ||= cache if since.nil? and file.exist? - if $VERBOSE + if verbose $stdout.puts "#{file} already exists" $stdout.flush end - if cache_save - save_cache(cache, file, name) - end return file.to_path end if dryrun puts "Download #{url} into #{file}" return end - if link_cache(cache, file, name, $VERBOSE) + if link_cache(cache, file, name, verbose: verbose) return file.to_path end - if !https? and URI::HTTPS === url - warn "*** using http instead of https ***" - url.scheme = 'http' - url = URI(url.to_s) - end - if $VERBOSE + if verbose $stdout.print "downloading #{name} ... " $stdout.flush end + mtime = nil + options = options.merge(http_options(file, since.nil? ? true : since)) begin - data = with_retry(9) do - url.read(options.merge(http_options(file, since.nil? ? true : since))) - end + data = with_retry(10) {url.read(options)} rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified - if $VERBOSE + case http_error.message + when /^304 / # 304 Not Modified + if verbose $stdout.puts "#{name} not modified" $stdout.flush end return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error @@ -216,20 +221,22 @@ class Downloader return file.to_path end raise + else + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end end - mtime = nil dest = (cache_save && cache && !cache.exist? ? cache : file) dest.parent.mkpath + dest.unlink if dest.symlink? && !dest.exist? dest.open("wb", 0600) do |f| f.write(data) f.chmod(mode_for(data)) - mtime = data.meta["last-modified"] end if mtime - mtime = Time.httpdate(mtime) dest.utime(mtime, mtime) end - if $VERBOSE + if verbose $stdout.puts "done" $stdout.flush end @@ -247,25 +254,38 @@ class Downloader dir ? File.join(dir, File.basename(name)) : name end + def self.default_cache_dir + if cache_dir = ENV['CACHE_DIR'] + return cache_dir unless cache_dir.empty? + end + ".downloaded-cache" + end + def self.cache_file(url, name, cache_dir = nil) case cache_dir when false return nil when nil - cache_dir = ENV['CACHE_DIR'] - if !cache_dir or cache_dir.empty? - cache_dir = ".downloaded-cache" - end + cache_dir = default_cache_dir end Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) end - def self.link_cache(cache, file, name, verbose = false) + def self.link_cache(cache, file, name, verbose: false) return false unless cache and cache.exist? return true if cache.eql?(file) if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] begin - file.make_symlink(cache.relative_path_from(file.parent)) + link = cache.relative_path_from(file.parent) + rescue ArgumentError + abs = cache.expand_path + link = abs.relative_path_from(file.parent.expand_path) + if link.to_s.count("/") > abs.to_s.count("/") + link = abs + end + end + begin + file.make_symlink(link) rescue SystemCallError else if verbose @@ -308,7 +328,7 @@ class Downloader times = 0 begin block.call - rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout => e + rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error times += 1 if times <= max_times @@ -323,50 +343,76 @@ class Downloader private_class_method :with_retry end -Downloader.https = https.freeze - if $0 == __FILE__ since = true options = {} + dl = nil + (args = []).singleton_class.__send__(:define_method, :downloader?) do |arg| + !dl and args.empty? and (dl = Downloader.find(arg)) + end until ARGV.empty? + if ARGV[0] == '--' + ARGV.shift + break if ARGV.empty? + ARGV.shift if args.downloader? ARGV[0] + args.concat(ARGV) + break + end + + if dl and dl.get_option(ARGV, options) + # the downloader dealt with the arguments, and should be removed + # from ARGV. + next + end + case ARGV[0] - when '-d' + when '-d', '--destdir' + ## -d, --destdir DIRECTORY Download into the directory destdir = ARGV[1] ARGV.shift - when '-p' - # strip directory names from the name to download, and add the - # prefix instead. + when '-p', '--prefix' + ## -p, --prefix Strip directory names from the name to download, + ## and add the prefix instead. prefix = ARGV[1] ARGV.shift - when '-e' + when '-e', '--exist', '--non-existent-only' + ## -e, --exist, --non-existent-only Skip already existent files. since = nil - when '-a' + when '-a', '--always' + ## -a, --always Download all files. since = false - when '-n', '--dryrun' + when '-u', '--update', '--if-modified' + ## -u, --update, --if-modified Download newer files only. + since = true + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Do not download actually. options[:dryrun] = true when '--cache-dir' + ## --cache-dir DIRECTORY Cache downloaded files in the directory. options[:cache_dir] = ARGV[1] ARGV.shift - when '--unicode-beta' - options[:unicode_beta] = ARGV[1] - ARGV.shift when /\A--cache-dir=(.*)/m options[:cache_dir] = $1 + when /\A--help\z/ + ## --help Print this message + puts "Usage: #$0 [options] relative-url..." + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ abort "#{$0}: unknown option #{ARGV[0]}" else - break + args << ARGV[0] unless args.downloader? ARGV[0] end ARGV.shift end - dl = Downloader.constants.find do |name| - ARGV[0].casecmp(name.to_s) == 0 - end unless ARGV.empty? - $VERBOSE = true + options[:verbose] = true if dl - dl = Downloader.const_get(dl) - ARGV.shift - ARGV.each do |name| + args.each do |name| dir = destdir if prefix name = name.sub(/\A\.\//, '') @@ -383,10 +429,10 @@ if $0 == __FILE__ end name = "#{prefix}/#{name}" end - dl.download(name, dir, since, options) + dl.download(name, dir, since, **options) end else - abort "usage: #{$0} url name" unless ARGV.size == 2 - Downloader.download(ARGV[0], ARGV[1], destdir, since, options) + abort "usage: #{$0} url name" unless args.size == 2 + Downloader.download(args[0], args[1], destdir, since, **options) end end diff --git a/tool/enc-case-folding.rb b/tool/enc-case-folding.rb new file mode 100755 index 0000000000..82fec7b625 --- /dev/null +++ b/tool/enc-case-folding.rb @@ -0,0 +1,416 @@ +#!/usr/bin/ruby +require 'stringio' + +# Usage (for case folding only): +# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +# $ ruby enc-case-folding.rb CaseFolding.txt -o casefold.h +# or (for case folding and case mapping): +# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +# $ wget http://www.unicode.org/Public/UNIDATA/UnicodeData.txt +# $ wget http://www.unicode.org/Public/UNIDATA/SpecialCasing.txt +# $ ruby enc-case-folding.rb -m . -o casefold.h +# using -d or --debug will include UTF-8 characters in comments for debugging + +class CaseFolding + module Util + module_function + + def hex_seq(v) + v.map { |i| "0x%04x" % i }.join(", ") + end + + def print_table_1(dest, type, mapping_data, data) + for k, v in data = data.sort + sk = (Array === k and k.length > 1) ? "{#{hex_seq(k)}}" : ("0x%04x" % k) + if type=='CaseUnfold_11' and v.length>1 + # reorder CaseUnfold_11 entries to avoid special treatment for U+03B9/U+03BC/U+A64B + item = mapping_data.map("%04X" % k[0]) + upper = item.upper if item + v = v.sort_by { |i| ("%04X"%i) == upper ? 0 : 1 } + end + ck = @debug ? ' /* ' + Array(k).pack("U*") + ' */' : '' + cv = @debug ? ' /* ' + Array(v).map{|c|[c].pack("U*")}.join(", ") + ' */' : '' + dest.print(" {#{sk}#{ck}, {#{v.length}#{mapping_data.flags(k, type, v)}, {#{hex_seq(v)}#{cv}}}},\n") + end + data + end + + def print_table(dest, type, mapping_data, data) + dest.print("static const #{type}_Type #{type}_Table[] = {\n") + i = 0 + ret = data.inject([]) do |a, (n, d)| + dest.print("#define #{n} (*(#{type}_Type (*)[#{d.size}])(#{type}_Table+#{i}))\n") + i += d.size + a.concat(print_table_1(dest, type, mapping_data, d)) + end + dest.print("};\n\n") + ret + end + end + + include Util + + attr_reader :fold, :fold_locale, :unfold, :unfold_locale, :version + + def load(filename) + pattern = /([0-9A-F]{4,6}); ([CFT]); ([0-9A-F]{4,6})(?: ([0-9A-F]{4,6}))?(?: ([0-9A-F]{4,6}))?;/ + + @fold = fold = {} + @unfold = unfold = [{}, {}, {}] + @debug = false + @version = nil + turkic = [] + + File.foreach(filename, mode: "rb") do |line| + @version ||= line[/-([0-9.]+).txt/, 1] + next unless res = pattern.match(line) + ch_from = res[1].to_i(16) + + if res[2] == 'T' + # Turkic case folding + turkic << ch_from + next + end + + # store folding data + ch_to = res[3..6].inject([]) do |a, i| + break a unless i + a << i.to_i(16) + end + fold[ch_from] = ch_to + + # store unfolding data + i = ch_to.length - 1 + (unfold[i][ch_to] ||= []) << ch_from + end + + # move locale dependent data to (un)fold_locale + @fold_locale = fold_locale = {} + @unfold_locale = unfold_locale = [{}, {}] + for ch_from in turkic + key = fold[ch_from] + i = key.length - 1 + unfold_locale[i][i == 0 ? key[0] : key] = unfold[i].delete(key) + fold_locale[ch_from] = fold.delete(ch_from) + end + self + end + + def range_check(code) + "#{code} <= MAX_CODE_VALUE && #{code} >= MIN_CODE_VALUE" + end + + def lookup_hash(key, type, data) + hash = "onigenc_unicode_#{key}_hash" + lookup = "onigenc_unicode_#{key}_lookup" + arity = Array(data[0][0]).size + gperf = %W"gperf -7 -k#{[*1..(arity*3)].join(',')} -F,-1 -c -j1 -i1 -t -T -E -C -H #{hash} -N #{lookup} -n" + argname = arity > 1 ? "codes" : "code" + argdecl = "const OnigCodePoint #{arity > 1 ? "*": ""}#{argname}" + n = 7 + m = (1 << n) - 1 + min, max = data.map {|c, *|c}.flatten.minmax + src = IO.popen(gperf, "r+") {|f| + f << "short\n%%\n" + data.each_with_index {|(k, _), i| + k = Array(k) + ks = k.map {|j| [(j >> n*2) & m, (j >> n) & m, (j) & m]}.flatten.map {|c| "\\x%.2x" % c}.join("") + f.printf "\"%s\", ::::/*%s*/ %d\n", ks, k.map {|c| "0x%.4x" % c}.join(","), i + } + f << "%%\n" + f.close_write + f.read + } + src.sub!(/^(#{hash})\s*\(.*?\).*?\n\{\n(.*)^\}/m) { + name = $1 + body = $2 + body.gsub!(/\(unsigned char\)str\[(\d+)\]/, "bits_#{arity > 1 ? 'at' : 'of'}(#{argname}, \\1)") + "#{name}(#{argdecl})\n{\n#{body}}" + } + src.sub!(/const short *\*\n^(#{lookup})\s*\(.*?\).*?\n\{\n(.*)^\}/m) { + name = $1 + body = $2 + body.sub!(/\benum\s+\{(\n[ \t]+)/, "\\&MIN_CODE_VALUE = 0x#{min.to_s(16)},\\1""MAX_CODE_VALUE = 0x#{max.to_s(16)},\\1") + body.gsub!(/(#{hash})\s*\(.*?\)/, "\\1(#{argname})") + body.gsub!(/\{"",-1}/, "-1") + body.gsub!(/\{"(?:[^"]|\\")+", *::::(.*)\}/, '\1') + body.sub!(/(\s+if\s)\(len\b.*\)/) do + "#$1(" << + (arity > 1 ? (0...arity).map {|i| range_check("#{argname}[#{i}]")}.join(" &&\n ") : range_check(argname)) << + ")" + end + v = nil + body.sub!(/(if\s*\(.*MAX_HASH_VALUE.*\)\n([ \t]*))\{(.*?)\n\2\}/m) { + pre = $1 + indent = $2 + s = $3 + s.sub!(/const char *\* *(\w+)( *= *wordlist\[\w+\]).\w+/, 'short \1 = wordlist[key]') + v = $1 + s.sub!(/\bif *\(.*\)/, "if (#{v} >= 0 && code#{arity}_equal(#{argname}, #{key}_Table[#{v}].from))") + "#{pre}{#{s}\n#{indent}}" + } + body.sub!(/\b(return\s+&)([^;]+);/, '\1'"#{key}_Table[#{v}].to;") + "static const #{type} *\n#{name}(#{argdecl})\n{\n#{body}}" + } + src + end + + def display(dest, mapping_data) + # print the header + dest.print("/* DO NOT EDIT THIS FILE. */\n") + dest.print("/* Generated by enc-case-folding.rb */\n\n") + + versions = version.scan(/\d+/) + dest.print("#if defined ONIG_UNICODE_VERSION_STRING && !( \\\n") + %w[MAJOR MINOR TEENY].zip(versions) do |n, v| + dest.print(" ONIG_UNICODE_VERSION_#{n} == #{v} && \\\n") + end + dest.print(" 1)\n") + dest.print("# error ONIG_UNICODE_VERSION_STRING mismatch\n") + dest.print("#endif\n") + dest.print("#define ONIG_UNICODE_VERSION_STRING #{version.dump}\n") + %w[MAJOR MINOR TEENY].zip(versions) do |n, v| + dest.print("#define ONIG_UNICODE_VERSION_#{n} #{v}\n") + end + dest.print("\n") + + # print folding data + + # CaseFold + CaseFold_Locale + name = "CaseFold_11" + data = print_table(dest, name, mapping_data, "CaseFold"=>fold, "CaseFold_Locale"=>fold_locale) + dest.print lookup_hash(name, "CodePointList3", data) + + # print unfolding data + + # CaseUnfold_11 + CaseUnfold_11_Locale + name = "CaseUnfold_11" + data = print_table(dest, name, mapping_data, name=>unfold[0], "#{name}_Locale"=>unfold_locale[0]) + dest.print lookup_hash(name, "CodePointList3", data) + + # CaseUnfold_12 + CaseUnfold_12_Locale + name = "CaseUnfold_12" + data = print_table(dest, name, mapping_data, name=>unfold[1], "#{name}_Locale"=>unfold_locale[1]) + dest.print lookup_hash(name, "CodePointList2", data) + + # CaseUnfold_13 + name = "CaseUnfold_13" + data = print_table(dest, name, mapping_data, name=>unfold[2]) + dest.print lookup_hash(name, "CodePointList2", data) + + # TitleCase + dest.print mapping_data.specials_output + end + + def debug! + @debug = true + end + + def self.load(*args) + new.load(*args) + end +end + +class MapItem + attr_accessor :upper, :lower, :title, :code + + def initialize(code, upper, lower, title) + @code = code + @upper = upper unless upper == '' + @lower = lower unless lower == '' + @title = title unless title == '' + end +end + +class CaseMapping + attr_reader :filename, :version + + def initialize(mapping_directory) + @mappings = {} + @specials = [] + @specials_length = 0 + @version = nil + File.foreach(File.join(mapping_directory, 'UnicodeData.txt'), mode: "rb") do |line| + next if line =~ /^</ + code, _, _, _, _, _, _, _, _, _, _, _, upper, lower, title = line.chomp.split ';' + unless upper and lower and title and (upper+lower+title)=='' + @mappings[code] = MapItem.new(code, upper, lower, title) + end + end + + @filename = File.join(mapping_directory, 'SpecialCasing.txt') + File.foreach(@filename, mode: "rb") do |line| + @version ||= line[/-([0-9.]+).txt/, 1] + line.chomp! + line, comment = line.split(/ *#/) + next if not line or line == '' + code, lower, title, upper, conditions = line.split(/ *; */) + unless conditions + item = @mappings[code] + item.lower = lower + item.title = title + item.upper = upper + end + end + end + + def map (from) + @mappings[from] + end + + def flags(from, type, to) + # types: CaseFold_11, CaseUnfold_11, CaseUnfold_12, CaseUnfold_13 + flags = "" + from = Array(from).map {|i| "%04X" % i}.join(" ") + to = Array(to).map {|i| "%04X" % i}.join(" ") + item = map(from) + specials = [] + case type + when 'CaseFold_11' + flags += '|F' + if item + flags += '|U' if to==item.upper + flags += '|D' if to==item.lower + unless item.upper == item.title + if item.code == item.title + flags += '|IT' + swap = case item.code + when '01C5' then '0064 017D' + when '01C8' then '006C 004A' + when '01CB' then '006E 004A' + when '01F2' then '0064 005A' + else # Greek + to.split(' ').first + ' 0399' + end + specials << swap + else + flags += '|ST' + specials << item.title + end + end + unless item.lower.nil? or item.lower==from or item.lower==to + specials << item.lower + flags += '|SL' + end + unless item.upper.nil? or item.upper==from or item.upper==to + specials << item.upper + flags += '|SU' + end + end + when 'CaseUnfold_11' + to = to.split(/ /) + if item + case to.first + when item.upper then flags += '|U' + when item.lower then flags += '|D' + else + raise "Unpredicted case 0 in enc/unicode/case_folding.rb. Please contact https://bugs.ruby-lang.org/." + end + unless item.upper == item.title + if item.code == item.title + flags += '|IT' # was unpredicted case 1 + elsif item.title==to[1] + flags += '|ST' + else + raise "Unpredicted case 2 in enc/unicode/case_folding.rb. Please contact https://bugs.ruby-lang.org/." + end + end + end + end + unless specials.empty? + flags += "|I(#{@specials_length})" + @specials_length += specials.map { |s| s.split(/ /).length }.reduce(:+) + @specials << specials + end + flags + end + + def debug! + @debug = true + end + + def specials_output + "static const OnigCodePoint CaseMappingSpecials[] = {\n" + + @specials.map do |sps| + ' ' + sps.map do |sp| + chars = sp.split(/ /) + ct = ' /* ' + Array(chars).map{|c|[c.to_i(16)].pack("U*")}.join(", ") + ' */' if @debug + " L(#{chars.length})|#{chars.map {|c| "0x"+c }.join(', ')}#{ct}," + end.join + "\n" + end.join + "};\n" + end + + def self.load(*args) + new(*args) + end +end + +class CaseMappingDummy + def flags(from, type, to) + "" + end + + def titlecase_output() '' end + def debug!() end +end + +if $0 == __FILE__ + require 'optparse' + dest = nil + mapping_directory = nil + mapping_data = nil + debug = false + fold_1 = false + ARGV.options do |opt| + opt.banner << " [INPUT]" + opt.on("--output-file=FILE", "-o", "output to the FILE instead of STDOUT") {|output| + dest = (output unless output == '-') + } + opt.on('--mapping-data-directory=DIRECTORY', '-m', 'data DIRECTORY of mapping files') { |directory| + mapping_directory = directory + } + opt.on('--debug', '-d') { + debug = true + } + opt.parse! + abort(opt.to_s) if ARGV.size > 1 + end + if mapping_directory + if ARGV[0] + warn "Either specify directory or individual file, but not both." + exit + end + filename = File.join(mapping_directory, 'CaseFolding.txt') + mapping_data = CaseMapping.load(mapping_directory) + end + filename ||= ARGV[0] || 'CaseFolding.txt' + data = CaseFolding.load(filename) + if mapping_data and data.version != mapping_data.version + abort "Unicode data version mismatch\n" \ + " #{filename} = #{data.version}\n" \ + " #{mapping_data.filename} = #{mapping_data.version}" + end + mapping_data ||= CaseMappingDummy.new + + if debug + data.debug! + mapping_data.debug! + end + f = StringIO.new + begin + data.display(f, mapping_data) + rescue Errno::ENOENT => e + raise unless /gperf/ =~ e.message + warn e.message + abort unless dest + File.utime(nil, nil, dest) # assume existing file is OK + exit + else + s = f.string + end + if dest + File.binwrite(dest, s) + else + STDOUT.print(s) + end +end diff --git a/tool/enc-emoji-citrus-gen.rb b/tool/enc-emoji-citrus-gen.rb index da9c8a6b62..0b37e48d3f 100644 --- a/tool/enc-emoji-citrus-gen.rb +++ b/tool/enc-emoji-citrus-gen.rb @@ -71,7 +71,7 @@ end def generate_to_ucs(params, pairs) pairs.sort_by! {|u, c| c } name = "EMOJI_#{params[:name]}%UCS" - open("#{name}.src", "w") do |io| + File.open("#{name}.src", "w") do |io| io.print header(params.merge(name: name.tr('%', '/'))) io.puts io.puts "BEGIN_MAP" @@ -83,7 +83,7 @@ end def generate_from_ucs(params, pairs) pairs.sort_by! {|u, c| u } name = "UCS%EMOJI_#{params[:name]}" - open("#{name}.src", "w") do |io| + File.open("#{name}.src", "w") do |io| io.print header(params.merge(name: name.tr('%', '/'))) io.puts io.puts "BEGIN_MAP" diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index f64cecb881..a89390ad8f 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -5,14 +5,33 @@ # # To use this, get UnicodeData.txt, Scripts.txt, PropList.txt, # PropertyAliases.txt, PropertyValueAliases.txt, DerivedCoreProperties.txt, -# DerivedAge.txt and Blocks.txt from unicode.org. +# DerivedAge.txt, Blocks.txt, emoji/emoji-data.txt, +# auxiliary/GraphemeBreakProperty.txt from unicode.org # (http://unicode.org/Public/UNIDATA/) And run following command. -# ruby1.9 tool/enc-unicode.rb data_dir > enc/unicode/name2ctype.kwd +# tool/enc-unicode.rb data_dir emoji_data_dir > enc/unicode/name2ctype.kwd # You can get source file for gperf. After this, simply make ruby. - -if ARGV[0] == "--header" - header = true - ARGV.shift +# Or directly run: +# tool/enc-unicode.rb --header data_dir emoji_data_dir > enc/unicode/<VERSION>/name2ctype.h +# +# There are Makefile rules that automate steps above: `make update-unicode` and +# `make enc/unicode/<VERSION>/name2ctype.h`. + +while arg = ARGV.shift + case arg + when "--" + break + when "--header" + header = true + when "--diff" + diff = ARGV.shift or abort "#{$0}: --diff=DIFF-COMMAND" + when /\A--diff=(.+)/m + diff = $1 + when /\A-/ + abort "#{$0}: unknown option #{arg}" + else + ARGV.unshift(arg) + break + end end unless ARGV.size == 2 abort "Usage: #{$0} data_directory emoji_data_directory" @@ -59,7 +78,7 @@ def parse_unicode_data(file) data = {'Any' => (0x0000..0x10ffff).to_a, 'Assigned' => [], 'ASCII' => (0..0x007F).to_a, 'NEWLINE' => [0x0a], 'Cn' => []} beg_cp = nil - IO.foreach(file) do |line| + File.foreach(file) do |line| fields = line.split(';') cp = fields[0].to_i(16) @@ -127,7 +146,8 @@ def define_posix_props(data) data['Space'] = data['White_Space'] data['Blank'] = data['Space_Separator'] + [0x0009] data['Cntrl'] = data['Cc'] - data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + + data['Connector_Punctuation'] + data['Join_Control'] data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - data['Surrogate'] - data['Unassigned'] data['Print'] = data['Graph'] + data['Space_Separator'] @@ -138,21 +158,31 @@ def parse_scripts(data, categories) {:fn => 'DerivedCoreProperties.txt', :title => 'Derived Property'}, {:fn => 'Scripts.txt', :title => 'Script'}, {:fn => 'PropList.txt', :title => 'Binary Property'}, - {:fn => 'emoji-data.txt', :title => 'Emoji'} + {:fn => 'emoji/emoji-data.txt', :title => 'Emoji'} ] current = nil cps = [] names = {} files.each do |file| data_foreach(file[:fn]) do |line| + # Parse Unicode data files and store code points and properties. if /^# Total (?:code points|elements): / =~ line data[current] = cps categories[current] = file[:title] (names[file[:title]] ||= []) << current cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line - current = $3 + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w(?:[\w\s;]*\w)?)/ =~ line + # $1: The first hexadecimal code point or the start of a range. + # $2: The end code point of the range, if present. + # If there's no range (just a single code point), $2 is nil. + # $3: The property or other info. + # Example: + # line = "0915..0939 ; InCB; Consonant # Lo [37] DEVANAGARI LETTER KA..DEVANAGARI LETTER HA" + # $1 = "0915" + # $2 = "0939" + # $3 = "InCB; Consonant" $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + current = $3.gsub(/\W+/, '_') end end end @@ -205,7 +235,7 @@ def parse_age(data) ages << current last_constname = constname cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\d+\.\d+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\d+\.\d+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -224,7 +254,7 @@ def parse_GraphemeBreakProperty(data) make_const(constname, cps, "Grapheme_Cluster_Break=#{current}") ages << current cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -236,7 +266,7 @@ def parse_block(data) cps = [] blocks = [] data_foreach('Blocks.txt') do |line| - if /^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+);\s*(.*)/ =~ line + if /^(\h+)\.\.(\h+);\s*(.*)/ =~ line cps = ($1.to_i(16)..$2.to_i(16)).to_a constname = constantize_blockname($3) data[constname] = cps @@ -253,23 +283,12 @@ def parse_block(data) blocks << constname end -# shim for Ruby 1.8 -unless {}.respond_to?(:key) - class Hash - alias key index - end -end - $const_cache = {} # make_const(property, pairs, name): Prints a 'static const' structure for a # given property, group of paired codepoints, and a human-friendly name for # the group def make_const(prop, data, name) - if name.empty? - puts "\n/* '#{prop}' */" - else - puts "\n/* '#{prop}': #{name} */" - end + puts "\n/* '#{prop}': #{name} */" # comment used to generate documentation if origprop = $const_cache.key(data) puts "#define CR_#{prop} CR_#{origprop}" else @@ -305,24 +324,25 @@ def constantize_blockname(name) end def get_file(name) - File.join(ARGV[name.start_with?("emoji-") ? 1 : 0], name) + File.join(ARGV[name.start_with?("emoji-[stz]") ? 1 : 0], name) end def data_foreach(name, &block) fn = get_file(name) warn "Reading #{name}" - if /^emoji-/ =~ name - sep = "" - pat = /^# #{Regexp.quote(File.basename(name))}.*^# Version: ([\d.]+)/m - type = :Emoji - else - sep = "\n" - pat = /^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/ - type = :Unicode - end File.open(fn, 'rb') do |f| - line = f.gets(sep) - unless version = line[pat, 1] + if /^emoji/ =~ name + line = f.gets("") + # Headers till Emoji 13 or 15 + version = line[/^# #{Regexp.quote(File.basename(name))}.*(?:^# Version:|Emoji Version) ([\d.]+)/m, 1] + type = :Emoji + else + # Headers since Emoji 14 or other Unicode data + line = f.gets("\n") + type = :Unicode + end + version ||= line[/^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/, 1] + unless version raise ArgumentError, <<-ERROR #{name}: no #{type} version #{line.gsub(/^/, '> ')} @@ -330,7 +350,7 @@ def data_foreach(name, &block) end if !(v = $versions[type]) $versions[type] = version - elsif v != version + elsif v != version and "#{v}.0" != version raise ArgumentError, <<-ERROR #{name}: #{type} version mismatch: #{version} to #{v} #{line.gsub(/^/, '> ')} @@ -420,8 +440,6 @@ define_posix_props(data) POSIX_NAMES.each do |name| if name == 'XPosixPunct' make_const(name, data[name], "[[:Punct:]]") - elsif name == 'Punct' - make_const(name, data[name], "") else make_const(name, data[name], "[[:#{name}:]]") end @@ -464,11 +482,7 @@ struct uniname2ctype_struct { }; #define uniname2ctype_offset(str) offsetof(struct uniname2ctype_pool_t, uniname2ctype_pool_##str) -static const struct uniname2ctype_struct *uniname2ctype_p( -#if !(/*ANSI*/+0) /* if ANSI, old style not to conflict with generated prototype */ - const char *, unsigned int -#endif -); +static const struct uniname2ctype_struct *uniname2ctype_p(register const char *str, register size_t len); %} struct uniname2ctype_struct; %% @@ -486,7 +500,11 @@ end output.ifdef :USE_UNICODE_PROPERTIES do props.each do |name| i += 1 - name = normalize_propname(name) + name = if name.start_with?('InCB') + name.downcase.gsub(/_/, '=') + else + normalize_propname(name) + end name_to_index[name] = i puts "%-40s %3d" % [name + ',', i] end @@ -547,6 +565,47 @@ output.restore if header require 'tempfile' + def diff_args(diff) + ok = IO.popen([diff, "-DDIFF_TEST", IO::NULL, "-"], "r+") do |f| + f.puts "Test for diffutils 3.8" + f.close_write + /^#if/ =~ f.read + end + if ok + proc {|macro, *inputs| + [diff, "-D#{macro}", *inputs] + } + else + IO.popen([diff, "--old-group-format=%<", "--new-group-format=%>", IO::NULL, IO::NULL], err: %i[child out], &:read) + unless $?.success? + abort "#{$0}: #{diff} -D does not work" + end + warn "Avoiding diffutils 3.8 bug#61193" + proc {|macro, *inputs| + [diff] + [ + "--old-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#endif /* ! @ */\n", + + "--new-group-format=" \ + "#ifdef @\n" \ + "%>" \ + "#endif /* @ */\n", + + "--changed-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#else /* @ */\n" \ + "%>" \ + "#endif /* @ */\n" + ].map {|opt| opt.gsub(/@/) {macro}} + inputs + } + end + end + + ifdef = diff_args(diff || "diff") + NAME2CTYPE = %w[gperf -7 -c -j1 -i1 -t -C -P -T -H uniname2ctype_hash -Q uniname2ctype_pool -N uniname2ctype_p] fds = [] @@ -556,9 +615,8 @@ if header IO.popen([*NAME2CTYPE, out: tmp], "w") {|f| output.show(f, *syms)} end while syms.pop fds.each(&:close) - ff = nil - IO.popen(%W[diff -DUSE_UNICODE_AGE_PROPERTIES #{fds[1].path} #{fds[0].path}], "r") {|age| - IO.popen(%W[diff -DUSE_UNICODE_PROPERTIES #{fds[2].path} -], "r", in: age) {|f| + IO.popen(ifdef["USE_UNICODE_AGE_PROPERTIES", fds[1].path, fds[0].path], "r") {|age| + IO.popen(ifdef["USE_UNICODE_PROPERTIES", fds[2].path, "-"], "r", in: age) {|f| ansi = false f.each {|line| if /ANSI-C code produced by gperf/ =~ line @@ -567,7 +625,7 @@ if header line.sub!(/\/\*ANSI\*\//, '1') if ansi line.gsub!(/\(int\)\((?:long|size_t)\)&\(\(struct uniname2ctype_pool_t \*\)0\)->uniname2ctype_pool_(str\d+),\s+/, 'uniname2ctype_offset(\1), ') - if ff = (!ff ? /^(uniname2ctype_hash) /=~line : /^\}/!~line) # no line can match both, exclusive flip-flop + if line.start_with?("uniname2ctype_hash\s") ... line.start_with?("}") line.sub!(/^( *(?:register\s+)?(.*\S)\s+hval\s*=\s*)(?=len;)/, '\1(\2)') end puts line diff --git a/tool/expand-config.rb b/tool/expand-config.rb index 81ffa6cb98..ac0ffbfd41 100755 --- a/tool/expand-config.rb +++ b/tool/expand-config.rb @@ -7,7 +7,17 @@ config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)$/, '\1true\2') rbconfig = Module.new {module_eval(config, conffile)}::RbConfig config = $expand ? rbconfig::CONFIG : rbconfig::MAKEFILE_CONFIG config["RUBY_RELEASE_DATE"] ||= - File.read(File.expand_path("../../version.h", __FILE__))[/^\s*#\s*define\s+RUBY_RELEASE_DATE\s+"(.*)"/, 1] + [ + ["revision.h"], + ["../../revision.h", __FILE__], + ["../../version.h", __FILE__], + ].find do |hdr, dir| + hdr = File.expand_path(hdr, dir) if dir + if date = File.read(hdr)[/^\s*#\s*define\s+RUBY_RELEASE_DATE(?:TIME)?\s+"([0-9-]*)/, 1] + break date + end +rescue +end while /\A(\w+)=(.*)/ =~ ARGV[0] config[$1] = $2 @@ -16,7 +26,7 @@ while /\A(\w+)=(.*)/ =~ ARGV[0] end if $output - output = open($output, "wb", $mode &&= $mode.oct) + output = File.open($output, "wb", $mode &&= $mode.oct) output.chmod($mode) if $mode else output = STDOUT diff --git a/tool/extlibs.rb b/tool/extlibs.rb index f021a2bf01..cef6712833 100755 --- a/tool/extlibs.rb +++ b/tool/extlibs.rb @@ -5,19 +5,45 @@ require 'digest' require_relative 'downloader' -require_relative 'lib/colorize' +begin + require_relative 'lib/colorize' +rescue LoadError +end class ExtLibs - def initialize + unless defined?(Colorize) + class Colorize + def pass(str) str; end + def fail(str) str; end + end + end + + class Vars < Hash + def pattern + /\$\((#{Regexp.union(keys)})\)/ + end + + def expand(str) + if empty? + str + else + str.gsub(pattern) {self[$1]} + end + end + end + + def initialize(mode = :all, cache_dir: nil) + @mode = mode + @cache_dir = cache_dir @colorize = Colorize.new end def cache_file(url, cache_dir) - Downloader.cache_file(url, nil, :cache_dir => cache_dir) + Downloader.cache_file(url, nil, cache_dir).to_path end def do_download(url, cache_dir) - Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) + Downloader.download(url, nil, nil, nil, cache_dir: cache_dir) end def do_checksum(cache, chksums) @@ -76,10 +102,50 @@ class ExtLibs $stdout.puts "applying #{patch} under #{dest}" $stdout.flush end - Process.wait(Process.spawn("patch", "-d", dest, "-i", patch, *args)) + Process.wait(Process.spawn(ENV.fetch("PATCH", "patch"), "-d", dest, "-i", patch, *args)) $?.success? or raise "failed to patch #{patch}" end + def do_link(file, src, dest) + file = File.join(dest, file) + if (target = src).start_with?("/") + target = File.join([".."] * file.count("/"), src) + end + return unless File.exist?(File.expand_path(target, File.dirname(file))) + File.unlink(file) rescue nil + begin + File.symlink(target, file) + rescue + else + if $VERBOSE + $stdout.puts "linked #{target} to #{file}" + $stdout.flush + end + return + end + begin + src = src.sub(/\A\//, '') + File.copy_stream(src, file) + rescue + if $VERBOSE + $stdout.puts "failed to link #{src} to #{file}: #{$!.message}" + end + else + if $VERBOSE + $stdout.puts "copied #{src} to #{file}" + end + end + end + + def do_exec(command, dir, dest) + dir = dir ? File.join(dest, dir) : dest + if $VERBOSE + $stdout.puts "running #{command.dump} under #{dir}" + $stdout.flush + end + system(command, chdir: dir) or raise "failed #{command.dump}" + end + def do_command(mode, dest, url, cache_dir, chksums) extracted = false base = /.*(?=\.tar(?:\.\w+)?\z)/ @@ -106,7 +172,78 @@ class ExtLibs extracted end - def run(argv) + def process(list) + mode = @mode + cache_dir = @cache_dir + after_extract = (mode == :all or mode == :patch) + success = true + if $VERBOSE + $stdout.puts "downloading for #{list}" + $stdout.flush + end + vars = Vars.new + extracted = false + dest = File.dirname(list) + url = chksums = nil + File.foreach(list) do |line| + line.sub!(/\s*#.*/, '') + if /^(\w+)\s*=\s*(.*)/ =~ line + vars[$1] = vars.expand($2) + next + end + if chksums + chksums.concat(line.split) + elsif /^\t/ =~ line + if extracted and after_extract + patch, *args = line.split.map {|s| vars.expand(s)} + do_patch(dest, patch, args) + end + next + elsif /^!\s*(?:chdir:\s*([^|\s]+)\|\s*)?(.*)/ =~ line + if extracted and after_extract + command = vars.expand($2.strip) + chdir = $1 and chdir = vars.expand(chdir) + do_exec(command, chdir, dest) + end + next + elsif /->/ =~ line + if extracted and after_extract + link, file = $`.strip, $'.strip + do_link(vars.expand(link), vars.expand(file), dest) + end + next + else + url, *chksums = line.split(' ') + end + if chksums.last == '\\' + chksums.pop + next + end + unless url + chksums = nil + next + end + url = vars.expand(url) + begin + extracted = do_command(mode, dest, url, cache_dir, chksums) + rescue => e + warn defined?(e.full_message) ? e.full_message : e.message + success = false + end + url = chksums = nil + end + success + end + + def process_under(dir) + success = true + Dir.glob("#{dir}/**/extlibs") do |list| + success &= process(list) + end + success + end + + def self.run(argv) cache_dir = nil mode = :all until argv.empty? @@ -136,49 +273,10 @@ class ExtLibs argv.shift end - success = true - argv.each do |dir| - Dir.glob("#{dir}/**/extlibs") do |list| - if $VERBOSE - $stdout.puts "downloading for #{list}" - $stdout.flush - end - extracted = false - dest = File.dirname(list) - url = chksums = nil - IO.foreach(list) do |line| - line.sub!(/\s*#.*/, '') - if chksums - chksums.concat(line.split) - elsif /^\t/ =~ line - if extracted and (mode == :all or mode == :patch) - patch, *args = line.split - do_patch(dest, patch, args) - end - next - else - url, *chksums = line.split(' ') - end - if chksums.last == '\\' - chksums.pop - next - end - next unless url - begin - extracted = do_command(mode, dest, url, cache_dir, chksums) - rescue => e - warn e.inspect - success = false - end - url = chksums = nil - end - end + extlibs = new(mode, cache_dir: cache_dir) + argv.inject(true) do |success, dir| + success & extlibs.process_under(dir) end - success - end - - def self.run(argv) - self.new.run(argv) end end diff --git a/tool/fake.rb b/tool/fake.rb index 42174052e2..2c458985d8 100644 --- a/tool/fake.rb +++ b/tool/fake.rb @@ -9,6 +9,15 @@ class File end end +[[libpathenv, "."], [preloadenv, libruby_so]].each do |env, path| + env or next + e = ENV[env] or next + e = e.split(File::PATH_SEPARATOR) + path = File.realpath(path, builddir) rescue next + e.delete(path) or next + ENV[env] = (e.join(File::PATH_SEPARATOR) unless e.empty?) +end + static = !!(defined?($static) && $static) $:.unshift(builddir) posthook = proc do @@ -24,6 +33,7 @@ posthook = proc do untrace_var(:$ruby, posthook) end prehook = proc do |extmk| +=begin pat = %r[(?:\A(?:\w:|//[^/]+)|\G)/[^/]*] dir = builddir.scan(pat) pwd = Dir.pwd.scan(pat) @@ -35,6 +45,7 @@ prehook = proc do |extmk| builddir = File.join((pwd.empty? ? ["."] : [".."]*pwd.size) + dir) builddir = "." if builddir.empty? end +=end join = proc {|*args| File.join(*args).sub!(/\A(?:\.\/)*/, '')} $topdir ||= builddir $top_srcdir ||= (File.identical?(top_srcdir, dir = join[$topdir, srcdir]) ? @@ -43,13 +54,17 @@ prehook = proc do |extmk| $extout_prefix = '$(extout)$(target_prefix)/' config = RbConfig::CONFIG mkconfig = RbConfig::MAKEFILE_CONFIG + $builtruby ||= File.join(builddir, config['RUBY_INSTALL_NAME'] + config['EXEEXT']) RbConfig.fire_update!("builddir", builddir) RbConfig.fire_update!("buildlibdir", builddir) RbConfig.fire_update!("libdir", builddir) + RbConfig.fire_update!("prefix", $topdir) RbConfig.fire_update!("top_srcdir", $top_srcdir ||= top_srcdir) RbConfig.fire_update!("extout", $extout) RbConfig.fire_update!("rubyhdrdir", "$(top_srcdir)/include") RbConfig.fire_update!("rubyarchhdrdir", "$(extout)/include/$(arch)") + RbConfig.fire_update!("rubyarchdir", "$(extout)/$(arch)") + RbConfig.fire_update!("rubylibdir", "$(extout)/common") RbConfig.fire_update!("libdirname", "buildlibdir") trace_var(:$ruby, posthook) untrace_var(:$extmk, prehook) diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb index ae3068d35c..127ea236f3 100755 --- a/tool/fetch-bundled_gems.rb +++ b/tool/fetch-bundled_gems.rb @@ -1,6 +1,16 @@ -#!ruby -an +#!ruby -alnF\s+|#.* BEGIN { require 'fileutils' + require_relative 'lib/colorize' + + color = Colorize.new + + if ARGV.first.start_with?("BUNDLED_GEMS=") + bundled_gems = ARGV.shift[13..-1] + sep = bundled_gems.include?(",") ? "," : " " + bundled_gems = bundled_gems.split(sep) + bundled_gems = nil if bundled_gems.empty? + end dir = ARGV.shift ARGF.eof? @@ -8,20 +18,37 @@ BEGIN { Dir.chdir(dir) } -n, v, u = $F -case n -when "minitest" - v = "master" -when "test-unit" -else - v = "v" + v +n, v, u, r = $F + +next unless n +next if bundled_gems&.all? {|pat| !File.fnmatch?(pat, n)} + +unless File.exist?("#{n}/.git") + puts "retrieving #{color.notice(n)} ..." + system(*%W"git clone --depth=1 --no-tags #{u} #{n}") or abort end -if File.directory?(n) - puts "updating #{n} ..." - system(*%W"git fetch", chdir: n) or abort +if r + puts "fetching #{color.notice(r)} ..." + system("git", "fetch", "origin", r, chdir: n) or abort + c = r else - puts "retrieving #{n} ..." - system(*%W"git clone #{u} #{n}") or abort + c = ["v#{v}", v].find do |c| + puts "fetching #{color.notice(c)} ..." + system("git", "fetch", "origin", "refs/tags/#{c}:refs/tags/#{c}", chdir: n) + end or abort +end + +checkout = %w"git -c advice.detachedHead=false checkout" +info = %[, r=#{color.info(r)}] if r +puts "checking out #{color.notice(c)} (v=#{color.info(v)}#{info}) ..." +unless system(*checkout, c, "--", chdir: n) + abort if r or !system(*checkout, v, "--", chdir: n) +end + +if r + unless File.exist? "#{n}/#{n}.gemspec" + require_relative "lib/bundled_gem" + BundledGem.dummy_gemspec("#{n}/#{n}.gemspec") + end end -system(*%W"git checkout #{v}", chdir: n) or abort diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb index 68c660efaa..2de8379606 100755 --- a/tool/file2lastrev.rb +++ b/tool/file2lastrev.rb @@ -8,37 +8,47 @@ require 'optparse' # this file run with BASERUBY, which may be older than 1.9, so no # require_relative require File.expand_path('../lib/vcs', __FILE__) +require File.expand_path('../lib/output', __FILE__) Program = $0 -@output = nil -def self.output=(output) - if @output and @output != output +@format = nil +def self.format=(format) + if @format and @format != format raise "you can specify only one of --changed, --revision.h and --doxygen" end - @output = output + @format = format end @suppress_not_found = false @limit = 20 +@output = Output.new -format = '%Y-%m-%dT%H:%M:%S%z' -srcdir = nil -parser = OptionParser.new {|opts| +time_format = '%Y-%m-%dT%H:%M:%S%z' +vcs = nil +create_only = false +OptionParser.new {|opts| + opts.banner << " paths..." + vcs_options = VCS.define_options(opts) + opts.new {@output.def_options(opts)} + srcdir = nil + opts.new opts.on("--srcdir=PATH", "use PATH as source directory") do |path| + abort "#{File.basename(Program)}: srcdir is already set" if srcdir srcdir = path + @output.vpath.add(srcdir) end opts.on("--changed", "changed rev") do - self.output = :changed + self.format = :changed end opts.on("--revision.h", "RUBY_REVISION macro") do - self.output = :revision_h + self.format = :revision_h end opts.on("--doxygen", "Doxygen format") do - self.output = :doxygen + self.format = :doxygen end opts.on("--modified[=FORMAT]", "modified time") do |fmt| - self.output = :modified - format = fmt if fmt + self.format = :modified + time_format = fmt if fmt end opts.on("--limit=NUM", "limit branch name length (#@limit)", Integer) do |n| @limit = n @@ -46,40 +56,28 @@ parser = OptionParser.new {|opts| opts.on("-q", "--suppress_not_found") do @suppress_not_found = true end + opts.order! rescue abort "#{File.basename(Program)}: #{$!}\n#{opts}" + begin + vcs = VCS.detect(srcdir || ".", vcs_options, opts.new) + rescue VCS::NotFoundError => e + abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found + opts.remove + (vcs = VCS::Null.new(nil)).set_options(vcs_options) + if @format == :revision_h + create_only = true # don't overwrite existing revision.h when .git doesn't exist + end + end } -parser.parse! rescue abort "#{File.basename(Program)}: #{$!}\n#{parser}" -vcs = nil -@output = - case @output +formatter = + case @format when :changed, nil Proc.new {|last, changed| - changed + changed || "" } when :revision_h Proc.new {|last, changed, modified, branch, title| - short = vcs.short_revision(last) - if /[^\x00-\x7f]/ =~ title and title.respond_to?(:force_encoding) - title = title.dup.force_encoding("US-ASCII") - end - [ - "#define RUBY_REVISION #{short.inspect}", - ("#define RUBY_FULL_REVISION #{last.inspect}" unless short == last), - if branch - e = '..' - limit = @limit - name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} - name = name.dump.sub(/\\#/, '#') - "#define RUBY_BRANCH_NAME #{name}" - end, - if title - title = title.dump.sub(/\\#/, '#') - "#define RUBY_LAST_COMMIT_TITLE #{title}" - end, - if modified - modified.utc.strftime('#define RUBY_RELEASE_DATETIME "%FT%TZ"') - end, - ].compact + vcs.revision_header(last, modified, modified, branch, title, limit: @limit).join("\n") } when :doxygen Proc.new {|last, changed| @@ -87,27 +85,22 @@ vcs = nil } when :modified Proc.new {|last, changed, modified| - modified.strftime(format) + modified.strftime(time_format) } else - raise "unknown output format `#{@output}'" + raise "unknown output format `#{@format}'" end -srcdir ||= File.dirname(File.dirname(Program)) -begin - vcs = VCS.detect(srcdir) -rescue VCS::NotFoundError => e - abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found -else - ok = true - (ARGV.empty? ? [nil] : ARGV).each do |arg| - begin - puts @output[*vcs.get_revisions(arg)] - rescue => e - next if @suppress_not_found and VCS::NotFoundError === e - warn "#{File.basename(Program)}: #{e.message}" - ok = false - end +ok = true +(ARGV.empty? ? [nil] : ARGV).each do |arg| + begin + data = formatter[*vcs.get_revisions(arg)] + data.sub!(/(?<!\A|\n)\z/, "\n") + @output.write(data, overwrite: true, create_only: create_only) + rescue => e + next if @suppress_not_found and VCS::NotFoundError === e + warn "#{File.basename(Program)}: #{e.message}" + ok = false end - exit ok end +exit ok diff --git a/tool/format-release b/tool/format-release index e3ad5a9563..8bb6154243 100755 --- a/tool/format-release +++ b/tool/format-release @@ -1,45 +1,39 @@ #!/usr/bin/env ruby -# -# Tool to update w.r-l.o files on release -# -# This receives `make dist`'s output, and prints -# -# * links and digests of tarballs for releate article -# * patch for below data files -# * _data/branches.yml -# * _data/downloads.yml -# * _data/releases.yml -# -# To apply the patch, `patch -p1 < patch.txt` -require "diffy" + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "diffy" +end + +require "open-uri" +require "yaml" +require_relative "./ruby-version" Diffy::Diff.default_options.merge!( include_diff_info: true, context: 1, ) -WWW = Dir.pwd # workdir of https://github.com/ruby/www.ruby-lang.org class Tarball - attr_reader :version, :sha256 + attr_reader :version, :size, :sha1, :sha256, :sha512 - def initialize(path, size, sha1, sha256, sha512) - /\A\/(?:[^\/\n]*\/)*(?:ruby-(?<ver>(?<xy>\d+\.\d+)\.\d+(?:-(?:preview|rc)\d+)?)[0-9a-z.\-]+)/ =~ path \ - or raise "tarball path doesn't mach regexp" - @path = path + def initialize(version, url, size, sha1, sha256, sha512) + @url = url @size = size @sha1 = sha1 @sha256 = sha256 @sha512 = sha512 - @version = ver - @xy = xy + @version = version + @xy = version[/\A\d+\.\d+/] end - def gz?; @path.end_with?('.gz'); end - def zip?; @path.end_with?('.zip'); end - def bz2?; @path.end_with?('.bz2'); end - def xz?; @path.end_with?('.xz'); end + def gz?; @url.end_with?('.gz'); end + def zip?; @url.end_with?('.zip'); end + def xz?; @url.end_with?('.xz'); end - def ext; @path[/(?:zip|tar\.(?:gz|bz2|xz))\z/]; end + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end def to_md <<eom @@ -57,37 +51,50 @@ eom # SHA1: 21f62c369661a2ab1b521fd2fa8191a4273e12a1 # SHA256: 97cea8aa63dfa250ba6902b658a7aa066daf817b22f82b7ee28f44aec7c2e394 # SHA512: 1e2042324821bb4e110af7067f52891606dcfc71e640c194ab1c117f0b941550e0b3ac36ad3511214ac80c536b9e5cfaf8789eec74cf56971a832ea8fc4e6d94 - def self.parse(io) - ary = [] - ver = nil - io.read.scan(/^\* .*(?:\n .*)*/) do |unit| - path, size, sha1, sha256, sha512 = unit.lines.map(&:chomp) - path.slice!(0, 2) - size = size[/\d+/].to_i - sha1 = sha1[/\h{40}/] - sha256 = sha256[/\h{64}/] - sha512 = sha512[/\h{128}/] - tarball = Tarball.new(path, size, sha1, sha256, sha512) - if ver - raise "mixed versions" if tarball.version != ver - else - ver = tarball.version - end - ary << tarball + def self.parse(wwwdir, version, rubydir) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + teeny = Integer($3) + + uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" + info = YAML.unsafe_load(URI(uri).read) + if info.size != 1 + raise "unexpected info.yml '#{uri}'" + end + tarballs = [] + info[0]["size"].each_key do |ext| + url = info[0]["url"][ext] + size = info[0]["size"][ext] + sha1 = info[0]["sha1"][ext] + sha256 = info[0]["sha256"][ext] + sha512 = info[0]["sha512"][ext] + tarball = Tarball.new(version, url, size, sha1, sha256, sha512) + tarballs << tarball end - xy = ver[/\A\d+\.\d+/] - ary.each do |tarball| - puts tarball.to_md + + if teeny == 0 + # show diff shortstat + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) + stat = `git -C #{rubydir} diff -l0 --shortstat #{prev_tag}..#{tag}` + files_changed, insertions, deletions = stat.scan(/\d+/) end - update_branches_yml(ver, xy) - update_downloads_yml(ver, xy) - update_releases_yml(ver, xy, ary) - ary + + xy = version[/\A\d+\.\d+/] + #puts "## Download\n\n" + #tarballs.each do |tarball| + # puts tarball.to_md + #end + update_branches_yml(version, xy, wwwdir) + update_downloads_yml(version, xy, wwwdir) + update_releases_yml(version, xy, tarballs, wwwdir, files_changed, insertions, deletions) + tarballs end - def self.update_branches_yml(ver, xy) + def self.update_branches_yml(ver, xy, wwwdir) filename = "_data/branches.yml" - orig_data = File.read(File.join(WWW, filename)) + orig_data = File.read(File.join(wwwdir, filename)) data = orig_data.dup if data.include?("\n- name: #{xy}\n") data.sub!(/\n- name: #{Regexp.escape(xy)}\n(?: .*\n)*/) do |node| @@ -121,9 +128,9 @@ eom end end - def self.update_downloads_yml(ver, xy) + def self.update_downloads_yml(ver, xy, wwwdir) filename = "_data/downloads.yml" - orig_data = File.read(File.join(WWW, filename)) + orig_data = File.read(File.join(wwwdir, filename)) data = orig_data.dup if /^preview:\n\n(?: .*\n)* - #{Regexp.escape(xy)}\./ =~ data @@ -148,29 +155,55 @@ eom end end - def self.update_releases_yml(ver, xy, ary) + def self.update_releases_yml(ver, xy, ary, wwwdir, files_changed, insertions, deletions) filename = "_data/releases.yml" - orig_data = File.read(File.join(WWW, filename)) + orig_data = File.read(File.join(wwwdir, filename)) data = orig_data.dup - date = Time.now + date = Time.now.utc # use utc to use previous day in midnight entry = <<eom - version: #{ver} date: #{date.strftime("%Y-%m-%d")} post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ +eom + + if /\.0(?:-\w+)?\z/ =~ ver + # preview, rc, or first release + entry <<= <<eom + tag: #{RubyVersion.tag(ver)} + stats: + files_changed: #{files_changed} + insertions: #{insertions} + deletions: #{deletions} +eom + end + + entry <<= <<eom url: gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip - bz2: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.bz2 xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz + size: + gz: #{ary.find{|x|x.gz? }.size} + zip: #{ary.find{|x|x.zip?}.size} + xz: #{ary.find{|x|x.xz? }.size} + sha1: + gz: #{ary.find{|x|x.gz? }.sha1} + zip: #{ary.find{|x|x.zip?}.sha1} + xz: #{ary.find{|x|x.xz? }.sha1} sha256: gz: #{ary.find{|x|x.gz? }.sha256} zip: #{ary.find{|x|x.zip?}.sha256} - bz2: #{ary.find{|x|x.bz2?}.sha256} xz: #{ary.find{|x|x.xz? }.sha256} + sha512: + gz: #{ary.find{|x|x.gz? }.sha512} + zip: #{ary.find{|x|x.zip?}.sha512} + xz: #{ary.find{|x|x.xz? }.sha512} eom if data.include?("\n- version: #{ver}\n") + # update existing entry + data.sub!(/\n- version: #{ver}\n(^ .*\n)*\n/, "\n#{entry}\n") elsif data.sub!(/\n# #{Regexp.escape(xy)} series\n/, "\\&\n#{entry}") else data.sub!(/^$/, "\n# #{xy} series\n\n#{entry}") @@ -185,11 +218,11 @@ eom diff.each_with_index do |line, index| case index when 0 - line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do "--- a/#{filename}\t#{$2}" end when 1 - line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do "+++ b/#{filename}\t#{$2}" end end @@ -199,6 +232,14 @@ eom end def main - Tarball.parse(ARGF) + wwwdir = ARGV.shift + version = ARGV.shift + rubydir = ARGV.shift + unless rubydir + STDERR.puts "usage: format-release <dir-of-w.r-l.o> <version> <ruby-dir>" + exit + end + Tarball.parse(wwwdir, version, rubydir) end + main diff --git a/tool/gem-unpack.rb b/tool/gem-unpack.rb deleted file mode 100755 index 0ddcea0704..0000000000 --- a/tool/gem-unpack.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'rubygems' -require 'rubygems/package' - -# This library is used by "make extract-gems" to -# unpack bundled gem files. - -def Gem.unpack(file, dir = nil) - pkg = Gem::Package.new(file) - spec = pkg.spec - target = spec.full_name - target = File.join(dir, target) if dir - pkg.extract_files target - spec_file = File.join(target, "#{spec.name}.gemspec") - open(spec_file, 'wb') do |f| - f.print spec.to_ruby - end - puts "Unpacked #{file}" -end diff --git a/tool/gen-github-release.rb b/tool/gen-github-release.rb new file mode 100755 index 0000000000..cdb66080d9 --- /dev/null +++ b/tool/gen-github-release.rb @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby + +if ARGV.size < 2 + puts "Usage: #{$0} <from version tag> <to version tag> [--no-dry-run]" + puts " : if --no-dry-run is specified, it will create a release on GitHub" + exit 1 +end + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "octokit" + gem "faraday-retry" + gem "nokogiri" +end + +require "open-uri" + +Octokit.configure do |c| + c.access_token = ENV['GITHUB_TOKEN'] + c.auto_paginate = true + c.per_page = 100 +end + +client = Octokit::Client.new + +note = "## What's Changed\n\n" + +notes = [] + +diff = client.compare("ruby/ruby", ARGV[0], ARGV[1]) +diff[:commits].each do |c| + if c[:commit][:message] =~ /\[(Backport|Feature|Bug) #(\d*)\]/ + url = "https://bugs.ruby-lang.org/issues/#{$2}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ - Ruby master - Ruby Issue Tracking System/, "") + elsif c[:commit][:message] =~ /\(#(\d*)\)/ + url = "https://github.com/ruby/ruby/pull/#{$1}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ · ruby\/ruby · GitHub/, "") + else + next + end + notes << "* [#{title}](#{url})" +rescue OpenURI::HTTPError + puts "Error: #{url}" +end + +notes.uniq! + +note << notes.join("\n") + +note << "\n\n" +note << "Note: This list is automatically generated by tool/gen-github-release.rb. Because of this, some commits may be missing.\n\n" +note << "## Full Changelog\n\n" +note << "https://github.com/ruby/ruby/compare/#{ARGV[0]}...#{ARGV[1]}\n\n" + +if ARGV[2] == "--no-dry-run" + name = ARGV[1].gsub(/^v/, "").gsub(/_/, ".") + prerelease = ARGV[1].match?(/rc|preview/) ? true : false + client.create_release("ruby/ruby", ARGV[1], name: name, body: note, make_latest: "false", prerelease: prerelease) + puts "Created a release: https://github.com/ruby/ruby/releases/tag/#{ARGV[1]}" +else + puts note +end diff --git a/tool/gen-mailmap.rb b/tool/gen-mailmap.rb new file mode 100755 index 0000000000..0cdedf1e7b --- /dev/null +++ b/tool/gen-mailmap.rb @@ -0,0 +1,47 @@ +#!/usr/bin/env ruby + +require "open-uri" +require "yaml" + +EMAIL_YML_URL = "https://cdn.jsdelivr.net/gh/ruby/git.ruby-lang.org/config/email.yml" + +email_yml = URI(EMAIL_YML_URL).read.sub(/\A(?:#.*\n)+/, "").gsub(/^# +(.+)$/) { $1 + ": []" } + +email = YAML.load(email_yml) +YAML.load(DATA.read).each do |name, mails| + email[name] ||= [] + email[name] |= mails +end + +File.open(File.join(__dir__, "../.mailmap"), "w") do |f| + email.each do |name, mails| + canonical = "#{ name }@ruby-lang.org" + mails.delete(canonical) + svn = "#{ name }@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" + ((mails | [canonical]) + [svn]).each do |mail| + f.puts "#{ name } <#{ canonical }> <#{ mail }>" + end + end +end + +puts "You'll see canonical names (SVN account names) by the following commands:" +puts +puts " git shortlog -ce" +puts " git log --pretty=format:'%cN <%cE>'" +puts " git log --use-mailmap --pretty=full" + +__END__ +git: +- svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e +- "(no author)@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" +kazu: +- znz@users.noreply.github.com +marcandre: +- github@marc-andre.ca +mrkn: +- mrkn@users.noreply.github.com +- muraken@b2dd03c8-39d4-4d8f-98ff-823fe69b080e +naruse: +- nurse@users.noreply.github.com +tenderlove: +- tenderlove@github.com diff --git a/tool/generic_erb.rb b/tool/generic_erb.rb index 6af995fc13..b6cb51474e 100644 --- a/tool/generic_erb.rb +++ b/tool/generic_erb.rb @@ -5,57 +5,31 @@ require 'erb' require 'optparse' -require_relative 'lib/vpath' -require_relative 'lib/colorize' +require_relative 'lib/output' -vpath = VPath.new -timestamp = nil -output = nil -ifchange = nil +out = Output.new source = false -color = nil templates = [] ARGV.options do |o| - o.on('-t', '--timestamp[=PATH]') {|v| timestamp = v || true} - o.on('-i', '--input=PATH') {|v| template << v} - o.on('-o', '--output=PATH') {|v| output = v} - o.on('-c', '--[no-]if-change') {|v| ifchange = v} + o.on('-i', '--input=PATH') {|v| templates << v} o.on('-x', '--source') {source = true} - o.on('--color') {color = true} - vpath.def_options(o) + out.def_options(o) o.order!(ARGV) templates << (ARGV.shift or abort o.to_s) if templates.empty? end -color = Colorize.new(color) -unchanged = color.pass("unchanged") -updated = color.fail("updated") + +# Used in prelude.c.tmpl and unicode_norm_gen.tmpl +output = out.path +vpath = out.vpath + +# A hack to prevent "unused variable" warnings +output, vpath = output, vpath result = templates.map do |template| - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(File.read(template), trim_mode: '%-') - else - erb = ERB.new(File.read(template), nil, '%-') - end + erb = ERB.new(File.read(template), trim_mode: '%-') erb.filename = template source ? erb.src : proc{erb.result(binding)}.call end result = result.size == 1 ? result[0] : result.join("") -if output - if ifchange and (vpath.open(output, "rb") {|f| f.read} rescue nil) == result - puts "#{output} #{unchanged}" - else - open(output, "wb") {|f| f.print result} - puts "#{output} #{updated}" - end - if timestamp - if timestamp == true - dir, base = File.split(output) - timestamp = File.join(dir, ".time." + base) - end - File.open(timestamp, 'a') {} - File.utime(nil, nil, timestamp) - end -else - print result -end +out.write(result) diff --git a/tool/gperf.sed b/tool/gperf.sed index 6b3e1980be..9550a522b9 100644 --- a/tool/gperf.sed +++ b/tool/gperf.sed @@ -1,21 +1,3 @@ -/ANSI-C code/{ - h - s/.*/ANSI:offset:/ - x -} -/\/\*!ANSI{\*\//{ - G - s/\/\*!ANSI{\*\/\(.*\)\/\*}!ANSI\*\/\(.*\)\nANSI:.*/\/\*\1\*\/\2/ -} -s/(int)([a-z_]*)&((struct \([a-zA-Z_0-9][a-zA-Z_0-9]*\)_t *\*)0)->\1_str\([1-9][0-9]*\),/gperf_offsetof(\1, \2),/g -/^#line/{ - G - x - s/:offset:/:/ - x - s/\(.*\)\(\n\).*:offset:.*/#define gperf_offsetof(s, n) (short)offsetof(struct s##_t, s##_str##n)\2\1/ - s/\n[^#].*// -} /^[a-zA-Z_0-9]*hash/,/^}/{ s/ hval = / hval = (unsigned int)/ s/ return / return (unsigned int)/ diff --git a/tool/id2token.rb b/tool/id2token.rb index d12ea9c08e..cf73095842 100755 --- a/tool/id2token.rb +++ b/tool/id2token.rb @@ -5,18 +5,15 @@ BEGIN { require 'optparse' - require_relative 'lib/vpath' - vpath = VPath.new - header = nil opt = OptionParser.new do |o| - vpath.def_options(o) - header = o.order!(ARGV).shift + o.order!(ARGV) end or abort opt.opt_s TOKENS = {} - h = vpath.read(header) rescue abort("#{header} not found in #{vpath.inspect}") - h.scan(/^#define\s+RUBY_TOKEN_(\w+)\s+(\d+)/) do |token, id| + defs = File.join(File.dirname(File.dirname(__FILE__)), "defs/id.def") + ids = eval(File.read(defs), nil, defs) + ids[:token_op].each do |_id, _op, token, id| TOKENS[token] = id end diff --git a/tool/ifchange b/tool/ifchange index 050c8d8f26..9e4a89533c 100755 --- a/tool/ifchange +++ b/tool/ifchange @@ -3,13 +3,29 @@ # Used in generating revision.h via Makefiles. +help() { + cat <<HELP +usage: $0 [options] target new-file +options: + --timestamp[=file] touch timestamp file. (default: prefixed with ".time". + under the directory of the target) + --keep[=suffix] keep old file with suffix. (default: '.old') + --empty assume unchanged if the new file is empty. + --color[=always|auto|never] colorize output. +HELP +} + set -e timestamp= keepsuffix= -empty= +srcavail=f color=auto until [ $# -eq 0 ]; do case "$1" in + --) + shift + break; + ;; --timestamp) timestamp=. ;; @@ -23,7 +39,7 @@ until [ $# -eq 0 ]; do keepsuffix=`expr \( "$1" : '[^=]*=\(.*\)' \)` ;; --empty) - empty=yes + srcavail=s ;; --color) color=always @@ -34,6 +50,14 @@ until [ $# -eq 0 ]; do --debug) set -x ;; + --help) + help + exit + ;; + --*) + echo "$0: unknown option: $1" 1>&2 + exit 1 + ;; *) break ;; @@ -41,6 +65,11 @@ until [ $# -eq 0 ]; do shift done +if [ "$#" != 2 ]; then + help + exit 1 +fi + target="$1" temp="$2" if [ "$temp" = - ]; then @@ -68,7 +97,7 @@ fi targetdir= case "$target" in */*) targetdir=`dirname "$target"`;; esac -if [ -f "$target" -a ! -${empty:+f}${empty:-s} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then +if [ -f "$target" -a ! -${srcavail} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then echo "$target ${msg_unchanged}unchanged${msg_reset}" rm -f "$temp" else diff --git a/tool/intern_ids.rb b/tool/intern_ids.rb new file mode 100755 index 0000000000..20483195e0 --- /dev/null +++ b/tool/intern_ids.rb @@ -0,0 +1,35 @@ +#!/usr/bin/ruby -sp +# $ ruby -i tool/intern_ids.rb -prefix=_ foo.c + +BEGIN { + $prefix ||= nil + + defs = File.join(File.dirname(__dir__), "defs/id.def") + ids = eval(File.read(defs), binding, defs) + table = {} + ids[:predefined].each {|v, t| table[t] = "id#{v}"} + ids[:token_op].each {|v, t, *| table[t] = "id#{v}"} + predefined = table.keys +} + +$_.gsub!(/rb_intern\("([^\"]+)"\)/) do + token = $1 + table[token] ||= "id" + id2varname(token, $prefix) +end + +END { + predefined.each {|t| table.delete(t)} + unless table.empty? + table = table.sort_by {|t, v| v} + + # Append at the last, then edit and move appropriately. + puts + puts "==== defs" + table.each {|t, v| puts "static ID #{v};"} + puts ">>>>" + puts + puts "==== init" + table.each {|t, v|puts "#{v} = rb_intern_const(\"#{t}\");"} + puts ">>>>" + end +} diff --git a/tool/leaked-globals b/tool/leaked-globals index de17038704..6118cd56e8 100755 --- a/tool/leaked-globals +++ b/tool/leaked-globals @@ -1,30 +1,106 @@ #!/usr/bin/ruby require_relative 'lib/colorize' +require 'shellwords' until ARGV.empty? case ARGV[0] - when /\ASYMBOL_PREFIX=(.*)/ + when /\A SYMBOL_PREFIX=(.*)/x SYMBOL_PREFIX = $1 - when /\ANM=(.*)/ # may be multiple words - NM = $1 + when /\A NM=(.*)/x # may be multiple words + NM = $1.shellsplit + when /\A PLATFORM=(.+)?/x + platform = $1 + when /\A SOEXT=(.+)?/x + soext = $1 + when /\A SYMBOLS_IN_EMPTYLIB=(.*)/x + SYMBOLS_IN_EMPTYLIB = $1.split(" ") + when /\A EXTSTATIC=(.+)?/x + EXTSTATIC = true else break end ARGV.shift end +SYMBOLS_IN_EMPTYLIB ||= nil +EXTSTATIC ||= false config = ARGV.shift count = 0 col = Colorize.new -REPLACE = File.read(config).scan(/\bAC_(?:REPLACE|CHECK)_FUNCS?\(\K\w+/) + +config_code = File.read(config) +REPLACE = config_code.scan(/\bAC_(?:REPLACE|CHECK)_FUNCS?\((\w+)/).flatten +# REPLACE << 'memcmp' if /\bAC_FUNC_MEMCMP\b/ =~ config_code +REPLACE.push('main', 'DllMain') +if platform and !platform.empty? + begin + h = File.read(platform) + rescue Errno::ENOENT + else + REPLACE.concat( + h .gsub(%r[/\*.*?\*/]m, " ") # delete block comments + .gsub(%r[//.*], " ") # delete oneline comments + .gsub(/^\s*#.*(?:\\\n.*)*/, "") # delete preprocessor directives + .gsub(/(?:\A|;)\K\s*typedef\s.*?;/m, "") + .scan(/\b((?!rb_|DEPRECATED|_)\w+)\s*\(.*\);/) + .flatten) + end +end +missing = File.dirname(config) + "/missing/" +ARGV.reject! do |n| + base = File.basename(n, ".*") + next true if REPLACE.include?(base) + unless (src = Dir.glob(missing + base + ".[cS]")).empty? + puts "Ignore #{col.skip(n)} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" + true + end +end + +# darwin's ld64 seems to require exception handling personality functions to be +# extern, so we allow the Rust one. +REPLACE.push("rust_eh_personality") if RUBY_PLATFORM.include?("darwin") + print "Checking leaked global symbols..." STDOUT.flush -IO.foreach("|#{NM} -Pgp #{ARGV.join(' ')}") do |line| +if soext + soext = /\.#{soext}(?:$|\.)/ + if EXTSTATIC + ARGV.delete_if {|n| soext =~ n} + elsif ARGV.size == 1 + so = soext =~ ARGV.first + end +end + +Pipe = Struct.new(:command) do + def open(&block) IO.popen(command, &block) end + def each(&block) open {|f| f.each(&block)} end +end + +Pipe.new(NM + ARGV).each do |line| + line.chomp! + next so = nil if line.empty? + if so.nil? and line.chomp!(":") + so = soext =~ line || false + next + end n, t, = line.split - next unless /[BDT]/ =~ t + next unless /[A-TV-Z]/ =~ t next unless n.sub!(/^#{SYMBOL_PREFIX}/o, "") next if n.include?(".") - next if /\A(?:Init_|InitVM_|ruby_|rb_|[Oo]nig|st_|dln_|mjit_|coroutine_|nu(?:comp|rat)_)/ =~ n + next if !so and n.start_with?("___asan_") + next if !so and n.start_with?("__odr_asan_") + next if !so and n.start_with?("__retguard_") + next if !so and n.start_with?("__dtrace") + case n + when /\A(?:Init_|InitVM_|pm_|[Oo]nig|dln_|coroutine_)/ + next + when /\Aruby_static_id_/ + next unless so + when /\A(?:RUBY_|ruby_|rb_)/ + next unless so and /_(threadptr|ec)_/ =~ n + when *SYMBOLS_IN_EMPTYLIB + next + end next if REPLACE.include?(n) puts col.fail("leaked") if count.zero? count += 1 diff --git a/tool/lib/-test-/integer.rb b/tool/lib/-test-/integer.rb index a224148f24..e60abf03a0 100644 --- a/tool/lib/-test-/integer.rb +++ b/tool/lib/-test-/integer.rb @@ -4,11 +4,11 @@ require '-test-/integer.so' module Test::Unit::Assertions def assert_fixnum(v, msg=nil) assert_instance_of(Integer, v, msg) - assert_predicate(v, :fixnum?, msg) + assert_send([Bug::Integer, :fixnum?, v], msg) end def assert_bignum(v, msg=nil) assert_instance_of(Integer, v, msg) - assert_predicate(v, :bignum?, msg) + assert_send([Bug::Integer, :bignum?, v], msg) end end diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb new file mode 100644 index 0000000000..daa1a1f235 --- /dev/null +++ b/tool/lib/_tmpdir.rb @@ -0,0 +1,100 @@ +template = "rubytest." + +# This path is only for tests. +# Assume the directory by these environment variables are safe. +base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| + next unless tmp and tmp.size <= 50 and File.directory?(tmp) + # On macOS, the default TMPDIR is very long, in spite of UNIX socket + # path length being limited. + # + # Also Rubygems creates its own temporary directory per tests, and + # some tests copy the full path of gemhome there. In that case, the + # path contains both temporary names twice, and can exceed path name + # limit very easily. + tmp +end +begin + tmpdir = File.join(base, template + Random.new_seed.to_s(36)[-6..-1]) + Dir.mkdir(tmpdir, 0o700) +rescue Errno::EEXIST + retry +end +# warn "tmpdir(#{tmpdir.size}) = #{tmpdir}" + +pid = $$ +END { + if pid == $$ + begin + Dir.rmdir(tmpdir) + rescue Errno::ENOENT + rescue Errno::ENOTEMPTY + require_relative "colorize" + colorize = Colorize.new + ls = Struct.new(:colorize) do + def mode_inspect(m, s) + [ + (m & 0o4 == 0 ? ?- : ?r), + (m & 0o2 == 0 ? ?- : ?w), + (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)), + ] + end + def decorate_path(path, st) + case + when st.directory? + color = "bold;blue" + type = "/" + when st.symlink? + color = "bold;cyan" + # type = "@" + when st.executable? + color = "bold;green" + type = "*" + when path.end_with?(".gem") + color = "green" + end + colorize.decorate(path, color) + (type || "") + end + def list_tree(parent, indent = "", &block) + children = Dir.children(parent).map do |child| + [child, path = File.join(parent, child), File.lstat(path)] + end + nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size + size_width = children.map {|child, path, st| st.size}.max.to_s.size + + children.each do |child, path, st| + m = st.mode + m = [ + (st.file? ? ?- : st.ftype[0]), + mode_inspect(m >> 6, (?s unless m & 04000 == 0)), + mode_inspect(m >> 3, (?s unless m & 02000 == 0)), + mode_inspect(m, (?t unless m & 01000 == 0)), + ].join("") + warn sprintf("%s* %s %*d %*d %s % s%s", + indent, m, nlink_width, st.nlink, size_width, st.size, + st.mtime.to_s, decorate_path(child, st), + (" -> " + decorate_path(File.readlink(path), File.stat(path)) if + st.symlink?)) + if st.directory? + list_tree(File.join(parent, child), indent + " ", &block) + end + yield path, st if block + end + end + end.new(colorize) + warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":" + Dir.chdir(tmpdir) do + ls.list_tree(".") do |path, st| + if st.directory? + Dir.rmdir(path) + else + File.unlink(path) + end + end + end + require "fileutils" + FileUtils.rm_rf(tmpdir) + end + end +} + +ENV["TMPDIR"] = ENV["SPEC_TEMP_DIR"] = ENV["GEM_TEST_TMPDIR"] = tmpdir diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb new file mode 100644 index 0000000000..9ad5ea220b --- /dev/null +++ b/tool/lib/bundle_env.rb @@ -0,0 +1,4 @@ +ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_PATH__SYSTEM"] = "true" +ENV["BUNDLE_WITHOUT"] = "lint doc" diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb new file mode 100644 index 0000000000..d2ed61a508 --- /dev/null +++ b/tool/lib/bundled_gem.rb @@ -0,0 +1,126 @@ +require 'fileutils' +require 'rubygems' +require 'rubygems/package' + +# This library is used by "make extract-gems" to +# unpack bundled gem files. + +module BundledGem + DEFAULT_GEMS_DEPENDENCIES = [ + "net-protocol", # net-ftp + "time", # net-ftp + "singleton", # prime + "ipaddr", # rinda + "forwardable", # prime, rinda + "strscan", # rexml + "psych" # rdoc + ] + + module_function + + def unpack(file, *rest) + pkg = Gem::Package.new(file) + prepare_test(pkg.spec, *rest) do |dir| + pkg.extract_files(dir) + FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)}) + end + puts "Unpacked #{file}" + rescue Gem::Package::FormatError, Errno::ENOENT + puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." + if file =~ /^gems\/(\w+)-/ + file = Dir.glob("gems/#{$1}-*.gem").first + end + retry + end + + def build(gemspec, version, outdir = ".", validation: true) + outdir = File.expand_path(outdir) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.load(gemfile) + abort "Failed to load #{gemspec}" unless spec + output = File.join(outdir, spec.file_name) + FileUtils.rm_rf(output) + package = Gem::Package.new(output) + package.spec = spec + package.build(validation == false) + end + end + + def copy(path, *rest) + path, n = File.split(path) + spec = Dir.chdir(path) {Gem::Specification.load(n)} or raise "Cannot load #{path}" + prepare_test(spec, *rest) do |dir| + FileUtils.rm_rf(dir) + files = spec.files.reject {|f| f.start_with?(".git")} + dirs = files.map {|f| File.dirname(f) if f.include?("/")}.uniq + FileUtils.mkdir_p(dirs.map {|d| d ? "#{dir}/#{d}" : dir}.sort_by {|d| d.count("/")}) + files.each do |f| + File.copy_stream(File.join(path, f), File.join(dir, f)) + end + end + puts "Copied #{path}" + end + + def prepare_test(spec, dir = ".") + target = spec.full_name + Gem.ensure_gem_subdirectories(dir) + gem_dir = File.join(dir, "gems", target) + yield gem_dir + spec_dir = spec.extensions.empty? ? "specifications" : File.join("gems", target) + if spec.extensions.empty? + spec.dependencies.reject! {|dep| DEFAULT_GEMS_DEPENDENCIES.include?(dep.name)} + end + File.binwrite(File.join(dir, spec_dir, "#{target}.gemspec"), spec.to_ruby) + unless spec.extensions.empty? + spec.dependencies.clear + File.binwrite(File.join(dir, spec_dir, ".bundled.#{target}.gemspec"), spec.to_ruby) + end + if spec.bindir and spec.executables + bindir = File.join(dir, "bin") + Dir.mkdir(bindir) rescue nil + spec.executables.each do |exe| + File.open(File.join(bindir, exe), "wb", 0o777) {|f| + f.print "#!ruby\n", + %[load File.realpath("../gems/#{target}/#{spec.bindir}/#{exe}", __dir__)\n] + } + end + end + FileUtils.rm_rf(Dir.glob("#{gem_dir}/.git*")) + end + + def dummy_gemspec(gemspec) + return if File.exist?(gemspec) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.new do |s| + s.name = gemfile.chomp(".gemspec") + s.version = + File.read("lib/#{s.name}.rb")[/VERSION = "(.+?)"/, 1] || + begin File.read("lib/#{s.name}/version.rb")[/VERSION = "(.+?)"/, 1]; rescue; nil; end || + raise("cannot find the version of #{ s.name } gem") + s.authors = ["DUMMY"] + s.email = ["dummy@ruby-lang.org"] + s.files = Dir.glob("{lib,ext}/**/*").select {|f| File.file?(f)} + s.licenses = ["Ruby"] + s.description = "DO NOT USE; dummy gemspec only for test" + s.summary = "(dummy gemspec)" + end + File.write(gemfile, spec.to_ruby) + end + end + + def checkout(gemdir, repo, rev, git: $git) + return unless rev or !git or git.empty? + unless File.exist?("#{gemdir}/.git") + puts "Cloning #{repo}" + command = "#{git} clone #{repo} #{gemdir}" + system(command) or raise "failed: #{command}" + end + puts "Update #{File.basename(gemdir)} to #{rev}" + command = "#{git} fetch origin #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + command = "#{git} checkout --detach #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + end +end diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb index 855e1331ad..0904312119 100644 --- a/tool/lib/colorize.rb +++ b/tool/lib/colorize.rb @@ -1,10 +1,14 @@ # frozen-string-literal: true class Colorize + # call-seq: + # Colorize.new(colorize = nil) + # Colorize.new(color: color, colors_file: colors_file) def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color)) @colors = @reset = nil - if color or (color == nil && STDOUT.tty?) - if (/\A\e\[.*m\z/ =~ IO.popen("tput smso", "r", :err => IO::NULL, &:read) rescue nil) + @color = opts && opts[:color] || color + if color or (color == nil && coloring?) + if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil) @beg = "\e[" colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} if opts and colors_file = opts[:colors_file] @@ -23,19 +27,48 @@ class Colorize end DEFAULTS = { - "pass"=>"32", "fail"=>"31;1", "skip"=>"33;1", + # color names "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", + "bold"=>"1", "underline"=>"4", "reverse"=>"7", + "bright_black"=>"90", "bright_red"=>"91", "bright_green"=>"92", "bright_yellow"=>"93", + "bright_blue"=>"94", "bright_magenta"=>"95", "bright_cyan"=>"96", "bright_white"=>"97", + + # abstract decorations + "pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold", + "note"=>"bright_yellow", "notice"=>"bright_yellow", "info"=>"bright_magenta", } - def decorate(str, name) - if @colors and color = (@colors[name] || DEFAULTS[name]) + def coloring? + STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?) + end + + # colorize.decorate(str, name = color_name) + def decorate(str, name = @color) + if coloring? and color = resolve_color(name) "#{@beg}#{color}m#{str}#{@reset}" else str end end + def resolve_color(color = @color, seen = {}, colors = nil) + return unless @colors + color.to_s.gsub(/\b[a-z][\w ]+/) do |n| + n.gsub!(/\W+/, "_") + n.downcase! + c = seen[n] and next c + if colors + c = colors[n] + elsif (c = (tbl = @colors)[n] || (tbl = DEFAULTS)[n]) + colors = tbl + else + next n + end + seen[n] = resolve_color(c, seen, colors) + end + end + DEFAULTS.each_key do |name| define_method(name) {|str| decorate(str, name) @@ -44,7 +77,6 @@ class Colorize end if $0 == __FILE__ - colorize = Colorize.new - col = ARGV.shift - ARGV.each {|str| puts colorize.decorate(str, col)} + colorize = Colorize.new(ARGV.shift) + ARGV.each {|str| puts colorize.decorate(str)} end diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb new file mode 100644 index 0000000000..e29a0e3c25 --- /dev/null +++ b/tool/lib/core_assertions.rb @@ -0,0 +1,1033 @@ +# frozen_string_literal: true + +module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + + module Unit + module Assertions + def assert_raises(*exp, &b) + raise NoMethodError, "use assert_raise", caller + end + + def _assertions= n # :nodoc: + @_assertions = n + end + + def _assertions # :nodoc: + @_assertions ||= 0 + end + + ## + # Returns a proc that will output +msg+ along with the default message. + + def message msg = nil, ending = nil, &default + proc { + ending ||= (ending_pattern = /(?<!\.)\z/; ".") + ending_pattern ||= /(?<!#{Regexp.quote(ending)})\z/ + msg = msg.call if Proc === msg + ary = [msg, (default.call if default)].compact.reject(&:empty?) + ary.map! {|str| str.to_s.sub(ending_pattern, ending) } + begin + ary.join("\n") + rescue Encoding::CompatibilityError + ary.map(&:b).join("\n") + end + } + end + end + + module CoreAssertions + require_relative 'envutil' + require 'pp' + begin + require '-test-/sanitizers' + rescue LoadError + # in test-unit-ruby-core gem + def sanitizers + nil + end + else + def sanitizers + Test::Sanitizers + end + end + module_function :sanitizers + + nil.pretty_inspect + + def mu_pp(obj) #:nodoc: + obj.pretty_inspect.chomp + end + + def assert_file + AssertFile + end + + FailDesc = proc do |status, message = "", out = ""| + now = Time.now + proc do + EnvUtil.failure_description(status, now, message, out) + end + end + + def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, + success: nil, failed: nil, gems: false, **opt) + args = Array(args).dup + unless gems.nil? + args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems") + end + stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) + desc = failed[status, message, stderr] if failed + desc ||= FailDesc[status, message, stderr] + if block_given? + raise "test_stdout ignored, use block only or without block" if test_stdout != [] + raise "test_stderr ignored, use block only or without block" if test_stderr != [] + yield(stdout.lines.map {|l| l.chomp }, stderr.lines.map {|l| l.chomp }, status) + else + all_assertions(desc) do |a| + [["stdout", test_stdout, stdout], ["stderr", test_stderr, stderr]].each do |key, exp, act| + a.for(key) do + if exp.is_a?(Regexp) + assert_match(exp, act) + elsif exp.all? {|e| String === e} + assert_equal(exp, act.lines.map {|l| l.chomp }) + else + assert_pattern_list(exp, act) + end + end + end + unless success.nil? + a.for("success?") do + if success + assert_predicate(status, :success?) + else + assert_not_predicate(status, :success?) + end + end + end + end + status + end + end + + if defined?(RubyVM::InstructionSequence) + def syntax_check(code, fname, line) + code = code.dup.force_encoding(Encoding::UTF_8) + RubyVM::InstructionSequence.compile(code, fname, fname, line) + :ok + ensure + raise if SyntaxError === $! + end + else + def syntax_check(code, fname, line) + code = code.b + code.sub!(/\A(?:\xef\xbb\xbf)?(\s*\#.*$)*(\n)?/n) { + "#$&#{"\n" if $1 && !$2}BEGIN{throw tag, :ok}\n" + } + code = code.force_encoding(Encoding::UTF_8) + catch {|tag| eval(code, binding, fname, line - 1)} + end + end + + def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) + # TODO: consider choosing some appropriate limit for RJIT and stop skipping this once it does not randomly fail + pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + # For previous versions which implemented MJIT + pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + # ASAN has the same problem - its shadow memory greatly increases memory usage + # (plus asan has better ways to detect memory leaks than this assertion) + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled? + + require_relative 'memory_status' + raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) + + token_dump, token_re = new_test_token + envs = args.shift if Array === args and Hash === args.first + args = [ + "--disable=gems", + "-r", File.expand_path("../memory_status", __FILE__), + *args, + "-v", "-", + ] + if defined? Memory::NO_MEMORY_LEAK_ENVS then + envs ||= {} + newenvs = envs.merge(Memory::NO_MEMORY_LEAK_ENVS) { |_, _, _| break } + envs = newenvs if newenvs + end + args.unshift(envs) if envs + cmd = [ + 'END {STDERR.puts '"#{token_dump}"'"FINAL=#{Memory::Status.new}"}', + prepare, + 'STDERR.puts('"#{token_dump}"'"START=#{$initial_status = Memory::Status.new}")', + '$initial_size = $initial_status.size', + code, + 'GC.start', + ].join("\n") + _, err, status = EnvUtil.invoke_ruby(args, cmd, true, true, **opt) + before = err.sub!(/^#{token_re}START=(\{.*\})\n/, '') && Memory::Status.parse($1) + after = err.sub!(/^#{token_re}FINAL=(\{.*\})\n/, '') && Memory::Status.parse($1) + assert(status.success?, FailDesc[status, message, err]) + ([:size, (rss && :rss)] & after.members).each do |n| + b = before[n] + a = after[n] + next unless a > 0 and b > 0 + assert_operator(a.fdiv(b), :<, limit, message(message) {"#{n}: #{b} => #{a}"}) + end + rescue LoadError + pend + end + + # :call-seq: + # assert_nothing_raised( *args, &block ) + # + #If any exceptions are given as arguments, the assertion will + #fail if one of those exceptions are raised. Otherwise, the test fails + #if any exceptions are raised. + # + #The final argument may be a failure message. + # + # assert_nothing_raised RuntimeError do + # raise Exception #Assertion passes, Exception is not a RuntimeError + # end + # + # assert_nothing_raised do + # raise Exception #Assertion fails + # end + def assert_nothing_raised(*args) + self._assertions += 1 + if Module === args.last + msg = nil + else + msg = args.pop + end + begin + yield + rescue Test::Unit::PendedError, *(Test::Unit::AssertionFailedError if args.empty?) + raise + rescue *(args.empty? ? Exception : args) => e + msg = message(msg) { + "Exception raised:\n<#{mu_pp(e)}>\n""Backtrace:\n" << + Test.filter_backtrace(e.backtrace).map{|frame| " #{frame}"}.join("\n") + } + raise Test::Unit::AssertionFailedError, msg.call, e.backtrace + end + end + + def prepare_syntax_check(code, fname = nil, mesg = nil, verbose: nil) + fname ||= caller_locations(2, 1)[0] + mesg ||= fname.to_s + verbose, $VERBOSE = $VERBOSE, verbose + case + when Array === fname + fname, line = *fname + when defined?(fname.path) && defined?(fname.lineno) + fname, line = fname.path, fname.lineno + else + line = 1 + end + yield(code, fname, line, message(mesg) { + if code.end_with?("\n") + "```\n#{code}```\n" + else + "```\n#{code}\n```\n""no-newline" + end + }) + ensure + $VERBOSE = verbose + end + + def assert_valid_syntax(code, *args, **opt) + prepare_syntax_check(code, *args, **opt) do |src, fname, line, mesg| + yield if defined?(yield) + assert_nothing_raised(SyntaxError, mesg) do + assert_equal(:ok, syntax_check(src, fname, line), mesg) + end + end + end + + def assert_normal_exit(testsrc, message = '', child_env: nil, **opt) + assert_valid_syntax(testsrc, caller_locations(1, 1)[0]) + if child_env + child_env = [child_env] + else + child_env = [] + end + out, _, status = EnvUtil.invoke_ruby(child_env + %W'-W0', testsrc, true, :merge_to_stdout, **opt) + assert !status.signaled?, FailDesc[status, message, out] + end + + def assert_ruby_status(args, test_stdin="", message=nil, **opt) + out, _, status = EnvUtil.invoke_ruby(args, test_stdin, true, :merge_to_stdout, **opt) + desc = FailDesc[status, message, out] + assert(!status.signaled?, desc) + message ||= "ruby exit status is not success:" + assert(status.success?, desc) + end + + ABORT_SIGNALS = Signal.list.values_at(*%w"ILL ABRT BUS SEGV TERM") + + def separated_runner(token, out = nil) + include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + + out = out ? IO.new(out, 'w') : STDOUT + + # avoid method redefinitions + out_write = out.method(:write) + integer_to_s = Integer.instance_method(:to_s) + array_pack = Array.instance_method(:pack) + marshal_dump = Marshal.method(:dump) + assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set) + assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get) + Test::Unit::Assertions.module_eval do + @_assertions = 0 + + undef _assertions= + define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)}) + + undef _assertions + define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)}) + end + # assume Method#call and UnboundMethod#bind_call need to work as the original + + at_exit { + assertions = assertions_ivar_get.call(:@_assertions) + out_write.call <<~OUT + <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}> + #{array_pack.bind_call([marshal_dump.call($!)], 'm0')} + </error id="#{token}"> + OUT + } + if defined?(Test::Unit::Runner) + Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) + elsif defined?(Test::Unit::AutoRunner) + Test::Unit::AutoRunner.need_auto_run = false + end + end + + def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **opt) + unless file and line + loc, = caller_locations(1,1) + file ||= loc.path + line ||= loc.lineno + end + capture_stdout = true + unless /mswin|mingw/ =~ RbConfig::CONFIG['host_os'] + capture_stdout = false + opt[:out] = Test::Unit::Runner.output if defined?(Test::Unit::Runner) + res_p, res_c = IO.pipe + opt[:ios] = [res_c] + end + token_dump, token_re = new_test_token + src = <<eom +# -*- coding: #{line += __LINE__; src.encoding}; -*- +BEGIN { + require "test/unit";include Test::Unit::Assertions;require #{__FILE__.dump};include Test::Unit::CoreAssertions + separated_runner #{token_dump}, #{res_c&.fileno || 'nil'} +} +#{line -= __LINE__; src} +eom + args = args.dup + args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) + args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + # power_assert 3 requires ruby 3.1 or later + args << "-W:no-experimental" if RUBY_VERSION < "3.1." + stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + + if sanitizers&.lsan_enabled? + # LSAN may output messages like the following line into stderr. We should ignore it. + # ==276855==Running thread 276851 was not suspended. False leaks are possible. + # See https://github.com/google/sanitizers/issues/1479 + stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "") + end + ensure + if res_c + res_c.close + res = res_p.read + res_p.close + else + res = stdout + end + raise if $! + abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + marshal_error = nil + assert(!abort, FailDesc[status, nil, stderr]) + res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do + self._assertions += $1.to_i + res = Marshal.load($2.unpack1("m")) or next + rescue => marshal_error + ignore_stderr = nil + res = nil + else + next if SystemExit === res + if bt = res.backtrace + bt.each do |l| + l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} + end + bt.concat(caller) + else + res.set_backtrace(caller) + end + raise res + end + + # really did it succeed? + unless ignore_stderr + # the body of assert_separately must not output anything to detect error + assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) + end + assert(status.success?, FailDesc[status, "assert_separately failed", stderr]) + raise marshal_error if marshal_error + end + + # Run Ractor-related test without influencing the main test suite + def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) + omit unless defined?(Ractor) + + # https://bugs.ruby-lang.org/issues/21262 + shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value) + shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join) + + if require + require = [require] unless require.is_a?(Array) + require = require.map {|r| "require #{r.inspect}"}.join("\n") + end + + if require_relative + dir = File.dirname(caller_locations[0,1][0].absolute_path) + full_path = File.expand_path(require_relative, dir) + require = "#{require}; require #{full_path.inspect}" + end + + assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{shim_value} + #{shim_join} + #{require} + previous_verbose = $VERBOSE + $VERBOSE = nil + Ractor.new {} # trigger initial warning + $VERBOSE = previous_verbose + #{src} + RUBY + end + + # :call-seq: + # assert_throw( tag, failure_message = nil, &block ) + # + #Fails unless the given block throws +tag+, returns the caught + #value otherwise. + # + #An optional failure message may be provided as the final argument. + # + # tag = Object.new + # assert_throw(tag, "#{tag} was not thrown!") do + # throw tag + # end + def assert_throw(tag, msg = nil) + ret = catch(tag) do + begin + yield(tag) + rescue UncaughtThrowError => e + thrown = e.tag + end + msg = message(msg) { + "Expected #{mu_pp(tag)} to have been thrown"\ + "#{%Q[, not #{thrown}] if thrown}" + } + assert(false, msg) + end + assert(true) + ret + end + + # :call-seq: + # assert_raise( *args, &block ) + # + #Tests if the given block raises an exception. Acceptable exception + #types may be given as optional arguments. If the last argument is a + #String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise NameError do + # puts x #Raises NameError, so assertion succeeds + # end + def assert_raise(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + return e if exp.include? Test::Unit::PendedError + raise e + rescue Exception => e + expected = exp.any? { |ex| + if ex.instance_of? Module then + e.kind_of? ex + else + e.instance_of? ex + end + } + + assert expected, proc { + flunk(message(msg) {"#{mu_pp(exp)} exception expected, not #{mu_pp(e)}"}) + } + + return e + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} expected but nothing was raised"}) + end + end + end + + # :call-seq: + # assert_raise_with_message(exception, expected, msg = nil, &block) + # + #Tests if the given block raises an exception with the expected + #message. + # + # assert_raise_with_message(RuntimeError, "foo") do + # nil #Fails, no Exceptions are raised + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise ArgumentError, "foo" #Fails, different Exception is raised + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise "bar" #Fails, RuntimeError is raised but the message differs + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise "foo" #Raises RuntimeError with the message, so assertion succeeds + # end + def assert_raise_with_message(exception, expected, msg = nil, &block) + case expected + when String + assert = :assert_equal + else + assert_respond_to(expected, :===) + assert = :assert_match + end + + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield + end + m = ex.message + msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} + + if assert == :assert_equal + assert_equal(expected, m, msg) + else + msg = message(msg) { "Expected #{mu_pp expected} to match #{mu_pp m}" } + assert expected =~ m, msg + block.binding.eval("proc{|_|$~=_}").call($~) + end + ex + end + + # :call-seq: + # assert_raise_kind_of(*args, &block) + # + #Tests if the given block raises one of the given exceptions or + #sub exceptions of the given exceptions. If the last argument + #is a String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise SystemCallErr do + # Dir.chdir(__FILE__) #Raises Errno::ENOTDIR, so assertion succeeds + # end + def assert_raise_kind_of(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + raise e unless exp.include? Test::Unit::PendedError + rescue *exp => e + pass + rescue Exception => e + flunk(message(msg) {"#{mu_pp(exp)} family exception expected, not #{mu_pp(e)}"}) + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} family expected but nothing was raised"}) + end + end + e + end + + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: + + # :call-seq: + # assert(test, [failure_message]) + # + #Tests if +test+ is true. + # + #+msg+ may be a String or a Proc. If +msg+ is a String, it will be used + #as the failure message. Otherwise, the result of calling +msg+ will be + #used as the message if the assertion fails. + # + #If no +msg+ is given, a default message will be used. + # + # assert(false, "This was expected to be true") + def assert(test, *msgs) + case msg = msgs.first + when String, Proc + when nil + msgs.shift + else + bt = caller.reject { |s| s.start_with?(TEST_DIR) } + raise ArgumentError, "assertion message must be String or Proc, but #{msg.class} was given.", bt + end unless msgs.empty? + super + end + + # :call-seq: + # assert_respond_to( object, method, failure_message = nil ) + # + #Tests if the given Object responds to +method+. + # + #An optional failure message may be provided as the final argument. + # + # assert_respond_to("hello", :reverse) #Succeeds + # assert_respond_to("hello", :does_not_exist) #Fails + def assert_respond_to(obj, (meth, *priv), msg = nil) + unless priv.empty? + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}#{" privately" if priv[0]}" + } + return assert obj.respond_to?(meth, *priv), msg + end + #get rid of overcounting + if caller_locations(1, 1)[0].path.start_with?(TEST_DIR) + return if obj.respond_to?(meth) + end + super(obj, meth, msg) + end + + # :call-seq: + # assert_not_respond_to( object, method, failure_message = nil ) + # + #Tests if the given Object does not respond to +method+. + # + #An optional failure message may be provided as the final argument. + # + # assert_not_respond_to("hello", :reverse) #Fails + # assert_not_respond_to("hello", :does_not_exist) #Succeeds + def assert_not_respond_to(obj, (meth, *priv), msg = nil) + unless priv.empty? + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to not respond to ##{meth}#{" privately" if priv[0]}" + } + return assert !obj.respond_to?(meth, *priv), msg + end + #get rid of overcounting + if caller_locations(1, 1)[0].path.start_with?(TEST_DIR) + return unless obj.respond_to?(meth) + end + refute_respond_to(obj, meth, msg) + end + + # pattern_list is an array which contains regexp, string and :*. + # :* means any sequence. + # + # pattern_list is anchored. + # Use [:*, regexp/string, :*] for non-anchored match. + def assert_pattern_list(pattern_list, actual, message=nil) + rest = actual + anchored = true + pattern_list.each_with_index {|pattern, i| + if pattern == :* + anchored = false + else + if anchored + match = rest.rindex(pattern, 0) + else + match = rest.index(pattern) + end + if match + post_match = $~ ? $~.post_match : rest[match+pattern.size..-1] + else + msg = message(msg) { + expect_msg = "Expected #{mu_pp pattern}\n" + if /\n[^\n]/ =~ rest + actual_mesg = +"to match\n" + rest.scan(/.*\n+/) { + actual_mesg << ' ' << $&.inspect << "+\n" + } + actual_mesg.sub!(/\+\n\z/, '') + else + actual_mesg = "to match " + mu_pp(rest) + end + actual_mesg << "\nafter #{i} patterns with #{actual.length - rest.length} characters" + expect_msg + actual_mesg + } + assert false, msg + end + rest = post_match + anchored = true + end + } + if anchored + assert_equal("", rest) + end + end + + def assert_warning(pat, msg = nil) + result = nil + stderr = EnvUtil.with_default_internal(of: pat) { + EnvUtil.verbose_warning { + result = yield + } + } + msg = message(msg) {diff pat, stderr} + assert(pat === stderr, msg) + result + end + + def assert_warn(*args) + assert_warning(*args) {$VERBOSE = false; yield} + end + + def assert_deprecated_warning(mesg = /deprecated/, &block) + assert_warning(mesg) do + EnvUtil.deprecation_warning(&block) + end + end + + def assert_deprecated_warn(mesg = /deprecated/, &block) + assert_warn(mesg) do + EnvUtil.deprecation_warning(&block) + end + end + + class << (AssertFile = Struct.new(:failure_message).new) + include Assertions + include CoreAssertions + def assert_file_predicate(predicate, *args) + if /\Anot_/ =~ predicate + predicate = $' + neg = " not" + end + result = File.__send__(predicate, *args) + result = !result if neg + mesg = "Expected file ".dup << args.shift.inspect + mesg << "#{neg} to be #{predicate}" + mesg << mu_pp(args).sub(/\A\[(.*)\]\z/m, '(\1)') unless args.empty? + mesg << " #{failure_message}" if failure_message + assert(result, mesg) + end + alias method_missing assert_file_predicate + + def for(message) + clone.tap {|a| a.failure_message = message} + end + end + + class AllFailures + attr_reader :failures + + def initialize + @count = 0 + @failures = {} + end + + def for(key) + @count += 1 + yield key + rescue Exception => e + @failures[key] = [@count, e] + end + + def foreach(*keys) + keys.each do |key| + @count += 1 + begin + yield key + rescue Exception => e + @failures[key] = [@count, e] + end + end + end + + def message + i = 0 + total = @count.to_s + fmt = "%#{total.size}d" + @failures.map {|k, (n, v)| + v = v.message + "\n#{i+=1}. [#{fmt%n}/#{total}] Assertion for #{k.inspect}\n#{v.b.gsub(/^/, ' | ').force_encoding(v.encoding)}" + }.join("\n") + end + + def pass? + @failures.empty? + end + end + + # threads should respond to shift method. + # Array can be used. + def assert_join_threads(threads, message = nil) + errs = [] + values = [] + while th = threads.shift + begin + values << th.value + rescue Exception + errs << [th, $!] + th = nil + end + end + values + ensure + if th&.alive? + th.raise(Timeout::Error.new) + th.join rescue errs << [th, $!] + end + if !errs.empty? + msg = "exceptions on #{errs.length} threads:\n" + + errs.map {|t, err| + "#{t.inspect}:\n" + + (err.respond_to?(:full_message) ? err.full_message(highlight: false, order: :top) : err.message) + }.join("\n---\n") + if message + msg = "#{message}\n#{msg}" + end + raise Test::Unit::AssertionFailedError, msg + end + end + + def assert_all?(obj, m = nil, &blk) + failed = [] + obj.each do |*a, &b| + unless blk.call(*a, &b) + failed << (a.size > 1 ? a : a[0]) + end + end + assert(failed.empty?, message(m) {failed.pretty_inspect}) + end + + def assert_all_assertions(msg = nil) + all = AllFailures.new + yield all + ensure + assert(all.pass?, message(msg) {all.message.chomp(".")}) + end + alias all_assertions assert_all_assertions + + def assert_all_assertions_foreach(msg = nil, *keys, &block) + all = AllFailures.new + all.foreach(*keys, &block) + ensure + assert(all.pass?, message(msg) {all.message.chomp(".")}) + end + alias all_assertions_foreach assert_all_assertions_foreach + + %w[ + CLOCK_THREAD_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID + CLOCK_MONOTONIC + ].find do |c| + if Process.const_defined?(c) + [c.to_sym, Process.const_get(c)].find do |clk| + begin + Process.clock_gettime(clk) + rescue + # Constants may be defined but not implemented, e.g., mingw. + else + unless Process.clock_getres(clk) < 1.0e-03 + next # needs msec precision + end + PERFORMANCE_CLOCK = clk + end + end + end + end + + # Expect +seq+ to respond to +first+ and +each+ methods, e.g., + # Array, Range, Enumerator::ArithmeticSequence and other + # Enumerable-s, and each elements should be size factors. + # + # :yield: each elements of +seq+. + def assert_linear_performance(seq, rehearsal: nil, pre: ->(n) {n}) + pend "No PERFORMANCE_CLOCK found" unless defined?(PERFORMANCE_CLOCK) + + # Timeout testing generally doesn't work when RJIT compilation happens. + rjit_enabled = defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + measure = proc do |arg, message| + st = Process.clock_gettime(PERFORMANCE_CLOCK) + yield(*arg) + t = (Process.clock_gettime(PERFORMANCE_CLOCK) - st) + assert_operator 0, :<=, t, message unless rjit_enabled + t + end + + first = seq.first + *arg = pre.call(first) + times = (0..(rehearsal || (2 * first))).map do + measure[arg, "rehearsal"].nonzero? + end + times.compact! + tmin, tmax = times.minmax + + # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) + tbase = 10 * tmax * [(tmax / tmin) ** 2 / 4, 1].max + info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" + + seq.each do |i| + next if i == first + t = tbase * i.fdiv(first) + *arg = pre.call(i) + message = "[#{i}]: in #{t}s #{info}" + Timeout.timeout(t, Timeout::Error, message) do + measure[arg, message] + end + end + end + + def diff(exp, act) + require 'pp' + q = PP.new(+"") + q.guard_inspect_key do + q.group(2, "expected: ") do + q.pp exp + end + q.text q.newline + q.group(2, "actual: ") do + q.pp act + end + q.flush + end + q.output + end + + def new_test_token + token = "\e[7;1m#{$$.to_s}:#{Time.now.strftime('%s.%L')}:#{rand(0x10000).to_s(16)}:\e[m" + return token.dump, Regexp.quote(token) + end + + # Platform predicates + + def self.mswin? + defined?(@mswin) ? @mswin : @mswin = RUBY_PLATFORM.include?('mswin') + end + private def mswin? + CoreAssertions.mswin? + end + + def self.mingw? + defined?(@mingw) ? @mingw : @mingw = RUBY_PLATFORM.include?('mingw') + end + private def mingw? + CoreAssertions.mingw? + end + + module_function def windows? + mswin? or mingw? + end + + def self.version_compare(expected, actual) + expected.zip(actual).each {|e, a| z = (e <=> a); return z if z.nonzero?} + 0 + end + + def self.version_match?(expected, actual) + if !actual + false + elsif expected.empty? + true + elsif expected.size == 1 and Range === (range = expected.first) + b, e = range.begin, range.end + return false if b and (c = version_compare(Array(b), actual)) > 0 + return false if e and (c = version_compare(Array(e), actual)) < 0 + return false if e and range.exclude_end? and c == 0 + true + else + version_compare(expected, actual).zero? + end + end + + def self.linux?(*ver) + unless defined?(@linux) + @linux = RUBY_PLATFORM.include?('linux') && `uname -r`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @linux + end + private def linux?(*ver) + CoreAssertions.linux?(*ver) + end + + def self.glibc?(*ver) + unless defined?(@glibc) + libc = `/usr/bin/ldd /bin/sh`[/^\s*libc.*=> *\K\S*/] + if libc and /version (\d+)\.(\d+)\.$/ =~ IO.popen([libc], &:read)[] + @glibc = [$1.to_i, $2.to_i] + else + @glibc = false + end + end + version_match? ver, @glibc + end + private def glibc?(*ver) + CoreAssertions.glibc?(*ver) + end + + def self.macos?(*ver) + unless defined?(@macos) + @macos = RUBY_PLATFORM.include?('darwin') && `sw_vers -productVersion`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @macos + end + private def macos?(*ver) + CoreAssertions.macos?(*ver) + end + end + end +end diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb new file mode 100644 index 0000000000..56b420a546 --- /dev/null +++ b/tool/lib/dump.gdb @@ -0,0 +1,17 @@ +set height 0 +set width 0 +set confirm off + +echo \n>>> Threads\n\n +info threads + +echo \n>>> Machine level backtrace\n\n +thread apply all info stack full + +echo \n>>> Dump Ruby level backtrace (if possible)\n\n +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +echo ">>> Finish\n" +detach +quit diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb new file mode 100644 index 0000000000..ed9cb89010 --- /dev/null +++ b/tool/lib/dump.lldb @@ -0,0 +1,13 @@ +script print("\n>>> Threads\n\n") +thread list + +script print("\n>>> Machine level backtrace\n\n") +thread backtrace all + +script print("\n>>> Dump Ruby level backtrace (if possible)\n\n") +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +script print(">>> Finish\n") +detach +quit diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index b4eb63da90..6089605056 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -15,23 +15,22 @@ end module EnvUtil def rubybin if ruby = ENV["RUBY"] - return ruby - end - ruby = "ruby" - exeext = RbConfig::CONFIG["EXEEXT"] - rubyexe = (ruby + exeext if exeext and !exeext.empty?) - 3.times do - if File.exist? ruby and File.executable? ruby and !File.directory? ruby - return File.expand_path(ruby) - end - if rubyexe and File.exist? rubyexe and File.executable? rubyexe - return File.expand_path(rubyexe) - end - ruby = File.join("..", ruby) - end - if defined?(RbConfig.ruby) + ruby + elsif defined?(RbConfig.ruby) RbConfig.ruby else + ruby = "ruby" + exeext = RbConfig::CONFIG["EXEEXT"] + rubyexe = (ruby + exeext if exeext and !exeext.empty?) + 3.times do + if File.exist? ruby and File.executable? ruby and !File.directory? ruby + return File.expand_path(ruby) + end + if rubyexe and File.exist? rubyexe and File.executable? rubyexe + return File.expand_path(rubyexe) + end + ruby = File.join("..", ruby) + end "ruby" end end @@ -47,12 +46,20 @@ module EnvUtil class << self attr_accessor :timeout_scale attr_reader :original_internal_encoding, :original_external_encoding, - :original_verbose + :original_verbose, :original_warning def capture_global_values @original_internal_encoding = Encoding.default_internal @original_external_encoding = Encoding.default_external @original_verbose = $VERBOSE + @original_warning = + if defined?(Warning.categories) + Warning.categories.to_h {|i| [i, Warning[i]]} + elsif defined?(Warning.[]) # 2.7+ + %i[deprecated experimental performance].to_h do |i| + [i, begin Warning[i]; rescue ArgumentError; end] + end.compact + end end end @@ -72,6 +79,72 @@ module EnvUtil end module_function :timeout + class Debugger + @list = [] + + attr_accessor :name + + def self.register(name, &block) + @list << new(name, &block) + end + + def initialize(name, &block) + @name = name + instance_eval(&block) + end + + def usable?; false; end + + def start(pid, *args) end + + def dump(pid, timeout: 60, reprieve: timeout&.div(4)) + dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err) + rescue Errno::ENOENT + return + else + return unless dpid + [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig| + begin + return EnvUtil.timeout(t) {Process.wait(dpid)} + rescue Timeout::Error + Process.kill(sig, dpid) + end + end + true + end + + # sudo -n: --non-interactive + PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM) + + def spawn(*args, **opts) + super(*PRECOMMAND, *args, **opts) + end + + register("gdb") do + class << self + def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end + def start(pid, *args, **opts) + spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts) + end + def command_file(file) "--command=#{file}"; end + end + end + + register("lldb") do + class << self + def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end + def start(pid, *args, **opts) + spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts) + end + def command_file(file) ["--source", file]; end + end + end + + def self.search + @debugger ||= @list.find(&:usable?) + end + end + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) reprieve = apply_timeout_scale(reprieve) if reprieve @@ -86,7 +159,15 @@ module EnvUtil when nil, false pgroup = pid end + + dumped = false while signal = signals.shift + + if !dumped and [:ABRT, :KILL].include?(signal) + Debugger.search&.dump(pid) + dumped = true + end + begin Process.kill signal, pgroup rescue Errno::EINVAL @@ -100,6 +181,8 @@ module EnvUtil begin Timeout.timeout(reprieve) {Process.wait(pid)} rescue Timeout::Error + else + break end end end @@ -109,7 +192,7 @@ module EnvUtil def invoke_ruby(args, stdin_data = "", capture_stdout = false, capture_stderr = false, encoding: nil, timeout: 10, reprieve: 1, timeout_error: Timeout::Error, - stdout_filter: nil, stderr_filter: nil, + stdout_filter: nil, stderr_filter: nil, ios: nil, signal: :TERM, rubybin: EnvUtil.rubybin, precommand: nil, **opt) @@ -125,6 +208,8 @@ module EnvUtil out_p.set_encoding(encoding) if out_p err_p.set_encoding(encoding) if err_p end + ios.each {|i, o = i|opt[i] = o} if ios + c = "C" child_env = {} LANG_ENVS.each {|lc| child_env[lc] = c} @@ -134,11 +219,24 @@ module EnvUtil if RUBYLIB and lib = child_env["RUBYLIB"] child_env["RUBYLIB"] = [lib, RUBYLIB].join(File::PATH_SEPARATOR) end + + # remain env + %w(ASAN_OPTIONS RUBY_ON_BUG).each{|name| + child_env[name] = ENV[name] if !child_env.key?(name) and ENV.key?(name) + } + args = [args] if args.kind_of?(String) - pid = spawn(child_env, *precommand, rubybin, *args, **opt) + # use the same parser as current ruby + if (args.none? { |arg| arg.start_with?("--parser=") } and + /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read)) + args = ["--parser=#{current_parser}"] + args + end + pid = spawn(child_env, *precommand, rubybin, *args, opt) in_c.close - out_c.close if capture_stdout - err_c.close if capture_stderr && capture_stderr != :merge_to_stdout + out_c&.close + out_c = nil + err_c&.close + err_c = nil if block_given? return yield in_p, out_p, err_p, pid else @@ -150,6 +248,7 @@ module EnvUtil timeout_error = nil else status = terminate(pid, signal, opt[:pgroup], reprieve) + terminated = Time.now end stdout = th_stdout.value if capture_stdout stderr = th_stderr.value if capture_stderr && capture_stderr != :merge_to_stdout @@ -161,7 +260,7 @@ module EnvUtil if timeout_error bt = caller_locations msg = "execution of #{bt.shift.label} expired timeout (#{timeout} sec)" - msg = Test::Unit::Assertions::FailDesc[status, msg, [stdout, stderr].join("\n")].() + msg = failure_description(status, terminated, msg, [stdout, stderr].join("\n")) raise timeout_error, msg, bt.map(&:to_s) end return stdout, stderr, status @@ -179,10 +278,11 @@ module EnvUtil end module_function :invoke_ruby - alias rubyexec invoke_ruby - class << self - alias rubyexec invoke_ruby + def current_parser + features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}] + features&.split&.include?("+PRISM") ? "prism" : "parse.y" end + module_function :current_parser def verbose_warning class << (stderr = "".dup) @@ -196,9 +296,25 @@ module EnvUtil ensure stderr, $stderr = $stderr, stderr $VERBOSE = EnvUtil.original_verbose + EnvUtil.original_warning&.each {|i, v| Warning[i] = v} end module_function :verbose_warning + if defined?(Warning.[]=) + def deprecation_warning + previous_deprecated = Warning[:deprecated] + Warning[:deprecated] = true + yield + ensure + Warning[:deprecated] = previous_deprecated + end + else + def deprecation_warning + yield + end + end + module_function :deprecation_warning + def default_warning $VERBOSE = false yield @@ -223,7 +339,30 @@ module EnvUtil end module_function :under_gc_stress - def with_default_external(enc) + def under_gc_compact_stress(val = :empty, &block) + raise "compaction doesn't work well on s390x. Omit the test in the caller." if RUBY_PLATFORM =~ /s390x/ # https://github.com/ruby/ruby/pull/5077 + + if GC.respond_to?(:auto_compact) + auto_compact = GC.auto_compact + GC.auto_compact = val + end + + under_gc_stress(&block) + ensure + GC.auto_compact = auto_compact if GC.respond_to?(:auto_compact) + end + module_function :under_gc_compact_stress + + def without_gc + prev_disabled = GC.disable + yield + ensure + GC.enable unless prev_disabled + end + module_function :without_gc + + def with_default_external(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_external = enc } yield ensure @@ -231,7 +370,8 @@ module EnvUtil end module_function :with_default_external - def with_default_internal(enc) + def with_default_internal(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_internal = enc } yield ensure @@ -241,7 +381,11 @@ module EnvUtil def labeled_module(name, &block) Module.new do - singleton_class.class_eval {define_method(:to_s) {name}; alias inspect to_s} + singleton_class.class_eval { + define_method(:to_s) {name} + alias inspect to_s + alias name to_s + } class_eval(&block) if block end end @@ -249,7 +393,11 @@ module EnvUtil def labeled_class(name, superclass = Object, &block) Class.new(superclass) do - singleton_class.class_eval {define_method(:to_s) {name}; alias inspect to_s} + singleton_class.class_eval { + define_method(:to_s) {name} + alias inspect to_s + alias name to_s + } class_eval(&block) if block end end @@ -266,16 +414,24 @@ module EnvUtil cmd = @ruby_install_name if "ruby-runner#{RbConfig::CONFIG["EXEEXT"]}" == cmd path = DIAGNOSTIC_REPORTS_PATH timeformat = DIAGNOSTIC_REPORTS_TIMEFORMAT - pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.crash" + pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.{crash,ips}" first = true 30.times do first ? (first = false) : sleep(0.1) Dir.glob(pat) do |name| log = File.read(name) rescue next - if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log - File.unlink(name) - File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil - return log + case name + when /\.crash\z/ + if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log + File.unlink(name) + File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil + return log + end + when /\.ips\z/ + if /^ *"pid" *: *#{pid},/ =~ log + File.unlink(name) + return log + end end end end @@ -286,6 +442,37 @@ module EnvUtil end end + def self.failure_description(status, now, message = "", out = "") + pid = status.pid + if signo = status.termsig + signame = Signal.signame(signo) + sigdesc = "signal #{signo}" + end + log = diagnostic_reports(signame, pid, now) + if signame + sigdesc = "SIG#{signame} (#{sigdesc})" + end + if status.coredump? + sigdesc = "#{sigdesc} (core dumped)" + end + full_message = ''.dup + message = message.call if Proc === message + if message and !message.empty? + full_message << message << "\n" + end + full_message << "pid #{pid}" + full_message << " exit #{status.exitstatus}" if status.exited? + full_message << " killed by #{sigdesc}" if sigdesc + if out and !out.empty? + full_message << "\n" << out.b.gsub(/^/, '| ') + full_message.sub!(/(?<!\n)\z/, "\n") + end + if log + full_message << "Diagnostic reports:\n" << log.b.gsub(/^/, '| ') + end + full_message + end + def self.gc_stress_to_class? unless defined?(@gc_stress_to_class) _, _, status = invoke_ruby(["-e""exit GC.respond_to?(:add_stress_to_class)"]) @@ -304,7 +491,6 @@ if defined?(RbConfig) end dir = File.dirname(ruby) CONFIG['bindir'] = dir - Gem::ConfigMap[:bindir] = dir if defined?(Gem::ConfigMap) end end diff --git a/tool/lib/gc_checker.rb b/tool/lib/gc_checker.rb new file mode 100644 index 0000000000..719da8cac0 --- /dev/null +++ b/tool/lib/gc_checker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module GCDisabledChecker + def before_setup + if @__gc_disabled__ = GC.enable # return true if GC is disabled + GC.disable + end + + super + end + + def after_teardown + super + + disabled = GC.enable + GC.disable if @__gc_disabled__ + + if @__gc_disabled__ != disabled + label = { + true => 'disabled', + false => 'enabled', + } + raise "GC was #{label[@__gc_disabled__]}, but is #{label[disabled]} after the test." + end + end +end + +module GCCompactChecker + def after_teardown + super + GC.compact + end +end + +Test::Unit::TestCase.include GCDisabledChecker +Test::Unit::TestCase.include GCCompactChecker if ENV['RUBY_TEST_GC_COMPACT'] diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb new file mode 100644 index 0000000000..1893e07657 --- /dev/null +++ b/tool/lib/gem_env.rb @@ -0,0 +1 @@ +ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__) diff --git a/tool/lib/iseq_loader_checker.rb b/tool/lib/iseq_loader_checker.rb index 1a1a694834..73784f8450 100644 --- a/tool/lib/iseq_loader_checker.rb +++ b/tool/lib/iseq_loader_checker.rb @@ -38,8 +38,14 @@ class RubyVM::InstructionSequence i2 end - CHECK_TO_A = ENV['RUBY_ISEQ_DUMP_DEBUG'] == 'to_a' - CHECK_TO_BINARY = ENV['RUBY_ISEQ_DUMP_DEBUG'] == 'to_binary' + opt = ENV['RUBY_ISEQ_DUMP_DEBUG'] + + if opt && caller.any?{|e| /test\/runner\.rb/ =~ e} + puts "RUBY_ISEQ_DUMP_DEBUG = #{opt}" if opt + end + + CHECK_TO_A = 'to_a' == opt + CHECK_TO_BINARY = 'to_binary' == opt def self.translate i1 # check to_a/load_iseq @@ -70,6 +76,15 @@ class RubyVM::InstructionSequence # return value i2_bin if CHECK_TO_BINARY end if CHECK_TO_A || CHECK_TO_BINARY + + if opt == "prism" + # If RUBY_ISEQ_DUMP_DEBUG is "prism", we'll set up + # InstructionSequence.load_iseq to intercept loading filepaths to compile + # using prism. + def self.load_iseq(filepath) + RubyVM::InstructionSequence.compile_file_prism(filepath) + end + end end #require_relative 'x'; exit(1) diff --git a/tool/lib/launchable.rb b/tool/lib/launchable.rb new file mode 100644 index 0000000000..38f4fe92b3 --- /dev/null +++ b/tool/lib/launchable.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +require 'json' +require 'uri' + +module Launchable + ## + # JsonStreamWriter writes a JSON file using a stream. + # By utilizing a stream, we can minimize memory usage, especially for large files. + class JsonStreamWriter + def initialize(path) + @file = File.open(path, "w") + @file.write("{") + @indent_level = 0 + @is_first_key_val = true + @is_first_obj = true + write_new_line + end + + def write_object obj + if @is_first_obj + @is_first_obj = false + else + write_comma + write_new_line + end + @indent_level += 1 + @file.write(to_json_str(obj)) + @indent_level -= 1 + @is_first_key_val = true + # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified. + # { + # "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds", + # "status": "TEST_PASSED", + # "duration": 2.7e-05, + # "createdAt": "2024-02-09 12:21:07 +0000", + # "stderr": null, + # "stdout": null + # }: null <- here + # }, + # To prevent this, IO#flush is called here. + @file.flush + end + + def write_array(key) + @indent_level += 1 + @file.write(to_json_str(key)) + write_colon + @file.write(" ", "[") + write_new_line + end + + def close + return if @file.closed? + close_array + @indent_level -= 1 + write_new_line + @file.write("}", "\n") + @file.flush + @file.close + end + + private + def to_json_str(obj) + json = JSON.pretty_generate(obj) + json.gsub(/^/, ' ' * (2 * @indent_level)) + end + + def write_indent + @file.write(" " * 2 * @indent_level) + end + + def write_new_line + @file.write("\n") + end + + def write_comma + @file.write(',') + end + + def write_colon + @file.write(":") + end + + def close_array + write_new_line + write_indent + @file.write("]") + @indent_level -= 1 + end + end +end diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb index af9200bf77..69df9a64b8 100644 --- a/tool/lib/leakchecker.rb +++ b/tool/lib/leakchecker.rb @@ -1,31 +1,35 @@ # frozen_string_literal: true class LeakChecker + @@try_lsof = nil # not-tried-yet + def initialize @fd_info = find_fds + @@skip = false @tempfile_info = find_tempfiles @thread_info = find_threads @env_info = find_env @encoding_info = find_encodings @old_verbose = $VERBOSE + @old_warning_flags = find_warning_flags end def check(test_name) + if /i386-solaris/ =~ RUBY_PLATFORM && /TestGem/ =~ test_name + GC.verify_internal_consistency + end + leaks = [ check_fd_leak(test_name), check_thread_leak(test_name), check_tempfile_leak(test_name), check_env(test_name), check_encodings(test_name), - check_safe(test_name), check_verbose(test_name), + check_warning_flags(test_name), ] GC.start if leaks.any? end - def check_safe test_name - puts "#{test_name}: $SAFE == #{$SAFE}" unless $SAFE == 0 - end - def check_verbose test_name puts "#{test_name}: $VERBOSE == #{$VERBOSE}" unless @old_verbose == $VERBOSE end @@ -34,19 +38,19 @@ class LeakChecker if IO.respond_to?(:console) and (m = IO.method(:console)).arity.nonzero? m[:close] end - fd_dir = "/proc/self/fd" - if File.directory?(fd_dir) - fds = Dir.open(fd_dir) {|d| - a = d.grep(/\A\d+\z/, &:to_i) - if d.respond_to? :fileno - a -= [d.fileno] - end - a - } - fds.sort - else - [] + %w"/proc/self/fd /dev/fd".each do |fd_dir| + if File.directory?(fd_dir) + fds = Dir.open(fd_dir) {|d| + a = d.grep(/\A\d+\z/, &:to_i) + if d.respond_to? :fileno + a -= [d.fileno] + end + a + } + return fds.sort + end end + [] end def check_fd_leak(test_name) @@ -60,7 +64,7 @@ class LeakChecker } end fd_leaked = live2 - live1 - if !fd_leaked.empty? + if !@@skip && !fd_leaked.empty? leaked = true h = {} ObjectSpace.each_object(IO) {|io| @@ -73,30 +77,54 @@ class LeakChecker end (h[fd] ||= []) << [io, autoclose, inspect] } - fd_leaked.each {|fd| + fd_leaked.select! {|fd| str = ''.dup + pos = nil if h[fd] str << ' :' h[fd].map {|io, autoclose, inspect| + if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] + pos = "#{ObjectSpace.allocation_sourcefile(io)}:#{ObjectSpace.allocation_sourceline(io)}" + end s = ' ' + inspect s << "(not-autoclose)" if !autoclose s }.sort.each {|s| str << s } + else + begin + io = IO.for_fd(fd, autoclose: false) + s = io.stat + rescue Errno::EBADF + # something un-stat-able + next + else + next if /darwin/ =~ RUBY_PLATFORM and [0, -1].include?(s.dev) + str << ' ' << s.inspect + ensure + io&.close + end end puts "Leaked file descriptor: #{test_name}: #{fd}#{str}" + puts " The IO was created at #{pos}" if pos + true } - #system("lsof -p #$$") if !fd_leaked.empty? + unless fd_leaked.empty? + unless @@try_lsof == false + @@try_lsof |= system(*%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + end + end h.each {|fd, list| next if list.length <= 1 if 1 < list.count {|io, autoclose, inspect| autoclose } str = list.map {|io, autoclose, inspect| " #{inspect}" + (autoclose ? "(autoclose)" : "") }.sort.join - puts "Multiple autoclose IO object for a file descriptor:#{str}" + puts "Multiple autoclose IO objects for a file descriptor in: #{test_name}: #{str}" end } end @fd_info = live2 + @@skip = false return leaked end @@ -108,14 +136,14 @@ class LeakChecker attr_accessor :count end - def new(data) + def new(...) LeakChecker::TempfileCounter.count += 1 - super(data) + super end } LeakChecker.const_set(:TempfileCounter, m) - class << Tempfile::Remover + class << Tempfile prepend LeakChecker::TempfileCounter end end @@ -127,8 +155,8 @@ class LeakChecker if prev_count == count [prev_count, []] else - tempfiles = ObjectSpace.each_object(Tempfile).find_all {|t| - t.instance_variable_defined?(:@tmpfile) and t.path + tempfiles = ObjectSpace.each_object(Tempfile).reject {|t| + t.instance_variables.empty? || t.closed? } [count, tempfiles] end @@ -154,7 +182,8 @@ class LeakChecker def find_threads Thread.list.find_all {|t| - t != Thread.current && t.alive? + t != Thread.current && t.alive? && + !(t.thread_variable?(:"\0__detached_thread__") && t.thread_variable_get(:"\0__detached_thread__")) } end @@ -181,15 +210,36 @@ class LeakChecker return leaked end - def find_env - ENV.to_h + e = ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] + begin + ENV["_Ruby_Env_Ignorecase_"] = ENV["_RUBY_ENV_IGNORECASE_"] = nil + ENV["_RUBY_ENV_IGNORECASE_"] = "ENV_CASE_TEST" + ENV_IGNORECASE = ENV["_Ruby_Env_Ignorecase_"] == "ENV_CASE_TEST" + ensure + ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] = e + end + + if ENV_IGNORECASE + def find_env + ENV.to_h {|k, v| [k.upcase, v]} + end + else + def find_env + ENV.to_h + end end def check_env(test_name) old_env = @env_info - new_env = ENV.to_h + new_env = find_env return false if old_env == new_env + if defined?(Bundler::EnvironmentPreserver) + bundler_prefix = Bundler::EnvironmentPreserver::BUNDLER_PREFIX + end (old_env.keys | new_env.keys).sort.each {|k| + # Don't report changed environment variables caused by Bundler's backups + next if bundler_prefix and k.start_with?(bundler_prefix) + if old_env.has_key?(k) if new_env.has_key?(k) if old_env[k] != new_env[k] @@ -211,30 +261,61 @@ class LeakChecker end def find_encodings - [Encoding.default_internal, Encoding.default_external] + { + 'Encoding.default_internal' => Encoding.default_internal, + 'Encoding.default_external' => Encoding.default_external, + 'STDIN.internal_encoding' => STDIN.internal_encoding, + 'STDIN.external_encoding' => STDIN.external_encoding, + 'STDOUT.internal_encoding' => STDOUT.internal_encoding, + 'STDOUT.external_encoding' => STDOUT.external_encoding, + 'STDERR.internal_encoding' => STDERR.internal_encoding, + 'STDERR.external_encoding' => STDERR.external_encoding, + } end def check_encodings(test_name) - old_internal, old_external = @encoding_info - new_internal, new_external = find_encodings + old_encoding_info = @encoding_info + @encoding_info = find_encodings leaked = false - if new_internal != old_internal - leaked = true - puts "Encoding.default_internal changed: #{test_name} : #{old_internal.inspect} to #{new_internal.inspect}" + @encoding_info.each do |key, new_encoding| + old_encoding = old_encoding_info[key] + if new_encoding != old_encoding + leaked = true + puts "#{key} changed: #{test_name} : #{old_encoding.inspect} to #{new_encoding.inspect}" + end end - if new_external != old_external - leaked = true - puts "Encoding.default_external changed: #{test_name} : #{old_external.inspect} to #{new_external.inspect}" + leaked + end + + WARNING_CATEGORIES = (Warning.respond_to?(:[]) ? %i[deprecated experimental] : []).freeze + + def find_warning_flags + WARNING_CATEGORIES.to_h do |category| + [category, Warning[category]] + end + end + + def check_warning_flags(test_name) + new_warning_flags = find_warning_flags + leaked = false + WARNING_CATEGORIES.each do |category| + if new_warning_flags[category] != @old_warning_flags[category] + leaked = true + puts "Warning[#{category.inspect}] changed: #{test_name} : #{@old_warning_flags[category]} to #{new_warning_flags[category]}" + end end - @encoding_info = [new_internal, new_external] return leaked end def puts(*a) - output = MiniTest::Unit.output + output = Test::Unit::Runner.output if defined?(output.set_encoding) output.set_encoding(nil, nil) end output.puts(*a) end + + def self.skip + @@skip = true + end end diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb index ad002b2dda..429e5f6a1d 100644 --- a/tool/lib/memory_status.rb +++ b/tool/lib/memory_status.rb @@ -12,7 +12,7 @@ module Memory PROC_FILE = procfile VM_PAT = pat def self.read_status - IO.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| + File.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| yield($1.downcase.intern, $2.to_i * 1024) if VM_PAT =~ l end end @@ -20,47 +20,68 @@ module Memory data.scan(pat) {|k, v| keys << k.downcase.intern} when /mswin|mingw/ =~ RUBY_PLATFORM - require 'fiddle/import' - require 'fiddle/types' - - module Win32 - extend Fiddle::Importer - dlload "kernel32.dll", "psapi.dll" - include Fiddle::Win32Types - typealias "SIZE_T", "size_t" - - PROCESS_MEMORY_COUNTERS = struct [ - "DWORD cb", - "DWORD PageFaultCount", - "SIZE_T PeakWorkingSetSize", - "SIZE_T WorkingSetSize", - "SIZE_T QuotaPeakPagedPoolUsage", - "SIZE_T QuotaPagedPoolUsage", - "SIZE_T QuotaPeakNonPagedPoolUsage", - "SIZE_T QuotaNonPagedPoolUsage", - "SIZE_T PagefileUsage", - "SIZE_T PeakPagefileUsage", - ] - - typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" - - extern "HANDLE GetCurrentProcess()", :stdcall - extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall - - module_function - def memory_info - size = PROCESS_MEMORY_COUNTERS.size - data = PROCESS_MEMORY_COUNTERS.malloc - data.cb = size - data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + keys.push(:size, :rss, :peak) + + begin + require 'fiddle/import' + require 'fiddle/types' + rescue LoadError + # Fallback to PowerShell command to get memory information for current process + def self.read_status + cmd = [ + "powershell.exe", "-NoProfile", "-Command", + "Get-Process -Id #{$$} | " \ + "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }" + ] + + IO.popen(cmd, "r", err: [:child, :out]) do |out| + if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read + yield :size, $1.to_i + yield :rss, $2.to_i + yield :peak, $3.to_i + end + end + end + else + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end end - end - keys << :peak << :size - def self.read_status - if info = Win32.memory_info - yield :peak, info.PeakPagefileUsage - yield :size, info.PagefileUsage + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end end end when (require_relative 'find_executable' @@ -94,6 +115,7 @@ if defined?(Memory::Status) Memory.read_status do |key, val| self[key] = val end + self end unless method_defined?(:_update) Header = members.map {|k| k.to_s.upcase.rjust(6)}.join('') diff --git a/tool/lib/minitest/README.txt b/tool/lib/minitest/README.txt deleted file mode 100644 index 368cc3aa4e..0000000000 --- a/tool/lib/minitest/README.txt +++ /dev/null @@ -1,457 +0,0 @@ -= minitest/{unit,spec,mock,benchmark} - -home :: https://github.com/seattlerb/minitest -rdoc :: http://docs.seattlerb.org/minitest -vim :: https://github.com/sunaku/vim-ruby-minitest - -== DESCRIPTION: - -minitest provides a complete suite of testing facilities supporting -TDD, BDD, mocking, and benchmarking. - - "I had a class with Jim Weirich on testing last week and we were - allowed to choose our testing frameworks. Kirk Haines and I were - paired up and we cracked open the code for a few test - frameworks... - - I MUST say that minitest is *very* readable / understandable - compared to the 'other two' options we looked at. Nicely done and - thank you for helping us keep our mental sanity." - - -- Wayne E. Seguin - -minitest/unit is a small and incredibly fast unit testing framework. -It provides a rich set of assertions to make your tests clean and -readable. - -minitest/spec is a functionally complete spec engine. It hooks onto -minitest/unit and seamlessly bridges test assertions over to spec -expectations. - -minitest/benchmark is an awesome way to assert the performance of your -algorithms in a repeatable manner. Now you can assert that your newb -co-worker doesn't replace your linear algorithm with an exponential -one! - -minitest/mock by Steven Baker, is a beautifully tiny mock (and stub) -object framework. - -minitest/pride shows pride in testing and adds coloring to your test -output. I guess it is an example of how to write IO pipes too. :P - -minitest/unit is meant to have a clean implementation for language -implementors that need a minimal set of methods to bootstrap a working -test suite. For example, there is no magic involved for test-case -discovery. - - "Again, I can't praise enough the idea of a testing/specing - framework that I can actually read in full in one sitting!" - - -- Piotr Szotkowski - -Comparing to rspec: - - rspec is a testing DSL. minitest is ruby. - - -- Adam Hawkins, "Bow Before MiniTest" - -minitest doesn't reinvent anything that ruby already provides, like: -classes, modules, inheritance, methods. This means you only have to -learn ruby to use minitest and all of your regular OO practices like -extract-method refactorings still apply. - -== FEATURES/PROBLEMS: - -* minitest/autorun - the easy and explicit way to run all your tests. -* minitest/unit - a very fast, simple, and clean test system. -* minitest/spec - a very fast, simple, and clean spec system. -* minitest/mock - a simple and clean mock/stub system. -* minitest/benchmark - an awesome way to assert your algorithm's performance. -* minitest/pride - show your pride in testing! -* Incredibly small and fast runner, but no bells and whistles. - -== RATIONALE: - -See design_rationale.rb to see how specs and tests work in minitest. - -== SYNOPSIS: - -Given that you'd like to test the following class: - - class Meme - def i_can_has_cheezburger? - "OHAI!" - end - - def will_it_blend? - "YES!" - end - end - -=== Unit tests - - require 'minitest/autorun' - - class TestMeme < MiniTest::Unit::TestCase - def setup - @meme = Meme.new - end - - def test_that_kitty_can_eat - assert_equal "OHAI!", @meme.i_can_has_cheezburger? - end - - def test_that_it_will_not_blend - refute_match /^no/i, @meme.will_it_blend? - end - - def test_that_will_be_skipped - skip "test this later" - end - end - -=== Specs - - require 'minitest/autorun' - - describe Meme do - before do - @meme = Meme.new - end - - describe "when asked about cheeseburgers" do - it "must respond positively" do - @meme.i_can_has_cheezburger?.must_equal "OHAI!" - end - end - - describe "when asked about blending possibilities" do - it "won't say no" do - @meme.will_it_blend?.wont_match /^no/i - end - end - end - -For matchers support check out: - -https://github.com/zenspider/minitest-matchers - -=== Benchmarks - -Add benchmarks to your regular unit tests. If the unit tests fail, the -benchmarks won't run. - - # optionally run benchmarks, good for CI-only work! - require 'minitest/benchmark' if ENV["BENCH"] - - class TestMeme < MiniTest::Unit::TestCase - # Override self.bench_range or default range is [1, 10, 100, 1_000, 10_000] - def bench_my_algorithm - assert_performance_linear 0.9999 do |n| # n is a range value - @obj.my_algorithm(n) - end - end - end - -Or add them to your specs. If you make benchmarks optional, you'll -need to wrap your benchmarks in a conditional since the methods won't -be defined. - - describe Meme do - if ENV["BENCH"] then - bench_performance_linear "my_algorithm", 0.9999 do |n| - 100.times do - @obj.my_algorithm(n) - end - end - end - end - -outputs something like: - - # Running benchmarks: - - TestBlah 100 1000 10000 - bench_my_algorithm 0.006167 0.079279 0.786993 - bench_other_algorithm 0.061679 0.792797 7.869932 - -Output is tab-delimited to make it easy to paste into a spreadsheet. - -=== Mocks - - class MemeAsker - def initialize(meme) - @meme = meme - end - - def ask(question) - method = question.tr(" ","_") + "?" - @meme.__send__(method) - end - end - - require 'minitest/autorun' - - describe MemeAsker do - before do - @meme = MiniTest::Mock.new - @meme_asker = MemeAsker.new @meme - end - - describe "#ask" do - describe "when passed an unpunctuated question" do - it "should invoke the appropriate predicate method on the meme" do - @meme.expect :will_it_blend?, :return_value - @meme_asker.ask "will it blend" - @meme.verify - end - end - end - end - -=== Stubs - - def test_stale_eh - obj_under_test = Something.new - - refute obj_under_test.stale? - - Time.stub :now, Time.at(0) do # stub goes away once the block is done - assert obj_under_test.stale? - end - end - -A note on stubbing: In order to stub a method, the method must -actually exist prior to stubbing. Use a singleton method to create a -new non-existing method: - - def obj_under_test.fake_method - ... - end - -=== Customizable Test Runner Types: - -MiniTest::Unit.runner=(runner) provides an easy way of creating custom -test runners for specialized needs. Justin Weiss provides the -following real-world example to create an alternative to regular -fixture loading: - - class MiniTestWithHooks::Unit < MiniTest::Unit - def before_suites - end - - def after_suites - end - - def _run_suites(suites, type) - begin - before_suites - super(suites, type) - ensure - after_suites - end - end - - def _run_suite(suite, type) - begin - suite.before_suite - super(suite, type) - ensure - suite.after_suite - end - end - end - - module MiniTestWithTransactions - class Unit < MiniTestWithHooks::Unit - include TestSetupHelper - - def before_suites - super - setup_nested_transactions - # load any data we want available for all tests - end - - def after_suites - teardown_nested_transactions - super - end - end - end - - MiniTest::Unit.runner = MiniTestWithTransactions::Unit.new - -== FAQ - -=== How to test SimpleDelegates? - -The following implementation and test: - - class Worker < SimpleDelegator - def work - end - end - - describe Worker do - before do - @worker = Worker.new(Object.new) - end - - it "must respond to work" do - @worker.must_respond_to :work - end - end - -outputs a failure: - - 1) Failure: - Worker#test_0001_must respond to work [bug11.rb:16]: - Expected #<Object:0x007f9e7184f0a0> (Object) to respond to #work. - -Worker is a SimpleDelegate which in 1.9+ is a subclass of BasicObject. -Expectations are put on Object (one level down) so the Worker -(SimpleDelegate) hits `method_missing` and delegates down to the -`Object.new` instance. That object doesn't respond to work so the test -fails. - -You can bypass `SimpleDelegate#method_missing` by extending the worker -with `MiniTest::Expectations`. You can either do that in your setup at -the instance level, like: - - before do - @worker = Worker.new(Object.new) - @worker.extend MiniTest::Expectations - end - -or you can extend the Worker class (within the test file!), like: - - class Worker - include ::MiniTest::Expectations - end - -== Known Extensions: - -capybara_minitest_spec :: Bridge between Capybara RSpec matchers and MiniTest::Spec expectations (e.g. page.must_have_content('Title')). -minispec-metadata :: Metadata for describe/it blocks - (e.g. `it 'requires JS driver', js: true do`) -minitest-ansi :: Colorize minitest output with ANSI colors. -minitest-around :: Around block for minitest. An alternative to setup/teardown dance. -minitest-capistrano :: Assertions and expectations for testing Capistrano recipes -minitest-capybara :: Capybara matchers support for minitest unit and spec -minitest-chef-handler :: Run Minitest suites as Chef report handlers -minitest-ci :: CI reporter plugin for MiniTest. -minitest-colorize :: Colorize MiniTest output and show failing tests instantly. -minitest-context :: Defines contexts for code reuse in MiniTest - specs that share common expectations. -minitest-debugger :: Wraps assert so failed assertions drop into - the ruby debugger. -minitest-display :: Patches MiniTest to allow for an easily configurable output. -minitest-emoji :: Print out emoji for your test passes, fails, and skips. -minitest-english :: Semantically symmetric aliases for assertions and expectations. -minitest-excludes :: Clean API for excluding certain tests you - don't want to run under certain conditions. -minitest-firemock :: Makes your MiniTest mocks more resilient. -minitest-great_expectations :: Generally useful additions to minitest's assertions and expectations -minitest-growl :: Test notifier for minitest via growl. -minitest-implicit-subject :: Implicit declaration of the test subject. -minitest-instrument :: Instrument ActiveSupport::Notifications when - test method is executed -minitest-instrument-db :: Store information about speed of test - execution provided by minitest-instrument in database -minitest-libnotify :: Test notifier for minitest via libnotify. -minitest-macruby :: Provides extensions to minitest for macruby UI testing. -minitest-matchers :: Adds support for RSpec-style matchers to minitest. -minitest-metadata :: Annotate tests with metadata (key-value). -minitest-mongoid :: Mongoid assertion matchers for MiniTest -minitest-must_not :: Provides must_not as an alias for wont in MiniTest -minitest-nc :: Test notifier for minitest via Mountain Lion's Notification Center -minitest-predicates :: Adds support for .predicate? methods -minitest-rails :: MiniTest integration for Rails 3.x -minitest-rails-capybara :: Capybara integration for MiniTest::Rails -minitest-reporters :: Create customizable MiniTest output formats -minitest-should_syntax :: RSpec-style +x.should == y+ assertions for MiniTest -minitest-shouldify :: Adding all manner of shoulds to MiniTest (bad idea) -minitest-spec-context :: Provides rspec-ish context method to MiniTest::Spec -minitest-spec-magic :: Minitest::Spec extensions for Rails and beyond -minitest-spec-rails :: Drop in MiniTest::Spec superclass for ActiveSupport::TestCase. -minitest-stub-const :: Stub constants for the duration of a block -minitest-tags :: add tags for minitest -minitest-wscolor :: Yet another test colorizer. -minitest_owrapper :: Get tests results as a TestResult object. -minitest_should :: Shoulda style syntax for minitest test::unit. -minitest_tu_shim :: minitest_tu_shim bridges between test/unit and minitest. -mongoid-minitest :: MiniTest matchers for Mongoid. -pry-rescue :: A pry plugin w/ minitest support. See pry-rescue/minitest.rb. - -== Unknown Extensions: - -Authors... Please send me a pull request with a description of your minitest extension. - -* assay-minitest -* detroit-minitest -* em-minitest-spec -* flexmock-minitest -* guard-minitest -* guard-minitest-decisiv -* minitest-activemodel -* minitest-ar-assertions -* minitest-capybara-unit -* minitest-colorer -* minitest-deluxe -* minitest-extra-assertions -* minitest-rails-shoulda -* minitest-spec -* minitest-spec-should -* minitest-sugar -* minitest_should -* mongoid-minitest -* spork-minitest - -== REQUIREMENTS: - -* Ruby 1.8, maybe even 1.6 or lower. No magic is involved. - -== INSTALL: - - sudo gem install minitest - -On 1.9, you already have it. To get newer candy you can still install -the gem, but you'll need to activate the gem explicitly to use it: - - require 'rubygems' - gem 'minitest' # ensures you're using the gem, and not the built in MT - require 'minitest/autorun' - - # ... usual testing stuffs ... - -DO NOTE: There is a serious problem with the way that ruby 1.9/2.0 -packages their own gems. They install a gem specification file, but -don't install the gem contents in the gem path. This messes up -Gem.find_files and many other things (gem which, gem contents, etc). - -Just install minitest as a gem for real and you'll be happier. - -== LICENSE: - -(The MIT License) - -Copyright (c) Ryan Davis, seattle.rb - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -'Software'), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tool/lib/minitest/autorun.rb b/tool/lib/minitest/autorun.rb deleted file mode 100644 index 844096623c..0000000000 --- a/tool/lib/minitest/autorun.rb +++ /dev/null @@ -1,14 +0,0 @@ -# encoding: utf-8 -# frozen_string_literal: true - -begin - require 'rubygems' - gem 'minitest' -rescue Gem::LoadError - # do nothing -end - -require 'minitest/unit' -require 'minitest/mock' - -MiniTest::Unit.autorun diff --git a/tool/lib/minitest/benchmark.rb b/tool/lib/minitest/benchmark.rb deleted file mode 100644 index b3f2bc28b3..0000000000 --- a/tool/lib/minitest/benchmark.rb +++ /dev/null @@ -1,418 +0,0 @@ -# encoding: utf-8 -# frozen_string_literal: true - -require 'minitest/unit' - -class MiniTest::Unit # :nodoc: - def run_benchmarks # :nodoc: - _run_anything :benchmark - end - - def benchmark_suite_header suite # :nodoc: - "\n#{suite}\t#{suite.bench_range.join("\t")}" - end - - class TestCase - ## - # Returns a set of ranges stepped exponentially from +min+ to - # +max+ by powers of +base+. Eg: - # - # bench_exp(2, 16, 2) # => [2, 4, 8, 16] - - def self.bench_exp min, max, base = 10 - min = (Math.log10(min) / Math.log10(base)).to_i - max = (Math.log10(max) / Math.log10(base)).to_i - - (min..max).map { |m| base ** m }.to_a - end - - ## - # Returns a set of ranges stepped linearly from +min+ to +max+ by - # +step+. Eg: - # - # bench_linear(20, 40, 10) # => [20, 30, 40] - - def self.bench_linear min, max, step = 10 - (min..max).step(step).to_a - rescue LocalJumpError # 1.8.6 - r = []; (min..max).step(step) { |n| r << n }; r - end - - ## - # Returns the benchmark methods (methods that start with bench_) - # for that class. - - def self.benchmark_methods # :nodoc: - public_instance_methods(true).grep(/^bench_/).map { |m| m.to_s }.sort - end - - ## - # Returns all test suites that have benchmark methods. - - def self.benchmark_suites - TestCase.test_suites.reject { |s| s.benchmark_methods.empty? } - end - - ## - # Specifies the ranges used for benchmarking for that class. - # Defaults to exponential growth from 1 to 10k by powers of 10. - # Override if you need different ranges for your benchmarks. - # - # See also: ::bench_exp and ::bench_linear. - - def self.bench_range - bench_exp 1, 10_000 - end - - ## - # Runs the given +work+, gathering the times of each run. Range - # and times are then passed to a given +validation+ proc. Outputs - # the benchmark name and times in tab-separated format, making it - # easy to paste into a spreadsheet for graphing or further - # analysis. - # - # Ranges are specified by ::bench_range. - # - # Eg: - # - # def bench_algorithm - # validation = proc { |x, y| ... } - # assert_performance validation do |n| - # @obj.algorithm(n) - # end - # end - - def assert_performance validation, &work - range = self.class.bench_range - - io.print "#{__name__}" - - times = [] - - range.each do |x| - GC.start - t0 = Time.now - instance_exec(x, &work) - t = Time.now - t0 - - io.print "\t%9.6f" % t - times << t - end - io.puts - - validation[range, times] - end - - ## - # Runs the given +work+ and asserts that the times gathered fit to - # match a constant rate (eg, linear slope == 0) within a given - # +threshold+. Note: because we're testing for a slope of 0, R^2 - # is not a good determining factor for the fit, so the threshold - # is applied against the slope itself. As such, you probably want - # to tighten it from the default. - # - # See http://www.graphpad.com/curvefit/goodness_of_fit.htm for - # more details. - # - # Fit is calculated by #fit_linear. - # - # Ranges are specified by ::bench_range. - # - # Eg: - # - # def bench_algorithm - # assert_performance_constant 0.9999 do |n| - # @obj.algorithm(n) - # end - # end - - def assert_performance_constant threshold = 0.99, &work - validation = proc do |range, times| - a, b, rr = fit_linear range, times - assert_in_delta 0, b, 1 - threshold - [a, b, rr] - end - - assert_performance validation, &work - end - - ## - # Runs the given +work+ and asserts that the times gathered fit to - # match a exponential curve within a given error +threshold+. - # - # Fit is calculated by #fit_exponential. - # - # Ranges are specified by ::bench_range. - # - # Eg: - # - # def bench_algorithm - # assert_performance_exponential 0.9999 do |n| - # @obj.algorithm(n) - # end - # end - - def assert_performance_exponential threshold = 0.99, &work - assert_performance validation_for_fit(:exponential, threshold), &work - end - - ## - # Runs the given +work+ and asserts that the times gathered fit to - # match a logarithmic curve within a given error +threshold+. - # - # Fit is calculated by #fit_logarithmic. - # - # Ranges are specified by ::bench_range. - # - # Eg: - # - # def bench_algorithm - # assert_performance_logarithmic 0.9999 do |n| - # @obj.algorithm(n) - # end - # end - - def assert_performance_logarithmic threshold = 0.99, &work - assert_performance validation_for_fit(:logarithmic, threshold), &work - end - - ## - # Runs the given +work+ and asserts that the times gathered fit to - # match a straight line within a given error +threshold+. - # - # Fit is calculated by #fit_linear. - # - # Ranges are specified by ::bench_range. - # - # Eg: - # - # def bench_algorithm - # assert_performance_linear 0.9999 do |n| - # @obj.algorithm(n) - # end - # end - - def assert_performance_linear threshold = 0.99, &work - assert_performance validation_for_fit(:linear, threshold), &work - end - - ## - # Runs the given +work+ and asserts that the times gathered curve - # fit to match a power curve within a given error +threshold+. - # - # Fit is calculated by #fit_power. - # - # Ranges are specified by ::bench_range. - # - # Eg: - # - # def bench_algorithm - # assert_performance_power 0.9999 do |x| - # @obj.algorithm - # end - # end - - def assert_performance_power threshold = 0.99, &work - assert_performance validation_for_fit(:power, threshold), &work - end - - ## - # Takes an array of x/y pairs and calculates the general R^2 value. - # - # See: http://en.wikipedia.org/wiki/Coefficient_of_determination - - def fit_error xys - y_bar = sigma(xys) { |x, y| y } / xys.size.to_f - ss_tot = sigma(xys) { |x, y| (y - y_bar) ** 2 } - ss_err = sigma(xys) { |x, y| (yield(x) - y) ** 2 } - - 1 - (ss_err / ss_tot) - end - - ## - # To fit a functional form: y = ae^(bx). - # - # Takes x and y values and returns [a, b, r^2]. - # - # See: http://mathworld.wolfram.com/LeastSquaresFittingExponential.html - - def fit_exponential xs, ys - n = xs.size - xys = xs.zip(ys) - sxlny = sigma(xys) { |x,y| x * Math.log(y) } - slny = sigma(xys) { |x,y| Math.log(y) } - sx2 = sigma(xys) { |x,y| x * x } - sx = sigma xs - - c = n * sx2 - sx ** 2 - a = (slny * sx2 - sx * sxlny) / c - b = ( n * sxlny - sx * slny ) / c - - return Math.exp(a), b, fit_error(xys) { |x| Math.exp(a + b * x) } - end - - ## - # To fit a functional form: y = a + b*ln(x). - # - # Takes x and y values and returns [a, b, r^2]. - # - # See: http://mathworld.wolfram.com/LeastSquaresFittingLogarithmic.html - - def fit_logarithmic xs, ys - n = xs.size - xys = xs.zip(ys) - slnx2 = sigma(xys) { |x,y| Math.log(x) ** 2 } - slnx = sigma(xys) { |x,y| Math.log(x) } - sylnx = sigma(xys) { |x,y| y * Math.log(x) } - sy = sigma(xys) { |x,y| y } - - c = n * slnx2 - slnx ** 2 - b = ( n * sylnx - sy * slnx ) / c - a = (sy - b * slnx) / n - - return a, b, fit_error(xys) { |x| a + b * Math.log(x) } - end - - - ## - # Fits the functional form: a + bx. - # - # Takes x and y values and returns [a, b, r^2]. - # - # See: http://mathworld.wolfram.com/LeastSquaresFitting.html - - def fit_linear xs, ys - n = xs.size - xys = xs.zip(ys) - sx = sigma xs - sy = sigma ys - sx2 = sigma(xs) { |x| x ** 2 } - sxy = sigma(xys) { |x,y| x * y } - - c = n * sx2 - sx**2 - a = (sy * sx2 - sx * sxy) / c - b = ( n * sxy - sx * sy ) / c - - return a, b, fit_error(xys) { |x| a + b * x } - end - - ## - # To fit a functional form: y = ax^b. - # - # Takes x and y values and returns [a, b, r^2]. - # - # See: http://mathworld.wolfram.com/LeastSquaresFittingPowerLaw.html - - def fit_power xs, ys - n = xs.size - xys = xs.zip(ys) - slnxlny = sigma(xys) { |x, y| Math.log(x) * Math.log(y) } - slnx = sigma(xs) { |x | Math.log(x) } - slny = sigma(ys) { | y| Math.log(y) } - slnx2 = sigma(xs) { |x | Math.log(x) ** 2 } - - b = (n * slnxlny - slnx * slny) / (n * slnx2 - slnx ** 2); - a = (slny - b * slnx) / n - - return Math.exp(a), b, fit_error(xys) { |x| (Math.exp(a) * (x ** b)) } - end - - ## - # Enumerates over +enum+ mapping +block+ if given, returning the - # sum of the result. Eg: - # - # sigma([1, 2, 3]) # => 1 + 2 + 3 => 7 - # sigma([1, 2, 3]) { |n| n ** 2 } # => 1 + 4 + 9 => 14 - - def sigma enum, &block - enum = enum.map(&block) if block - enum.inject { |sum, n| sum + n } - end - - ## - # Returns a proc that calls the specified fit method and asserts - # that the error is within a tolerable threshold. - - def validation_for_fit msg, threshold - proc do |range, times| - a, b, rr = send "fit_#{msg}", range, times - assert_operator rr, :>=, threshold - [a, b, rr] - end - end - end -end - -class MiniTest::Spec - ## - # This is used to define a new benchmark method. You usually don't - # use this directly and is intended for those needing to write new - # performance curve fits (eg: you need a specific polynomial fit). - # - # See ::bench_performance_linear for an example of how to use this. - - def self.bench name, &block - define_method "bench_#{name.gsub(/\W+/, '_')}", &block - end - - ## - # Specifies the ranges used for benchmarking for that class. - # - # bench_range do - # bench_exp(2, 16, 2) - # end - # - # See Unit::TestCase.bench_range for more details. - - def self.bench_range &block - return super unless block - - meta = (class << self; self; end) - meta.send :define_method, "bench_range", &block - end - - ## - # Create a benchmark that verifies that the performance is linear. - # - # describe "my class" do - # bench_performance_linear "fast_algorithm", 0.9999 do |n| - # @obj.fast_algorithm(n) - # end - # end - - def self.bench_performance_linear name, threshold = 0.99, &work - bench name do - assert_performance_linear threshold, &work - end - end - - ## - # Create a benchmark that verifies that the performance is constant. - # - # describe "my class" do - # bench_performance_constant "zoom_algorithm!" do |n| - # @obj.zoom_algorithm!(n) - # end - # end - - def self.bench_performance_constant name, threshold = 0.99, &work - bench name do - assert_performance_constant threshold, &work - end - end - - ## - # Create a benchmark that verifies that the performance is exponential. - # - # describe "my class" do - # bench_performance_exponential "algorithm" do |n| - # @obj.algorithm(n) - # end - # end - - def self.bench_performance_exponential name, threshold = 0.99, &work - bench name do - assert_performance_exponential threshold, &work - end - end -end diff --git a/tool/lib/minitest/mock.rb b/tool/lib/minitest/mock.rb deleted file mode 100644 index 224b06cb89..0000000000 --- a/tool/lib/minitest/mock.rb +++ /dev/null @@ -1,196 +0,0 @@ -# encoding: utf-8 -# frozen_string_literal: true - -class MockExpectationError < StandardError; end # :nodoc: - -## -# A simple and clean mock object framework. - -module MiniTest # :nodoc: - - ## - # All mock objects are an instance of Mock - - class Mock - alias :__respond_to? :respond_to? - - skip_methods = %w(object_id respond_to_missing? inspect === to_s) - - instance_methods.each do |m| - undef_method m unless skip_methods.include?(m.to_s) || m =~ /^__/ - end - - def initialize # :nodoc: - @expected_calls = Hash.new { |calls, name| calls[name] = [] } - @actual_calls = Hash.new { |calls, name| calls[name] = [] } - end - - ## - # Expect that method +name+ is called, optionally with +args+ or a - # +blk+, and returns +retval+. - # - # @mock.expect(:meaning_of_life, 42) - # @mock.meaning_of_life # => 42 - # - # @mock.expect(:do_something_with, true, [some_obj, true]) - # @mock.do_something_with(some_obj, true) # => true - # - # @mock.expect(:do_something_else, true) do |a1, a2| - # a1 == "buggs" && a2 == :bunny - # end - # - # +args+ is compared to the expected args using case equality (ie, the - # '===' operator), allowing for less specific expectations. - # - # @mock.expect(:uses_any_string, true, [String]) - # @mock.uses_any_string("foo") # => true - # @mock.verify # => true - # - # @mock.expect(:uses_one_string, true, ["foo"] - # @mock.uses_one_string("bar") # => true - # @mock.verify # => raises MockExpectationError - - def expect(name, retval, args=[], &blk) - if block_given? - raise ArgumentError, "args ignored when block given" unless args.empty? - @expected_calls[name] << { :retval => retval, :block => blk } - else - raise ArgumentError, "args must be an array" unless Array === args - @expected_calls[name] << { :retval => retval, :args => args } - end - self - end - - def __call name, data # :nodoc: - case data - when Hash then - "#{name}(#{data[:args].inspect[1..-2]}) => #{data[:retval].inspect}" - else - data.map { |d| __call name, d }.join ", " - end - end - - ## - # Verify that all methods were called as expected. Raises - # +MockExpectationError+ if the mock object was not called as - # expected. - - def verify - @expected_calls.each do |name, calls| - calls.each do |expected| - msg1 = "expected #{__call name, expected}" - msg2 = "#{msg1}, got [#{__call name, @actual_calls[name]}]" - - raise MockExpectationError, msg2 if - @actual_calls.has_key?(name) and - not @actual_calls[name].include?(expected) - - raise MockExpectationError, msg1 unless - @actual_calls.has_key?(name) and - @actual_calls[name].include?(expected) - end - end - true - end - - def method_missing(sym, *args) # :nodoc: - unless @expected_calls.has_key?(sym) then - raise NoMethodError, "unmocked method %p, expected one of %p" % - [sym, @expected_calls.keys.sort_by(&:to_s)] - end - - index = @actual_calls[sym].length - expected_call = @expected_calls[sym][index] - - unless expected_call then - raise MockExpectationError, "No more expects available for %p: %p" % - [sym, args] - end - - expected_args, retval, val_block = - expected_call.values_at(:args, :retval, :block) - - if val_block then - raise MockExpectationError, "mocked method %p failed block w/ %p" % - [sym, args] unless val_block.call(args) - - # keep "verify" happy - @actual_calls[sym] << expected_call - return retval - end - - if expected_args.size != args.size then - raise ArgumentError, "mocked method %p expects %d arguments, got %d" % - [sym, expected_args.size, args.size] - end - - fully_matched = expected_args.zip(args).all? { |mod, a| - mod === a or mod == a - } - - unless fully_matched then - raise MockExpectationError, "mocked method %p called with unexpected arguments %p" % - [sym, args] - end - - @actual_calls[sym] << { - :retval => retval, - :args => expected_args.zip(args).map { |mod, a| mod === a ? mod : a } - } - - retval - end - - def respond_to?(sym, include_private = false) # :nodoc: - return true if @expected_calls.has_key?(sym.to_sym) - return __respond_to?(sym, include_private) - end - end -end - -class Object # :nodoc: - - ## - # Add a temporary stubbed method replacing +name+ for the duration - # of the +block+. If +val_or_callable+ responds to #call, then it - # returns the result of calling it, otherwise returns the value - # as-is. Cleans up the stub at the end of the +block+. The method - # +name+ must exist before stubbing. - # - # def test_stale_eh - # obj_under_test = Something.new - # refute obj_under_test.stale? - # - # Time.stub :now, Time.at(0) do - # assert obj_under_test.stale? - # end - # end - - def stub name, val_or_callable, &block - new_name = "__minitest_stub__#{name}" - - metaclass = class << self; self; end - - if respond_to? name and not methods.map(&:to_s).include? name.to_s then - metaclass.send :define_method, name do |*args| - super(*args) - end - end - - metaclass.send :alias_method, new_name, name - - metaclass.send :define_method, name do |*args| - if val_or_callable.respond_to? :call then - val_or_callable.call(*args) - else - val_or_callable - end - end - - yield self - ensure - metaclass.send :undef_method, name - metaclass.send :alias_method, name, new_name - metaclass.send :undef_method, new_name - end -end diff --git a/tool/lib/minitest/unit.rb b/tool/lib/minitest/unit.rb deleted file mode 100644 index d922a5fc41..0000000000 --- a/tool/lib/minitest/unit.rb +++ /dev/null @@ -1,1420 +0,0 @@ -# encoding: utf-8 -# frozen_string_literal: true - -require "optparse" -require "rbconfig" -require "leakchecker" - -## -# Minimal (mostly drop-in) replacement for test-unit. -# -# :include: README.txt - -module MiniTest - - def self.const_missing name # :nodoc: - case name - when :MINI_DIR then - msg = "MiniTest::MINI_DIR was removed. Don't violate other's internals." - warn "WAR\NING: #{msg}" - warn "WAR\NING: Used by #{caller.first}." - const_set :MINI_DIR, "bad value" - else - super - end - end - - ## - # Assertion base class - - class Assertion < Exception; end - - ## - # Assertion raised when skipping a test - - class Skip < Assertion; end - - class << self - ## - # Filter object for backtraces. - - attr_accessor :backtrace_filter - end - - class BacktraceFilter # :nodoc: - def filter bt - return ["No backtrace"] unless bt - - new_bt = [] - - unless $DEBUG then - bt.each do |line| - break if line =~ /lib\/minitest/ - new_bt << line - end - - new_bt = bt.reject { |line| line =~ /lib\/minitest/ } if new_bt.empty? - new_bt = bt.dup if new_bt.empty? - else - new_bt = bt.dup - end - - new_bt - end - end - - self.backtrace_filter = BacktraceFilter.new - - def self.filter_backtrace bt # :nodoc: - backtrace_filter.filter bt - end - - ## - # MiniTest Assertions. All assertion methods accept a +msg+ which is - # printed if the assertion fails. - - module Assertions - ## - # Returns the diff command to use in #diff. Tries to intelligently - # figure out what diff to use. - - def self.diff - unless defined? @diff - exe = RbConfig::CONFIG['EXEEXT'] - @diff = %W"gdiff#{exe} diff#{exe}".find do |diff| - if system(diff, "-u", __FILE__, __FILE__) - break "#{diff} -u" - end - end - end - - @diff - end - - ## - # Set the diff command to use in #diff. - - def self.diff= o - @diff = o - end - - ## - # Returns a diff between +exp+ and +act+. If there is no known - # diff command or if it doesn't make sense to diff the output - # (single line, short output), then it simply returns a basic - # comparison between the two. - - def diff exp, act - require "tempfile" - - expect = mu_pp_for_diff exp - butwas = mu_pp_for_diff act - result = nil - - need_to_diff = - MiniTest::Assertions.diff && - (expect.include?("\n") || - butwas.include?("\n") || - expect.size > 30 || - butwas.size > 30 || - expect == butwas) - - return "Expected: #{mu_pp exp}\n Actual: #{mu_pp act}" unless - need_to_diff - - tempfile_a = nil - tempfile_b = nil - - Tempfile.open("expect") do |a| - tempfile_a = a - a.puts expect - a.flush - - Tempfile.open("butwas") do |b| - tempfile_b = b - b.puts butwas - b.flush - - result = `#{MiniTest::Assertions.diff} #{a.path} #{b.path}` - result.sub!(/^\-\-\- .+/, "--- expected") - result.sub!(/^\+\+\+ .+/, "+++ actual") - - if result.empty? then - klass = exp.class - result = [ - "No visible difference in the #{klass}#inspect output.\n", - "You should look at the implementation of #== on ", - "#{klass} or its members.\n", - expect, - ].join - end - end - end - - result - ensure - tempfile_a.close! if tempfile_a - tempfile_b.close! if tempfile_b - end - - ## - # This returns a human-readable version of +obj+. By default - # #inspect is called. You can override this to use #pretty_print - # if you want. - - def mu_pp obj - s = obj.inspect - s = s.encode Encoding.default_external if defined? Encoding - s - end - - ## - # This returns a diff-able human-readable version of +obj+. This - # differs from the regular mu_pp because it expands escaped - # newlines and makes hex-values generic (like object_ids). This - # uses mu_pp to do the first pass and then cleans it up. - - def mu_pp_for_diff obj - mu_pp(obj).gsub(/(?<!\\)(?:\\\\)*\K\\n/, "\n").gsub(/:0x[a-fA-F0-9]{4,}/m, ':0xXXXXXX') - end - - def _assertions= n # :nodoc: - @_assertions = n - end - - def _assertions # :nodoc: - @_assertions ||= 0 - end - - ## - # Fails unless +test+ is a true value. - - def assert test, msg = nil - msg ||= "Failed assertion, no message given." - self._assertions += 1 - unless test then - msg = msg.call if Proc === msg - raise MiniTest::Assertion, msg - end - true - end - - ## - # Fails unless +obj+ is empty. - - def assert_empty obj, msg = nil - msg = message(msg) { "Expected #{mu_pp(obj)} to be empty" } - assert_respond_to obj, :empty? - assert obj.empty?, msg - end - - ## - # Fails unless <tt>exp == act</tt> printing the difference between - # the two, if possible. - # - # If there is no visible difference but the assertion fails, you - # should suspect that your #== is buggy, or your inspect output is - # missing crucial details. - # - # For floats use assert_in_delta. - # - # See also: MiniTest::Assertions.diff - - def assert_equal exp, act, msg = nil - msg = message(msg, "") { diff exp, act } - assert exp == act, msg - end - - ## - # For comparing Floats. Fails unless +exp+ and +act+ are within +delta+ - # of each other. - # - # assert_in_delta Math::PI, (22.0 / 7.0), 0.01 - - def assert_in_delta exp, act, delta = 0.001, msg = nil - n = (exp - act).abs - msg = message(msg) { - "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}" - } - assert delta >= n, msg - end - - ## - # For comparing Floats. Fails unless +exp+ and +act+ have a relative - # error less than +epsilon+. - - def assert_in_epsilon a, b, epsilon = 0.001, msg = nil - assert_in_delta a, b, [a.abs, b.abs].min * epsilon, msg - end - - ## - # Fails unless +collection+ includes +obj+. - - def assert_includes collection, obj, msg = nil - msg = message(msg) { - "Expected #{mu_pp(collection)} to include #{mu_pp(obj)}" - } - assert_respond_to collection, :include? - assert collection.include?(obj), msg - end - - ## - # Fails unless +obj+ is an instance of +cls+. - - def assert_instance_of cls, obj, msg = nil - msg = message(msg) { - "Expected #{mu_pp(obj)} to be an instance of #{cls}, not #{obj.class}" - } - - assert obj.instance_of?(cls), msg - end - - ## - # Fails unless +obj+ is a kind of +cls+. - - def assert_kind_of cls, obj, msg = nil # TODO: merge with instance_of - msg = message(msg) { - "Expected #{mu_pp(obj)} to be a kind of #{cls}, not #{obj.class}" } - - assert obj.kind_of?(cls), msg - end - - ## - # Fails unless +matcher+ <tt>=~</tt> +obj+. - - def assert_match matcher, obj, msg = nil - msg = message(msg) { "Expected #{mu_pp matcher} to match #{mu_pp obj}" } - assert_respond_to matcher, :"=~" - matcher = Regexp.new Regexp.escape matcher if String === matcher - assert matcher =~ obj, msg - end - - ## - # Fails unless +obj+ is nil - - def assert_nil obj, msg = nil - msg = message(msg) { "Expected #{mu_pp(obj)} to be nil" } - assert obj.nil?, msg - end - - ## - # For testing with binary operators. - # - # assert_operator 5, :<=, 4 - - def assert_operator o1, op, o2 = (predicate = true; nil), msg = nil - return assert_predicate o1, op, msg if predicate - msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op} #{mu_pp(o2)}" } - assert o1.__send__(op, o2), msg - end - - ## - # Fails if stdout or stderr do not output the expected results. - # Pass in nil if you don't care about that streams output. Pass in - # "" if you require it to be silent. Pass in a regexp if you want - # to pattern match. - # - # NOTE: this uses #capture_io, not #capture_subprocess_io. - # - # See also: #assert_silent - - def assert_output stdout = nil, stderr = nil - out, err = capture_io do - yield - end - - err_msg = Regexp === stderr ? :assert_match : :assert_equal if stderr - out_msg = Regexp === stdout ? :assert_match : :assert_equal if stdout - - y = send err_msg, stderr, err, "In stderr" if err_msg - x = send out_msg, stdout, out, "In stdout" if out_msg - - (!stdout || x) && (!stderr || y) - end - - ## - # For testing with predicates. - # - # assert_predicate str, :empty? - # - # This is really meant for specs and is front-ended by assert_operator: - # - # str.must_be :empty? - - def assert_predicate o1, op, msg = nil - msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op}" } - assert o1.__send__(op), msg - end - - ## - # Fails unless the block raises one of +exp+. Returns the - # exception matched so you can check the message, attributes, etc. - - def assert_raises *exp - msg = "#{exp.pop}.\n" if String === exp.last - - begin - yield - rescue MiniTest::Skip => e - return e if exp.include? MiniTest::Skip - raise e - rescue Exception => e - expected = exp.any? { |ex| - if ex.instance_of? Module then - e.kind_of? ex - else - e.instance_of? ex - end - } - - assert expected, proc { - exception_details(e, "#{msg}#{mu_pp(exp)} exception expected, not") - } - - return e - end - - exp = exp.first if exp.size == 1 - - flunk "#{msg}#{mu_pp(exp)} expected but nothing was raised." - end - - ## - # Fails unless +obj+ responds to +meth+. - - def assert_respond_to obj, meth, msg = nil - msg = message(msg) { - "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}" - } - assert obj.respond_to?(meth), msg - end - - ## - # Fails unless +exp+ and +act+ are #equal? - - def assert_same exp, act, msg = nil - msg = message(msg) { - data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] - "Expected %s (oid=%d) to be the same as %s (oid=%d)" % data - } - assert exp.equal?(act), msg - end - - ## - # +send_ary+ is a receiver, message and arguments. - # - # Fails unless the call returns a true value - # TODO: I should prolly remove this from specs - - def assert_send send_ary, m = nil - recv, msg, *args = send_ary - m = message(m) { - "Expected #{mu_pp(recv)}.#{msg}(*#{mu_pp(args)}) to return true" } - assert recv.__send__(msg, *args), m - end - - ## - # Fails if the block outputs anything to stderr or stdout. - # - # See also: #assert_output - - def assert_silent - assert_output "", "" do - yield - end - end - - ## - # Fails unless the block throws +sym+ - - def assert_throws sym, msg = nil - default = "Expected #{mu_pp(sym)} to have been thrown" - caught = true - catch(sym) do - begin - yield - rescue ThreadError => e # wtf?!? 1.8 + threads == suck - default += ", not \:#{e.message[/uncaught throw \`(\w+?)\'/, 1]}" - rescue ArgumentError => e # 1.9 exception - default += ", not #{e.message.split(/ /).last}" - rescue NameError => e # 1.8 exception - default += ", not #{e.name.inspect}" - end - caught = false - end - - assert caught, message(msg) { default } - end - - ## - # Captures $stdout and $stderr into strings: - # - # out, err = capture_io do - # puts "Some info" - # warn "You did a bad thing" - # end - # - # assert_match %r%info%, out - # assert_match %r%bad%, err - # - # NOTE: For efficiency, this method uses StringIO and does not - # capture IO for subprocesses. Use #capture_subprocess_io for - # that. - - def capture_io - require 'stringio' - - captured_stdout, captured_stderr = StringIO.new, StringIO.new - - synchronize do - orig_stdout, orig_stderr = $stdout, $stderr - $stdout, $stderr = captured_stdout, captured_stderr - - begin - yield - ensure - $stdout = orig_stdout - $stderr = orig_stderr - end - end - - return captured_stdout.string, captured_stderr.string - end - - ## - # Captures $stdout and $stderr into strings, using Tempfile to - # ensure that subprocess IO is captured as well. - # - # out, err = capture_subprocess_io do - # system "echo Some info" - # system "echo You did a bad thing 1>&2" - # end - # - # assert_match %r%info%, out - # assert_match %r%bad%, err - # - # NOTE: This method is approximately 10x slower than #capture_io so - # only use it when you need to test the output of a subprocess. - - def capture_subprocess_io - require 'tempfile' - - captured_stdout, captured_stderr = Tempfile.new("out"), Tempfile.new("err") - - synchronize do - orig_stdout, orig_stderr = $stdout.dup, $stderr.dup - $stdout.reopen captured_stdout - $stderr.reopen captured_stderr - - begin - yield - - $stdout.rewind - $stderr.rewind - - [captured_stdout.read, captured_stderr.read] - ensure - $stdout.reopen orig_stdout - $stderr.reopen orig_stderr - orig_stdout.close - orig_stderr.close - captured_stdout.close! - captured_stderr.close! - end - end - end - - ## - # Returns details for exception +e+ - - def exception_details e, msg - [ - "#{msg}", - "Class: <#{e.class}>", - "Message: <#{e.message.inspect}>", - "---Backtrace---", - "#{MiniTest::filter_backtrace(e.backtrace).join("\n")}", - "---------------", - ].join "\n" - end - - ## - # Fails with +msg+ - - def flunk msg = nil - msg ||= "Epic Fail!" - assert false, msg - end - - ## - # Returns a proc that will output +msg+ along with the default message. - - def message msg = nil, ending = ".", &default - proc { - msg = msg.call.chomp(".") if Proc === msg - custom_message = "#{msg}.\n" unless msg.nil? or msg.to_s.empty? - "#{custom_message}#{default.call}#{ending}" - } - end - - ## - # used for counting assertions - - def pass msg = nil - assert true - end - - ## - # Fails if +test+ is a true value - - def refute test, msg = nil - msg ||= "Failed refutation, no message given" - not assert(! test, msg) - end - - ## - # Fails if +obj+ is empty. - - def refute_empty obj, msg = nil - msg = message(msg) { "Expected #{mu_pp(obj)} to not be empty" } - assert_respond_to obj, :empty? - refute obj.empty?, msg - end - - ## - # Fails if <tt>exp == act</tt>. - # - # For floats use refute_in_delta. - - def refute_equal exp, act, msg = nil - msg = message(msg) { - "Expected #{mu_pp(act)} to not be equal to #{mu_pp(exp)}" - } - refute exp == act, msg - end - - ## - # For comparing Floats. Fails if +exp+ is within +delta+ of +act+. - # - # refute_in_delta Math::PI, (22.0 / 7.0) - - def refute_in_delta exp, act, delta = 0.001, msg = nil - n = (exp - act).abs - msg = message(msg) { - "Expected |#{exp} - #{act}| (#{n}) to not be <= #{delta}" - } - refute delta >= n, msg - end - - ## - # For comparing Floats. Fails if +exp+ and +act+ have a relative error - # less than +epsilon+. - - def refute_in_epsilon a, b, epsilon = 0.001, msg = nil - refute_in_delta a, b, a * epsilon, msg - end - - ## - # Fails if +collection+ includes +obj+. - - def refute_includes collection, obj, msg = nil - msg = message(msg) { - "Expected #{mu_pp(collection)} to not include #{mu_pp(obj)}" - } - assert_respond_to collection, :include? - refute collection.include?(obj), msg - end - - ## - # Fails if +obj+ is an instance of +cls+. - - def refute_instance_of cls, obj, msg = nil - msg = message(msg) { - "Expected #{mu_pp(obj)} to not be an instance of #{cls}" - } - refute obj.instance_of?(cls), msg - end - - ## - # Fails if +obj+ is a kind of +cls+. - - def refute_kind_of cls, obj, msg = nil # TODO: merge with instance_of - msg = message(msg) { "Expected #{mu_pp(obj)} to not be a kind of #{cls}" } - refute obj.kind_of?(cls), msg - end - - ## - # Fails if +matcher+ <tt>=~</tt> +obj+. - - def refute_match matcher, obj, msg = nil - msg = message(msg) {"Expected #{mu_pp matcher} to not match #{mu_pp obj}"} - assert_respond_to matcher, :"=~" - matcher = Regexp.new Regexp.escape matcher if String === matcher - refute matcher =~ obj, msg - end - - ## - # Fails if +obj+ is nil. - - def refute_nil obj, msg = nil - msg = message(msg) { "Expected #{mu_pp(obj)} to not be nil" } - refute obj.nil?, msg - end - - ## - # Fails if +o1+ is not +op+ +o2+. Eg: - # - # refute_operator 1, :>, 2 #=> pass - # refute_operator 1, :<, 2 #=> fail - - def refute_operator o1, op, o2 = (predicate = true; nil), msg = nil - return refute_predicate o1, op, msg if predicate - msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op} #{mu_pp(o2)}"} - refute o1.__send__(op, o2), msg - end - - ## - # For testing with predicates. - # - # refute_predicate str, :empty? - # - # This is really meant for specs and is front-ended by refute_operator: - # - # str.wont_be :empty? - - def refute_predicate o1, op, msg = nil - msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op}" } - refute o1.__send__(op), msg - end - - ## - # Fails if +obj+ responds to the message +meth+. - - def refute_respond_to obj, meth, msg = nil - msg = message(msg) { "Expected #{mu_pp(obj)} to not respond to #{meth}" } - - refute obj.respond_to?(meth), msg - end - - ## - # Fails if +exp+ is the same (by object identity) as +act+. - - def refute_same exp, act, msg = nil - msg = message(msg) { - data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] - "Expected %s (oid=%d) to not be the same as %s (oid=%d)" % data - } - refute exp.equal?(act), msg - end - - ## - # Skips the current test. Gets listed at the end of the run but - # doesn't cause a failure exit code. - - def skip msg = nil, bt = caller - msg ||= "Skipped, no message given" - @skip = true - raise MiniTest::Skip, msg, bt - end - - ## - # Was this testcase skipped? Meant for #teardown. - - def skipped? - defined?(@skip) and @skip - end - - ## - # Takes a block and wraps it with the runner's shared mutex. - - def synchronize - Minitest::Unit.runner.synchronize do - yield - end - end - end - - class Unit # :nodoc: - VERSION = "4.7.5" # :nodoc: - - attr_accessor :report, :failures, :errors, :skips # :nodoc: - attr_accessor :assertion_count # :nodoc: - attr_writer :test_count # :nodoc: - attr_accessor :start_time # :nodoc: - attr_accessor :help # :nodoc: - attr_accessor :verbose # :nodoc: - attr_writer :options # :nodoc: - - ## - # :attr: - # - # if true, installs an "INFO" signal handler (only available to BSD and - # OS X users) which prints diagnostic information about the test run. - # - # This is auto-detected by default but may be overridden by custom - # runners. - - attr_accessor :info_signal - - ## - # Lazy accessor for options. - - def options - @options ||= {} - end - - @@installed_at_exit ||= false - @@out = $stdout - @@after_tests = [] - - ## - # A simple hook allowing you to run a block of code after _all_ of - # the tests are done. Eg: - # - # MiniTest::Unit.after_tests { p $debugging_info } - - def self.after_tests &block - @@after_tests << block - end - - ## - # Registers MiniTest::Unit to run tests at process exit - - def self.autorun - at_exit { - # don't run if there was a non-exit exception - next if $! and not $!.kind_of? SystemExit - - # the order here is important. The at_exit handler must be - # installed before anyone else gets a chance to install their - # own, that way we can be assured that our exit will be last - # to run (at_exit stacks). - exit_code = nil - - at_exit { - @@after_tests.reverse_each(&:call) - exit false if exit_code && exit_code != 0 - } - - exit_code = MiniTest::Unit.new.run ARGV - } unless @@installed_at_exit - @@installed_at_exit = true - end - - ## - # Returns the stream to use for output. - - def self.output - @@out - end - - ## - # Sets MiniTest::Unit to write output to +stream+. $stdout is the default - # output - - def self.output= stream - @@out = stream - end - - ## - # Tells MiniTest::Unit to delegate to +runner+, an instance of a - # MiniTest::Unit subclass, when MiniTest::Unit#run is called. - - def self.runner= runner - @@runner = runner - end - - ## - # Returns the MiniTest::Unit subclass instance that will be used - # to run the tests. A MiniTest::Unit instance is the default - # runner. - - def self.runner - @@runner ||= self.new - end - - ## - # Return all plugins' run methods (methods that start with "run_"). - - def self.plugins - @@plugins ||= (["run_tests"] + - public_instance_methods(false). - grep(/^run_/).map { |s| s.to_s }).uniq - end - - ## - # Return the IO for output. - - def output - self.class.output - end - - def puts *a # :nodoc: - output.puts(*a) - end - - def print *a # :nodoc: - output.print(*a) - end - - def test_count # :nodoc: - @test_count ||= 0 - end - - ## - # Runner for a given +type+ (eg, test vs bench). - - def _run_anything type - suites = TestCase.send "#{type}_suites" - return if suites.empty? - - puts - puts "# Running #{type}s:" - puts - - @test_count, @assertion_count = 0, 0 - test_count = assertion_count = 0 - sync = output.respond_to? :"sync=" # stupid emacs - old_sync, output.sync = output.sync, true if sync - - count = 0 - begin - start = Time.now - - results = _run_suites suites, type - - @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } - @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } - test_count += @test_count - assertion_count += @assertion_count - t = Time.now - start - count += 1 - unless @repeat_count - puts - puts - end - puts "Finished%s %ss in %.6fs, %.4f tests/s, %.4f assertions/s.\n" % - [(@repeat_count ? "(#{count}/#{@repeat_count}) " : ""), type, - t, @test_count.fdiv(t), @assertion_count.fdiv(t)] - end while @repeat_count && count < @repeat_count && - report.empty? && failures.zero? && errors.zero? - - output.sync = old_sync if sync - - report.each_with_index do |msg, i| - puts "\n%3d) %s" % [i + 1, msg] - end - - puts - @test_count = test_count - @assertion_count = assertion_count - - status - end - - ## - # Runs all the +suites+ for a given +type+. - # - - def _run_suites suites, type - suites.map { |suite| _run_suite suite, type } - end - - ## - # Run a single +suite+ for a given +type+. - - def _run_suite suite, type - header = "#{type}_suite_header" - puts send(header, suite) if respond_to? header - - filter = options[:filter] || '/./' - filter = Regexp.new $1 if filter =~ /\/(.*)\// - - all_test_methods = suite.send "#{type}_methods" - - filtered_test_methods = all_test_methods.find_all { |m| - filter === m || filter === "#{suite}##{m}" - } - - leakchecker = LeakChecker.new - - assertions = filtered_test_methods.map { |method| - inst = suite.new method - inst._assertions = 0 - - print "#{suite}##{method} = " if @verbose - - start_time = Time.now if @verbose - result = inst.run self - - print "%.2f s = " % (Time.now - start_time) if @verbose - print result - puts if @verbose - $stdout.flush - - unless defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # compiler process is wrongly considered as leak - leakchecker.check("#{inst.class}\##{inst.__name__}") - end - - inst._assertions - } - - return assertions.size, assertions.inject(0) { |sum, n| sum + n } - end - - ## - # Record the result of a single test. Makes it very easy to gather - # information. Eg: - # - # class StatisticsRecorder < MiniTest::Unit - # def record suite, method, assertions, time, error - # # ... record the results somewhere ... - # end - # end - # - # MiniTest::Unit.runner = StatisticsRecorder.new - # - # NOTE: record might be sent more than once per test. It will be - # sent once with the results from the test itself. If there is a - # failure or error in teardown, it will be sent again with the - # error or failure. - - def record suite, method, assertions, time, error - end - - def location e # :nodoc: - last_before_assertion = "" - e.backtrace.reverse_each do |s| - break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ - last_before_assertion = s - end - last_before_assertion.sub(/:in .*$/, '') - end - - ## - # Writes status for failed test +meth+ in +klass+ which finished with - # exception +e+ - - def puke klass, meth, e - e = case e - when MiniTest::Skip then - @skips += 1 - return "S" unless @verbose - "Skipped:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" - when MiniTest::Assertion then - @failures += 1 - "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" - else - @errors += 1 - bt = MiniTest::filter_backtrace(e.backtrace).join "\n " - "Error:\n#{klass}##{meth}:\n#{e.class}: #{e.message.b}\n #{bt}\n" - end - @report << e - e[0, 1] - end - - def initialize # :nodoc: - @report = [] - @errors = @failures = @skips = 0 - @verbose = false - @mutex = Thread::Mutex.new - @info_signal = Signal.list['INFO'] - @repeat_count = nil - end - - def synchronize # :nodoc: - if @mutex then - @mutex.synchronize { yield } - else - yield - end - end - - def process_args args = [] # :nodoc: - options = {} - orig_args = args.dup - - OptionParser.new do |opts| - opts.banner = 'minitest options:' - opts.version = MiniTest::Unit::VERSION - - opts.on '-h', '--help', 'Display this help.' do - puts opts - exit - end - - opts.on '-s', '--seed SEED', Integer, "Sets random seed" do |m| - options[:seed] = m.to_i - end - - opts.on '-v', '--verbose', "Verbose. Show progress processing files." do - options[:verbose] = true - end - - opts.on '-n', '--name PATTERN', "Filter test names on pattern (e.g. /foo/)" do |a| - options[:filter] = a - end - - opts.parse! args - orig_args -= args - end - - unless options[:seed] then - srand - options[:seed] = srand % 0xFFFF - orig_args << "--seed" << options[:seed].to_s - end - - srand options[:seed] - - self.verbose = options[:verbose] - @help = orig_args.map { |s| s =~ /[\s|&<>$()]/ ? s.inspect : s }.join " " - - options - end - - ## - # Begins the full test run. Delegates to +runner+'s #_run method. - - def run args = [] - self.class.runner._run(args) - end - - ## - # Top level driver, controls all output and filtering. - - def _run args = [] - args = process_args args # ARGH!! blame test/unit process_args - self.options.merge! args - - puts "Run options: #{help}" - - self.class.plugins.each do |plugin| - send plugin - break unless report.empty? - end - - return failures + errors if self.test_count > 0 # or return nil... - rescue Interrupt - abort 'Interrupted' - end - - ## - # Runs test suites matching +filter+. - - def run_tests - _run_anything :test - end - - ## - # Writes status to +io+ - - def status io = self.output - format = "%d tests, %d assertions, %d failures, %d errors, %d skips" - io.puts format % [test_count, assertion_count, failures, errors, skips] - end - - ## - # Provides a simple set of guards that you can use in your tests - # to skip execution if it is not applicable. These methods are - # mixed into TestCase as both instance and class methods so you - # can use them inside or outside of the test methods. - # - # def test_something_for_mri - # skip "bug 1234" if jruby? - # # ... - # end - # - # if windows? then - # # ... lots of test methods ... - # end - - module Guard - - ## - # Is this running on jruby? - - def jruby? platform = RUBY_PLATFORM - "java" == platform - end - - ## - # Is this running on mri? - - def maglev? platform = defined?(RUBY_ENGINE) && RUBY_ENGINE - "maglev" == platform - end - - module_function :maglev? - - ## - # Is this running on mri? - - def mri? platform = RUBY_DESCRIPTION - /^ruby/ =~ platform - end - - ## - # Is this running on rubinius? - - def rubinius? platform = defined?(RUBY_ENGINE) && RUBY_ENGINE - "rbx" == platform - end - - ## - # Is this running on windows? - - def windows? platform = RUBY_PLATFORM - /mswin|mingw/ =~ platform - end - - ## - # Is this running on mingw? - - def mingw? platform = RUBY_PLATFORM - /mingw/ =~ platform - end - - end - - ## - # Provides before/after hooks for setup and teardown. These are - # meant for library writers, NOT for regular test authors. See - # #before_setup for an example. - - module LifecycleHooks - ## - # Runs before every test, after setup. This hook is meant for - # libraries to extend minitest. It is not meant to be used by - # test developers. - # - # See #before_setup for an example. - - def after_setup; end - - ## - # Runs before every test, before setup. This hook is meant for - # libraries to extend minitest. It is not meant to be used by - # test developers. - # - # As a simplistic example: - # - # module MyMinitestPlugin - # def before_setup - # super - # # ... stuff to do before setup is run - # end - # - # def after_setup - # # ... stuff to do after setup is run - # super - # end - # - # def before_teardown - # super - # # ... stuff to do before teardown is run - # end - # - # def after_teardown - # # ... stuff to do after teardown is run - # super - # end - # end - # - # class MiniTest::Unit::TestCase - # include MyMinitestPlugin - # end - - def before_setup; end - - ## - # Runs after every test, before teardown. This hook is meant for - # libraries to extend minitest. It is not meant to be used by - # test developers. - # - # See #before_setup for an example. - - def before_teardown; end - - ## - # Runs after every test, after teardown. This hook is meant for - # libraries to extend minitest. It is not meant to be used by - # test developers. - # - # See #before_setup for an example. - - def after_teardown; end - end - - ## - # Subclass TestCase to create your own tests. Typically you'll want a - # TestCase subclass per implementation class. - # - # See MiniTest::Assertions - - class TestCase - include LifecycleHooks - include Guard - extend Guard - - attr_reader :__name__ # :nodoc: - - PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, - Interrupt, SystemExit] # :nodoc: - - ## - # Runs the tests reporting the status to +runner+ - - def run runner - trap "INFO" do - runner.report.each_with_index do |msg, i| - warn "\n%3d) %s" % [i + 1, msg] - end - warn '' - time = runner.start_time ? Time.now - runner.start_time : 0 - warn "Current Test: %s#%s %.2fs" % [self.class, self.__name__, time] - runner.status $stderr - end if runner.info_signal - - start_time = Time.now - - result = "" - begin - @passed = nil - self.before_setup - self.setup - self.after_setup - self.run_test self.__name__ - result = "." unless io? - time = Time.now - start_time - runner.record self.class, self.__name__, self._assertions, time, nil - @passed = true - rescue *PASSTHROUGH_EXCEPTIONS - raise - rescue Exception => e - @passed = Skip === e - time = Time.now - start_time - runner.record self.class, self.__name__, self._assertions, time, e - result = runner.puke self.class, self.__name__, e - ensure - %w{ before_teardown teardown after_teardown }.each do |hook| - begin - self.send hook - rescue *PASSTHROUGH_EXCEPTIONS - raise - rescue Exception => e - @passed = false - runner.record self.class, self.__name__, self._assertions, time, e - result = runner.puke self.class, self.__name__, e - end - end - trap 'INFO', 'DEFAULT' if runner.info_signal - end - result - end - - alias :run_test :__send__ - - def initialize name # :nodoc: - @__name__ = name - @__io__ = nil - @passed = nil - @@current = self # FIX: make thread local - end - - def self.current # :nodoc: - @@current # FIX: make thread local - end - - ## - # Return the output IO object - - def io - @__io__ = true - MiniTest::Unit.output - end - - ## - # Have we hooked up the IO yet? - - def io? - @__io__ - end - - def self.reset # :nodoc: - @@test_suites = {} - end - - reset - - ## - # Make diffs for this TestCase use #pretty_inspect so that diff - # in assert_equal can be more details. NOTE: this is much slower - # than the regular inspect but much more usable for complex - # objects. - - def self.make_my_diffs_pretty! - require 'pp' - - define_method :mu_pp do |o| - o.pretty_inspect - end - end - - def self.inherited klass # :nodoc: - @@test_suites[klass] = true - super - end - - def self.test_order # :nodoc: - :random - end - - def self.test_suites # :nodoc: - @@test_suites.keys.sort_by { |ts| ts.name.to_s } - end - - def self.test_methods # :nodoc: - methods = public_instance_methods(true).grep(/^test/).map { |m| m.to_s } - - case self.test_order - when :parallel - max = methods.size - ParallelEach.new methods.sort.sort_by { rand max } - when :random then - max = methods.size - methods.sort.sort_by { rand max } - when :alpha, :sorted then - methods.sort - else - raise "Unknown test_order: #{self.test_order.inspect}" - end - end - - ## - # Returns true if the test passed. - - def passed? - @passed - end - - ## - # Runs before every test. Use this to set up before each test - # run. - - def setup; end - - ## - # Runs after every test. Use this to clean up after each test - # run. - - def teardown; end - - include MiniTest::Assertions - end # class TestCase - end # class Unit - - Test = Unit::TestCase -end # module MiniTest - -Minitest = MiniTest # :nodoc: because ugh... I typo this all the time diff --git a/tool/lib/output.rb b/tool/lib/output.rb new file mode 100644 index 0000000000..8cb426ae4a --- /dev/null +++ b/tool/lib/output.rb @@ -0,0 +1,70 @@ +require_relative 'vpath' +require_relative 'colorize' + +class Output + attr_reader :path, :vpath + + def initialize(path: nil, timestamp: nil, ifchange: nil, color: nil, + overwrite: false, create_only: false, vpath: VPath.new) + @path = path + @timestamp = timestamp + @ifchange = ifchange + @color = color + @overwrite = overwrite + @create_only = create_only + @vpath = vpath + end + + COLOR_WHEN = { + 'always' => true, 'auto' => nil, 'never' => false, + nil => true, false => false, + } + + def def_options(opt) + opt.separator(" Output common options:") + opt.on('-o', '--output=PATH') {|v| @path = v} + opt.on('-t', '--timestamp[=PATH]') {|v| @timestamp = v || true} + opt.on('-c', '--[no-]if-change') {|v| @ifchange = v} + opt.on('--[no-]color=[WHEN]', COLOR_WHEN.keys) {|v| @color = COLOR_WHEN[v]} + opt.on('--[no-]create-only') {|v| @create_only = v} + opt.on('--[no-]overwrite') {|v| @overwrite = v} + @vpath.def_options(opt) + end + + def write(data, overwrite: @overwrite, create_only: @create_only) + unless @path + $stdout.print data + return true + end + color = Colorize.new(@color) + unchanged = color.pass("unchanged") + updated = color.fail("updated") + outpath = nil + + if (@ifchange or overwrite or create_only) and (@vpath.open(@path, "rb") {|f| + outpath = f.path + if @ifchange or create_only + original = f.read + (@ifchange and original == data) or (create_only and !original.empty?) + end + } rescue false) + puts "#{outpath} #{unchanged}" + written = false + else + unless overwrite and outpath and (File.binwrite(outpath, data) rescue nil) + File.binwrite(outpath = @path, data) + end + puts "#{outpath} #{updated}" + written = true + end + if timestamp = @timestamp + if timestamp == true + dir, base = File.split(@path) + timestamp = File.join(dir, ".time." + base) + end + File.binwrite(timestamp, '') + File.utime(nil, nil, timestamp) + end + written + end +end diff --git a/tool/lib/path.rb b/tool/lib/path.rb new file mode 100644 index 0000000000..f16a164338 --- /dev/null +++ b/tool/lib/path.rb @@ -0,0 +1,101 @@ +module Path + module_function + + def clean(path) + path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') + nil while path.sub!(/[^\/]+\/\.\.\//, '') + path + end + + def relative(path, base) + path = clean(path) + base = clean(base) + path, base = [path, base].map{|s|s.split("/")} + until path.empty? or base.empty? or path[0] != base[0] + path.shift + base.shift + end + path, base = [path, base].map{|s|s.join("/")} + if base.empty? + path + elsif base.start_with?("../") or File.absolute_path?(base) + File.expand_path(path) + else + base.gsub!(/[^\/]+/, '..') + File.join(base, path) + end + end + + def clean_link(src, dest) + begin + link = File.readlink(dest) + rescue + else + return if link == src + File.unlink(dest) + end + yield src, dest + end + + # Extensions to FileUtils + + module Mswin + def ln_safe(src, dest, real_src, *opt) + cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] + cmd[1, 0] = opt + return if system("cmd", "/c", *cmd) + # TODO: use RUNAS or something + puts cmd.join(" ") + end + + def ln_dir_safe(src, dest, real_src) + ln_safe(src, dest, "/d") + end + end + + module HardlinkExcutable + def ln_exe(relative_src, dest, src) + ln(src, dest, force: true) + end + end + + def ln_safe(src, dest, real_src) + ln_sf(src, dest) + rescue Errno::ENOENT + # Windows disallows to create broken symboic links, probably because + # it is a kind of reparse points. + raise if File.exist?(real_src) + end + + alias ln_dir_safe ln_safe + alias ln_exe ln_safe + + def ln_relative(src, dest, executable = false) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + if executable + return (ln_exe(relative(src, parent), dest, src) if File.exist?(src)) + end + clean_link(relative(src, parent), dest) {|s, d| ln_safe(s, d, src)} + end + + def ln_dir_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative(src, parent), dest) {|s, d| ln_dir_safe(s, d, src)} + end + + case (CROSS_COMPILING || RUBY_PLATFORM) + when /linux|darwin|solaris/ + prepend HardlinkExcutable + extend HardlinkExcutable + when /mingw|mswin/ + unless File.respond_to?(:symlink) + prepend Mswin + extend Mswin + end + else + end +end diff --git a/tool/lib/profile_test_all.rb b/tool/lib/profile_test_all.rb index 2c360d76dd..fb434e314d 100644 --- a/tool/lib/profile_test_all.rb +++ b/tool/lib/profile_test_all.rb @@ -20,7 +20,7 @@ require 'objspace' -class MiniTest::Unit::TestCase +class Test::Unit::TestCase alias orig_run run file = ENV['RUBY_TEST_ALL_PROFILE'] diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb new file mode 100644 index 0000000000..7b889163b0 --- /dev/null +++ b/tool/lib/test/jobserver.rb @@ -0,0 +1,47 @@ +module Test + module JobServer + end +end + +class << Test::JobServer + def connect(makeflags = ENV["MAKEFLAGS"]) + return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags + begin + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end + rescue + r&.close + nil + else + return r, w + end + end + + def acquire_possible(r, w, max) + return unless tokens = r.read_nonblock(max - 1, exception: false) + if (jobs = tokens.size) > 0 + jobserver, w = w, nil + at_exit do + jobserver.print(tokens) + jobserver.close + end + end + return jobs + 1 + rescue Errno::EBADF + ensure + r&.close + w&.close + end + + def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"]) + if max > 1 and (r, w = connect(makeflags)) + acquire_possible(r, w, max) + end + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index d237a9a0e9..2663b7b76a 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -1,26 +1,138 @@ # frozen_string_literal: true -begin - gem 'minitest', '< 5.0.0' if defined? Gem -rescue Gem::LoadError + +# Enable deprecation warnings for test-all, so deprecated methods/constants/functions are dealt with early. +Warning[:deprecated] = true + +if ENV['BACKTRACE_FOR_DEPRECATION_WARNINGS'] + Warning.extend Module.new { + def warn(message, category: nil, **kwargs) + if category == :deprecated and $stderr.respond_to?(:puts) + $stderr.puts nil, message, caller, nil + else + super + end + end + } end -require 'minitest/unit' -require 'test/unit/assertions' + require_relative '../envutil' require_relative '../colorize' -require 'test/unit/testcase' +require_relative '../leakchecker' +require_relative '../test/unit/testcase' +require_relative '../test/jobserver' require 'optparse' # See Test::Unit module Test + ## # Test::Unit is an implementation of the xUnit testing framework for Ruby. - # - # If you are writing new test code, please use MiniTest instead of Test::Unit. - # - # Test::Unit has been left in the standard library to support legacy test - # suites. module Unit - TEST_UNIT_IMPLEMENTATION = 'test/unit compatibility layer using minitest' # :nodoc: + ## + # Assertion base class + + class AssertionFailedError < Exception; end + + ## + # Assertion raised when skipping a test + + class PendedError < AssertionFailedError; end + + class << self + ## + # Extract the location where the last assertion method was + # called. Returns "<empty>" if _e_ does not have backtrace, or + # an empty string if no assertion method location was found. + + def location e + last_before_assertion = nil + + return '<empty>' unless e&.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /:in \W(?:.*\#)?(?:assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + return "" unless last_before_assertion + /:in / =~ last_before_assertion ? $` : last_before_assertion + end + end + + module Order + class NoSort + def initialize(seed) + end + + def sort_by_name(list) + list + end + + alias sort_by_string sort_by_name + + def group(list) + list + end + end + + class Alpha < NoSort + def sort_by_name(list) + list.sort_by(&:name) + end + + def sort_by_string(list) + list.sort + end + + end + + # shuffle test suites based on CRC32 of their names + Shuffle = Struct.new(:seed, :salt) do + def initialize(seed) + self.class::CRC_TBL ||= (0..255).map {|i| + (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } + }.freeze + + salt = [seed].pack("V").unpack1("H*") + super(seed, "\n#{salt}".freeze).freeze + end + + def sort_by_name(list) + list.sort_by {|e| randomize_key(e.name)} + end + + def sort_by_string(list) + list.sort_by {|e| randomize_key(e)} + end + + def group(list) + list + end + + private + + def crc32(str, crc32 = 0xffffffff) + crc_tbl = self.class::CRC_TBL + str.each_byte do |data| + crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) + end + crc32 + end + + def randomize_key(name) + crc32(salt, crc32(name)) ^ 0xffffffff + end + end + + Types = { + random: Shuffle, + alpha: Alpha, + sorted: Alpha, + nosort: NoSort, + } + Types.default_proc = proc {|_, order| + raise "Unknown test_order: #{order.inspect}" + } + end module RunCount # :nodoc: all @@run_count = 0 @@ -64,14 +176,28 @@ module Test args = @init_hook.call(args, options) if @init_hook non_options(args, options) @run_options = orig_args - @help = orig_args.map { |s| s =~ /[\s|&<>$()]/ ? s.inspect : s }.join " " + + order = options[:test_order] + if seed = options[:seed] + order ||= :random + elsif (order ||= :random) == :random + seed = options[:seed] = rand(0x10000) + orig_args.unshift "--seed=#{seed}" + end + Test::Unit::TestCase.test_order = order if order + order = Test::Unit::TestCase.test_order + @order = Test::Unit::Order::Types[order].new(seed) + + @help = "\n" + orig_args.map { |s| + " " + (s =~ /[\s|&<>$()]/ ? s.inspect : s) + }.join("\n") + @options = options end private def setup_options(opts, options) - opts.separator 'minitest options:' - opts.version = MiniTest::Unit::VERSION + opts.separator 'test-unit options:' opts.on '-h', '--help', 'Display this help.' do puts opts @@ -79,7 +205,7 @@ module Test end opts.on '-s', '--seed SEED', Integer, "Sets random seed" do |m| - options[:seed] = m + options[:seed] = m.to_i end opts.on '-v', '--verbose', "Verbose. Show progress processing files." do @@ -91,8 +217,9 @@ module Test (options[:filter] ||= []) << a end - opts.on '--test-order=random|alpha|sorted', [:random, :alpha, :sorted] do |a| - MiniTest::Unit::TestCase.test_order = a + orders = Test::Unit::Order::Types.keys + opts.on "--test-order=#{orders.join('|')}", orders do |a| + options[:test_order] = a end end @@ -106,6 +233,9 @@ module Test filter = nil elsif negative.empty? and positive.size == 1 and pos_pat !~ positive[0] filter = positive[0] + unless /\A[A-Z]\w*(?:::[A-Z]\w*)*#/ =~ filter + filter = /##{Regexp.quote(filter)}\z/ + end else filter = Regexp.union(*positive.map! {|s| Regexp.new(s[pos_pat, 1] || "\\A#{Regexp.quote(s)}\\z")}) end @@ -113,12 +243,6 @@ module Test negative = Regexp.union(*negative.map! {|s| Regexp.new(s[neg_pat, 1])}) filter = /\A(?=.*#{filter})(?!.*#{negative})/ end - if Regexp === filter - # bypass conversion in minitest - def filter.=~(other) # :nodoc: - super unless Regexp === other - end - end options[:filter] = filter end true @@ -126,6 +250,8 @@ module Test end module Parallel # :nodoc: all + attr_accessor :prefix + def process_args(args = []) return @options if @options options = super @@ -137,22 +263,10 @@ module Test def non_options(files, options) @jobserver = nil - makeflags = ENV.delete("MAKEFLAGS") - if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(\d+),(\d+)/ =~ makeflags - begin - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) - rescue - r.close if r - nil - else - r.close_on_exec = true - w.close_on_exec = true - @jobserver = [r, w] - options[:parallel] ||= 1 - end + if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS")) + options[:parallel] ||= 256 # number of tokens to acquire first end + @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) super end @@ -168,13 +282,17 @@ module Test opts.separator "parallel test options:" - options[:retry] = true + options[:retry] = false opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| options[:testing] = true & t # For testing options[:parallel] = a.to_i end + opts.on '--worker-timeout=N', Integer, "Timeout workers not responding in N seconds" do |a| + options[:worker_timeout] = a + end + opts.on '--separate', "Restart job process after one testcase has done" do options[:parallel] ||= 1 options[:separate] = true @@ -188,24 +306,35 @@ module Test options[:retry] = false end - opts.on '--ruby VAL', "Path to ruby which is used at -j option" do |a| + opts.on '--ruby VAL', "Path to ruby which is used at -j option", + "Also used as EnvUtil.rubybin by some assertion methods" do |a| options[:ruby] = a.split(/ /).reject(&:empty?) end + + opts.on '--timetable-data=FILE', "Path to timetable data" do |a| + options[:timetable_data] = a + end end class Worker def self.launch(ruby,args=[]) scale = EnvUtil.timeout_scale io = IO.popen([*ruby, "-W1", - "#{File.dirname(__FILE__)}/unit/parallel.rb", + "#{__dir__}/unit/parallel.rb", *("--timeout-scale=#{scale}" if scale), *args], "rb+") new(io, io.pid, :waiting) end attr_reader :quit_called + attr_accessor :start_time + attr_accessor :response_at + attr_accessor :current + + @@worker_number = 0 def initialize(io, pid, status) + @num = (@@worker_number += 1) @io = io @pid = pid @status = status @@ -214,20 +343,31 @@ module Test @loadpath = [] @hooks = {} @quit_called = false + @response_at = nil + end + + def name + "Worker #{@num}" end def puts(*args) @io.puts(*args) end - def run(task,type) - @file = File.basename(task, ".rb") + def run(task, type, base = nil) + if base + @file = task.delete_prefix(base).chomp(".rb") + else + @file = File.basename(task, ".rb") + end @real_file = task begin puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" @loadpath = $:.dup puts "run #{task} #{type}" @status = :prepare + @start_time = Time.now + @response_at = @start_time rescue Errno::EPIPE died rescue IOError @@ -244,6 +384,7 @@ module Test def read res = (@status == :quit) ? @io.read : @io.gets + @response_at = Time.now res && res.chomp end @@ -253,16 +394,19 @@ module Test rescue IOError end - def quit + def quit(reason = :normal) return if @io.closed? @quit_called = true - @io.puts "quit" + @io.puts "quit #{reason}" rescue Errno::EPIPE => e warn "#{@pid}:#{@status.to_s.ljust(7)}:#{@file}: #{e.message}" end def kill - Process.kill(:KILL, @pid) + EnvUtil::Debugger.search&.dump(@pid) + signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV + Process.kill(signal, @pid) + warn "worker #{to_s} does not respond; #{signal} is sent" rescue Errno::ESRCH end @@ -316,10 +460,26 @@ module Test real_file = worker.real_file and warn "running file: #{real_file}" @need_quit = true warn "" - warn "Some worker was crashed. It seems ruby interpreter's bug" - warn "or, a bug of test/unit/parallel.rb. try again without -j" + warn "A test worker crashed. It might be an interpreter bug or" + warn "a bug in test/unit/parallel.rb. Try again without the -j" warn "option." warn "" + if File.exist?('core') + require 'fileutils' + require 'time' + Dir.glob('/tmp/test-unit-core.*').each do |f| + if Time.now - File.mtime(f) > 7 * 24 * 60 * 60 # 7 days + warn "Deleting an old core file: #{f}" + FileUtils.rm(f) + end + end + core_path = "/tmp/test-unit-core.#{Time.now.utc.iso8601}" + warn "A core file is found. Saving it at: #{core_path.dump}" + FileUtils.mv('core', core_path) + cmd = ['gdb', RbConfig.ruby, '-c', core_path, '-ex', 'bt', '-batch'] + p cmd # debugging why it's not working + system(*cmd) + end STDERR.flush exit c end @@ -358,30 +518,45 @@ module Test @ios.delete worker.io end - def quit_workers + def quit_workers(&cond) return if @workers.empty? + closed = [] if cond @workers.reject! do |worker| + next unless cond&.call(worker) begin - Timeout.timeout(1) do - worker.quit + Timeout.timeout(5) do + worker.quit(cond ? :timeout : :normal) end rescue Errno::EPIPE rescue Timeout::Error end - worker.close + closed&.push worker + begin + Timeout.timeout(1) do + worker.close + end + rescue Timeout::Error + worker.kill + retry + end + @ios.delete worker.io end - return if @workers.empty? + return if (closed ||= @workers).empty? + pids = closed.map(&:pid) begin - Timeout.timeout(0.2 * @workers.size) do + Timeout.timeout(1 * closed.size) do Process.waitall end rescue Timeout::Error - @workers.each do |worker| - worker.kill + if pids + Process.kill(:KILL, *pids) rescue nil + pids = nil + retry end - @worker.clear end + @workers.clear unless cond + closed end FakeClass = Struct.new(:name) @@ -393,6 +568,7 @@ module Test worker = @workers_hash[io] cmd = worker.read cmd.sub!(/\A\.+/, '') if cmd # read may return nil + case cmd when '' # just only dots, ignore @@ -410,38 +586,49 @@ module Test worker.quit worker = launch_worker end - worker.run(task, type) + worker.run(task, type, (@prefix unless @options[:job_status] == :replace)) @test_count += 1 jobs_status(worker) + when /^start (.+?)$/ + worker.current = Marshal.load($1.unpack1("m")) when /^done (.+?)$/ begin - r = Marshal.load($1.unpack("m")[0]) + r = Marshal.load($1.unpack1("m")) rescue - print "unknown object: #{$1.unpack("m")[0].dump}" + print "unknown object: #{$1.unpack1("m").dump}" return true end result << r[0..1] unless r[0..1] == [nil,nil] rep << {file: worker.real_file, report: r[2], result: r[3], testcase: r[5]} $:.push(*r[4]).uniq! jobs_status(worker) if @options[:job_status] == :replace + return true when /^record (.+?)$/ begin - r = Marshal.load($1.unpack("m")[0]) + r = Marshal.load($1.unpack1("m")) + + suite = r.first + key = [worker.name, suite] + if @records[key] + @records[key][1] = worker.start_time = Time.now + else + @records[key] = [worker.start_time, Time.now] + end rescue => e - print "unknown record: #{e.message} #{$1.unpack("m")[0].dump}" + print "unknown record: #{e.message} #{$1.unpack1("m").dump}" return true end record(fake_class(r[0]), *r[1..-1]) when /^p (.+?)$/ del_jobs_status - print $1.unpack("m")[0] + print $1.unpack1("m") jobs_status(worker) if @options[:job_status] == :replace when /^after (.+?)$/ - @warnings << Marshal.load($1.unpack("m")[0]) + @warnings << Marshal.load($1.unpack1("m")) when /^bye (.+?)$/ - after_worker_down worker, Marshal.load($1.unpack("m")[0]) + after_worker_down worker, Marshal.load($1.unpack1("m")) when /^bye$/, nil if shutting_down || worker.quit_called after_worker_quit worker @@ -455,6 +642,8 @@ module Test end def _run_parallel suites, type, result + @records = {} + if @options[:parallel] < 1 warn "Error: parameter of -j option should be greater than 0." return @@ -462,7 +651,8 @@ module Test # Require needed thing for parallel running require 'timeout' - @tasks = @files.dup # Array of filenames. + @tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames. + @need_quit = false @dead_workers = [] # Array of dead workers. @warnings = [] @@ -474,26 +664,48 @@ module Test @ios = [] # Array of worker IOs @job_tokens = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver begin - [@tasks.size, @options[:parallel]].min.times {launch_worker} + while true + newjobs = [@tasks.size, @options[:parallel]].min - @workers.size + if newjobs > 0 + if @jobserver + t = @jobserver[0].read_nonblock(newjobs, exception: false) + @job_tokens << t if String === t + newjobs = @job_tokens.size + 1 - @workers.size + end + newjobs.times {launch_worker} + end + + timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0), 0].max + @worker_timeout - while _io = IO.select(@ios)[0] - break if _io.any? do |io| + if !(_io = IO.select(@ios, nil, nil, timeout)) + timeout = Time.now - @worker_timeout + quit_workers {|w| w.response_at&.<(timeout) }&.map {|w| + rep << {file: w.real_file, result: nil, testcase: w.current[0], error: w.current} + } + elsif _io.first.any? {|io| @need_quit or (deal(io, type, result, rep).nil? and !@workers.any? {|x| [:running, :prepare].include? x.status}) + } + break end - if @jobserver and @job_tokens and !@tasks.empty? and !@workers.any? {|x| x.status == :ready} - t = @jobserver[0].read_nonblock([@tasks.size, @options[:parallel]].min, exception: false) - if String === t - @job_tokens << t - t.size.times {launch_worker} - end + if @tasks.empty? + break if @workers.empty? + next # wait for all workers to finish end end rescue Interrupt => ex @interrupt = ex return result ensure + if file = @options[:timetable_data] + File.open(file, 'w'){|f| + @records.each{|(worker, suite), (st, ed)| + f.puts '[' + [worker.dump, suite.dump, st.to_f * 1_000, ed.to_f * 1_000].join(", ") + '],' + } + } + end + if @interrupt @ios.select!{|x| @workers_hash[x].status == :running } while !@ios.empty? && (__io = IO.select(@ios,[],[],10)) @@ -507,14 +719,50 @@ module Test unless @interrupt || !@options[:retry] || @need_quit parallel = @options[:parallel] @options[:parallel] = false - suites, rep = rep.partition {|r| r[:testcase] && r[:file] && r[:report].any? {|e| !e[2].is_a?(MiniTest::Skip)}} - suites.map {|r| r[:file]}.uniq.each {|file| require file} - suites.map! {|r| eval("::"+r[:testcase])} + suites, rep = rep.partition {|r| + r[:testcase] && r[:file] && + (!r.key?(:report) || r[:report].any? {|e| !e[2].is_a?(Test::Unit::PendedError)}) + } + suites.map {|r| File.realpath(r[:file])}.uniq.each {|file| require file} del_status_line or puts + error, suites = suites.partition {|r| r[:error]} unless suites.empty? - puts "\n""Retrying..." + puts "\n" + @failed_output.puts "Failed tests:" + suites.each {|r| + r[:report].each {|c, m, e| + @failed_output.puts "#{c}##{m}: #{e&.class}: #{e&.message&.slice(/\A.*/)}" + } + } + @failed_output.puts "\n" + puts "Retrying..." + @verbose = options[:verbose] + suites.map! {|r| ::Object.const_get(r[:testcase])} _run_suites(suites, type) end + unless error.empty? + puts "\n""Retrying hung up testcases..." + error = error.map do |r| + begin + ::Object.const_get(r[:testcase]) + rescue NameError + # testcase doesn't specify the correct case, so show `r` for information + require 'pp' + + $stderr.puts "Retrying is failed because the file and testcase is not consistent:" + PP.pp r, $stderr + @errors += 1 + nil + end + end.compact + verbose = @verbose + job_status = options[:job_status] + options[:verbose] = @verbose = true + options[:job_status] = :normal + result.concat _run_suites(error, type) + options[:verbose] = @verbose = verbose + options[:job_status] = job_status + end @options[:parallel] = parallel end unless @options[:retry] @@ -522,20 +770,28 @@ module Test end unless rep.empty? rep.each do |r| - r[:report].each do |f| + if r[:error] + puke(*r[:error], Timeout::Error.new) + next + end + r[:report]&.each do |f| puke(*f) if f end end if @options[:retry] - @errors += rep.map{|x| x[:result][0] }.inject(:+) - @failures += rep.map{|x| x[:result][1] }.inject(:+) - @skips += rep.map{|x| x[:result][2] }.inject(:+) + rep.each do |x| + (e, f, s = x[:result]) or next + @errors += e + @failures += f + @skips += s + end end end unless @warnings.empty? warn "" @warnings.uniq! {|w| w[1].message} @warnings.each do |w| + @errors += 1 warn "#{w[0]}: #{w[1].message} (#{w[1].class})" end warn "" @@ -605,7 +861,7 @@ module Test end end - def record(suite, method, assertions, time, error) + def record(suite, method, assertions, time, error, source_location = nil) if @options.values_at(:longest, :most_asserted).any? @tops ||= {} rec = [suite.name, method, assertions, time, error] @@ -684,7 +940,7 @@ module Test end def jobs_status(worker) - return if !@options[:job_status] or @options[:verbose] + return if !@options[:job_status] or @verbose if @options[:job_status] == :replace status_line = @workers.map(&:to_s).join(" ") else @@ -703,7 +959,7 @@ module Test end def _prepare_run(suites, type) - options[:job_status] ||= :replace if @tty && !@verbose + options[:job_status] ||= @tty ? :replace : :normal unless @verbose case options[:color] when :always color = true @@ -719,11 +975,14 @@ module Test @output = Output.new(self) unless @options[:testing] filter = options[:filter] type = "#{type}_methods" - total = if filter - suites.inject(0) {|n, suite| n + suite.send(type).grep(filter).size} - else - suites.inject(0) {|n, suite| n + suite.send(type).size} - end + total = suites.sum {|suite| + methods = suite.send(type) + if filter + methods.count {|method| filter === "#{suite}##{method}"} + else + methods.size + end + } @test_count = 0 @total_tests = total.to_s(10) end @@ -761,7 +1020,7 @@ module Test end first, msg = msg.split(/$/, 2) first = sprintf("%3d) %s", @report_count += 1, first) - $stdout.print(sep, @colorize.decorate(first, color), msg, "\n") + @failed_output.print(sep, @colorize.decorate(first, color), msg, "\n") sep = nil end report.clear @@ -817,7 +1076,7 @@ module Test runner.add_status(" = #$1") when /\A\.+\z/ runner.succeed - when /\A[EFS]\z/ + when /\A\.*[EFST][EFST.]*\z/ runner.failed(s) else $stdout.print(s) @@ -856,7 +1115,8 @@ module Test def setup_options(parser, options) super parser.separator "globbing options:" - parser.on '-b', '--basedir=DIR', 'Base directory of test suites.' do |dir| + parser.on '-B', '--base-directory DIR', 'Base directory to glob.' do |dir| + raise OptionParser::InvalidArgument, "not a directory: #{dir}" unless File.directory?(dir) options[:base_directory] = dir end parser.on '-x', '--exclude REGEXP', 'Exclude test files on pattern.' do |pattern| @@ -864,6 +1124,18 @@ module Test end end + def complement_test_name f, orig_f + basename = File.basename(f) + + if /\.rb\z/ !~ basename + return File.join(File.dirname(f), basename+'.rb') + elsif /\Atest_/ !~ basename + return File.join(File.dirname(f), 'test_'+basename) + end if f.end_with?(basename) # otherwise basename is dirname/ + + raise ArgumentError, "file not found: #{orig_f}" + end + def non_options(files, options) paths = [options.delete(:base_directory), nil].uniq if reject = options.delete(:reject) @@ -871,6 +1143,7 @@ module Test end files.map! {|f| f = f.tr(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR + orig_f = f while true ret = ((paths if /\A\.\.?(?:\z|\/)/ !~ f) || [nil]).any? do |prefix| if prefix @@ -897,11 +1170,7 @@ module Test end end if !ret - if /\.rb\z/ =~ f - raise ArgumentError, "file not found: #{f}" - else - f = "#{f}.rb" - end + f = complement_test_name(f, orig_f) else break ret end @@ -912,29 +1181,66 @@ module Test end end - module GCStressOption # :nodoc: all + module OutputOption # :nodoc: all + def setup_options(parser, options) + super + parser.separator "output options:" + + options[:failed_output] = $stdout + parser.on '--stderr-on-failure', 'Use stderr to print failure messages' do + options[:failed_output] = $stderr + end + parser.on '--stdout-on-failure', 'Use stdout to print failure messages', '(default)' do + options[:failed_output] = $stdout + end + end + + def process_args(args = []) + return @options if @options + options = super + @failed_output = options[:failed_output] + options + end + end + + module GCOption # :nodoc: all def setup_options(parser, options) super parser.separator "GC options:" parser.on '--[no-]gc-stress', 'Set GC.stress as true' do |flag| options[:gc_stress] = flag end + parser.on '--[no-]gc-compact', 'GC.compact every time' do |flag| + options[:gc_compact] = flag + end end def non_options(files, options) if options.delete(:gc_stress) - MiniTest::Unit::TestCase.class_eval do + Test::Unit::TestCase.class_eval do oldrun = instance_method(:run) define_method(:run) do |runner| begin gc_stress, GC.stress = GC.stress, true - oldrun.bind(self).call(runner) + oldrun.bind_call(self, runner) ensure GC.stress = gc_stress end end end end + if options.delete(:gc_compact) + Test::Unit::TestCase.class_eval do + oldrun = instance_method(:run) + define_method(:run) do |runner| + begin + oldrun.bind_call(self, runner) + ensure + GC.compact + end + end + end + end super end end @@ -958,8 +1264,13 @@ module Test puts "#{f}: #{$!}" end } + @load_failed = errors.size.nonzero? result end + + def run(*) + super or @load_failed + end end module RepeatOption # :nodoc: all @@ -970,10 +1281,15 @@ module Test parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| options[:repeat_count] = n end + options[:keep_repeating] = false + parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n| + options[:keep_repeating] = true + end end def _run_anything(type) @repeat_count = @options[:repeat_count] + @keep_repeating = @options[:keep_repeating] super end end @@ -1061,21 +1377,425 @@ module Test end end - class Runner < MiniTest::Unit # :nodoc: all - include Test::Unit::Options - include Test::Unit::StatusLine - include Test::Unit::Parallel - include Test::Unit::Statistics - include Test::Unit::Skipping - include Test::Unit::GlobOption - include Test::Unit::RepeatOption - include Test::Unit::LoadPathOption - include Test::Unit::GCStressOption - include Test::Unit::ExcludesOption - include Test::Unit::TimeoutOption - include Test::Unit::RunCount - - class << self; undef autorun; end + module LaunchableOption + module Nothing + private + def setup_options(opts, options) + super + opts.define_tail 'Launchable options:' + # This is expected to be called by Test::Unit::Worker. + opts.on_tail '--launchable-test-reports=PATH', String, 'Do nothing' + end + end + + def record(suite, method, assertions, time, error, source_location = nil) + if writer = @options[:launchable_test_reports] + if loc = (source_location || suite.instance_method(method).source_location) + path, lineno = loc + # Launchable JSON schema is defined at + # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code. + e = case error + when nil + status = 'TEST_PASSED' + nil + when Test::Unit::PendedError + status = 'TEST_SKIPPED' + "Skipped:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Test::Unit::AssertionFailedError + status = 'TEST_FAILED' + "Failure:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Timeout::Error + status = 'TEST_FAILED' + "Timeout:\n#{suite.name}##{method}\n" + else + status = 'TEST_FAILED' + bt = Test::filter_backtrace(error.backtrace).join "\n " + "Error:\n#{suite.name}##{method}:\n#{error.class}: #{error.message.b}\n #{bt}\n" + end + repo_path = File.expand_path("#{__dir__}/../../../") + relative_path = path.delete_prefix("#{repo_path}/") + # The test path is a URL-encoded representation. + # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18 + test_path = {file: relative_path, class: suite.name, testcase: method}.map{|key, val| + "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}" + }.join('#') + end + end + super + ensure + if writer && test_path && status + # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified. + # In such cases, we proceed to execute the operation here. + writer.write_object( + { + testPath: test_path, + status: status, + duration: time, + createdAt: Time.now.to_s, + stderr: e, + stdout: nil, + data: { + lineNumber: lineno + } + } + ) + end + end + + private + def setup_options(opts, options) + super + opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path| + require_relative '../launchable' + options[:launchable_test_reports] = writer = Launchable::JsonStreamWriter.new(path) + writer.write_array('testCases') + main_pid = Process.pid + at_exit { + # This block is executed when the fork block in a test is completed. + # Therefore, we need to verify whether all tests have been completed. + stack = caller + if stack.size == 0 && main_pid == Process.pid && $!.is_a?(SystemExit) + writer.close + end + } + end + + def encode_test_path_component component + component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26') + end + end + end + + class Runner # :nodoc: all + + attr_accessor :report, :failures, :errors, :skips # :nodoc: + attr_accessor :assertion_count # :nodoc: + attr_writer :test_count # :nodoc: + attr_accessor :start_time # :nodoc: + attr_accessor :help # :nodoc: + attr_accessor :verbose # :nodoc: + attr_writer :options # :nodoc: + + ## + # :attr: + # + # if true, installs an "INFO" signal handler (only available to BSD and + # OS X users) which prints diagnostic information about the test run. + # + # This is auto-detected by default but may be overridden by custom + # runners. + + attr_accessor :info_signal + + ## + # Lazy accessor for options. + + def options + @options ||= {seed: 42} + end + + @@installed_at_exit ||= false + @@out = $stdout + @@after_tests = [] + @@current_repeat_count = 0 + + ## + # A simple hook allowing you to run a block of code after _all_ of + # the tests are done. Eg: + # + # Test::Unit::Runner.after_tests { p $debugging_info } + + def self.after_tests &block + @@after_tests << block + end + + ## + # Returns the stream to use for output. + + def self.output + @@out + end + + ## + # Sets Test::Unit::Runner to write output to +stream+. $stdout is the default + # output + + def self.output= stream + @@out = stream + end + + ## + # Tells Test::Unit::Runner to delegate to +runner+, an instance of a + # Test::Unit::Runner subclass, when Test::Unit::Runner#run is called. + + def self.runner= runner + @@runner = runner + end + + ## + # Returns the Test::Unit::Runner subclass instance that will be used + # to run the tests. A Test::Unit::Runner instance is the default + # runner. + + def self.runner + @@runner ||= self.new + end + + ## + # Return all plugins' run methods (methods that start with "run_"). + + def self.plugins + @@plugins ||= (["run_tests"] + + public_instance_methods(false). + grep(/^run_/).map { |s| s.to_s }).uniq + end + + ## + # Return the IO for output. + + def output + self.class.output + end + + def puts *a # :nodoc: + output.puts(*a) + end + + def print *a # :nodoc: + output.print(*a) + end + + def test_count # :nodoc: + @test_count ||= 0 + end + + ## + # Runner for a given +type+ (eg, test vs bench). + + def self.current_repeat_count + @@current_repeat_count + end + + def _run_anything type + suites = Test::Unit::TestCase.send "#{type}_suites" + return if suites.empty? + + suites = @order.sort_by_name(suites) + + puts + puts "# Running #{type}s:" + puts + + @test_count, @assertion_count = 0, 0 + test_count = assertion_count = 0 + sync = output.respond_to? :"sync=" # stupid emacs + old_sync, output.sync = output.sync, true if sync + + @@current_repeat_count = 0 + begin + start = Time.now + + results = _run_suites suites, type + + @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } + @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } + test_count += @test_count + assertion_count += @assertion_count + t = Time.now - start + @@current_repeat_count += 1 + unless @repeat_count + puts + puts + end + puts "Finished%s %ss in %.6fs, %.4f tests/s, %.4f assertions/s.\n" % + [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, + t, @test_count.fdiv(t), @assertion_count.fdiv(t)] + end while @repeat_count && @@current_repeat_count < @repeat_count && + (@keep_repeating || report.empty? && failures.zero? && errors.zero?) + + output.sync = old_sync if sync + + report.each_with_index do |msg, i| + puts "\n%3d) %s" % [i + 1, msg] + end + + puts + @test_count = test_count + @assertion_count = assertion_count + + status + end + + ## + # Run a single +suite+ for a given +type+. + + def _run_suite suite, type + header = "#{type}_suite_header" + puts send(header, suite) if respond_to? header + + filter = options[:filter] + + all_test_methods = suite.send "#{type}_methods" + if filter + all_test_methods.select! {|method| + filter === "#{suite}##{method}" + } + end + all_test_methods = @order.sort_by_name(all_test_methods) + + leakchecker = LeakChecker.new + if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] + require "objspace" + trace = true + end + + assertions = all_test_methods.map { |method| + + inst = suite.new method + _start_method(inst) + inst._assertions = 0 + + print "#{suite}##{method.inspect.sub(/\A:/, '')} = " if @verbose + + start_time = Time.now if @verbose + result = + if trace + ObjectSpace.trace_object_allocations {inst.run self} + else + inst.run self + end + + print "%.2f s = " % (Time.now - start_time) if @verbose + print result + puts if @verbose + $stdout.flush + + leakchecker.check("#{inst.class}\##{inst.__name__}") + + _end_method(inst) + + inst._assertions + } + return assertions.size, assertions.inject(0) { |sum, n| sum + n } + end + + def _start_method(inst) + end + def _end_method(inst) + end + + ## + # Record the result of a single test. Makes it very easy to gather + # information. Eg: + # + # class StatisticsRecorder < Test::Unit::Runner + # def record suite, method, assertions, time, error + # # ... record the results somewhere ... + # end + # end + # + # Test::Unit::Runner.runner = StatisticsRecorder.new + # + # NOTE: record might be sent more than once per test. It will be + # sent once with the results from the test itself. If there is a + # failure or error in teardown, it will be sent again with the + # error or failure. + + def record suite, method, assertions, time, error, source_location = nil + end + + def location e # :nodoc: + Test::Unit.location e + end + + ## + # Writes status for failed test +meth+ in +klass+ which finished with + # exception +e+ + + def initialize # :nodoc: + @report = [] + @errors = @failures = @skips = 0 + @verbose = false + @mutex = Thread::Mutex.new + @info_signal = Signal.list['INFO'] + @repeat_count = nil + end + + def synchronize # :nodoc: + if @mutex then + @mutex.synchronize { yield } + else + yield + end + end + + def inspect + "#<#{self.class.name}: " << + instance_variables.filter_map do |var| + next if var == :@option_parser # too big + "#{var}=#{instance_variable_get(var).inspect}" + end.join(", ") << ">" + end + + ## + # Top level driver, controls all output and filtering. + + def _run args = [] + args = process_args args # ARGH!! blame test/unit process_args + self.options.merge! args + + puts "Run options: #{help}" + + self.class.plugins.each do |plugin| + send plugin + break unless report.empty? + end + + return (failures + errors).nonzero? # or return nil... + rescue Interrupt + abort 'Interrupted' + end + + ## + # Runs test suites matching +filter+. + + def run_tests + _run_anything :test + end + + ## + # Writes status to +io+ + + def status io = self.output + format = "%d tests, %d assertions, %d failures, %d errors, %d skips" + io.puts format % [test_count, assertion_count, failures, errors, skips] + end + + prepend Test::Unit::Options + prepend Test::Unit::StatusLine + prepend Test::Unit::Parallel + prepend Test::Unit::Statistics + prepend Test::Unit::Skipping + prepend Test::Unit::GlobOption + prepend Test::Unit::OutputOption + prepend Test::Unit::RepeatOption + prepend Test::Unit::LoadPathOption + prepend Test::Unit::GCOption + prepend Test::Unit::ExcludesOption + prepend Test::Unit::TimeoutOption + prepend Test::Unit::RunCount + prepend Test::Unit::LaunchableOption::Nothing + + ## + # Begins the full test run. Delegates to +runner+'s #_run method. + + def run(argv = []) + self.class.runner._run(argv) + rescue NoMemoryError + system("cat /proc/meminfo") if File.exist?("/proc/meminfo") + system("ps x -opid,args,%cpu,%mem,nlwp,rss,vsz,wchan,stat,start,time,etime,blocked,caught,ignored,pending,f") if File.exist?("/bin/ps") + raise + end @@stop_auto_run = false def self.autorun @@ -1087,16 +1807,30 @@ module Test @@installed_at_exit = true end - alias mini_run_suite _run_suite + alias orig_run_suite _run_suite - # Overriding of MiniTest::Unit#puke + # Overriding of Test::Unit::Runner#puke def puke klass, meth, e - # TODO: - # this overriding is for minitest feature that skip messages are - # hidden when not verbose (-v), note this is temporally. n = report.size - rep = super - if MiniTest::Skip === e and /no message given\z/ =~ e.message + e = case e + when Test::Unit::PendedError then + @skips += 1 + return "S" unless @verbose + "Skipped:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Test::Unit::AssertionFailedError then + @failures += 1 + "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Timeout::Error + @errors += 1 + "Timeout:\n#{klass}##{meth}\n" + else + @errors += 1 + bt = Test::filter_backtrace(e.backtrace).join "\n " + "Error:\n#{klass}##{meth}:\n#{e.class}: #{e.message.b}\n #{bt}\n" + end + @report << e + rep = e[0, 1] + if Test::Unit::PendedError === e and /no message given\z/ =~ e.message report.slice!(n..-1) rep = "." end @@ -1107,6 +1841,7 @@ module Test class AutoRunner # :nodoc: all class Runner < Test::Unit::Runner include Test::Unit::RequireFiles + include Test::Unit::LaunchableOption end attr_accessor :to_run, :options @@ -1114,10 +1849,12 @@ module Test def initialize(force_standalone = false, default_dir = nil, argv = ARGV) @force_standalone = force_standalone @runner = Runner.new do |files, options| - options[:base_directory] ||= default_dir + base = options[:base_directory] ||= default_dir + @runner.prefix = base ? (base + "/") : nil files << default_dir if files.empty? and default_dir @to_run = files yield self if block_given? + $LOAD_PATH.unshift base if base files end Runner.runner = @runner @@ -1156,30 +1893,4 @@ module Test end end -module MiniTest # :nodoc: all - class Unit - end -end - -class MiniTest::Unit::TestCase # :nodoc: all - test_order = self.test_order - class << self - attr_writer :test_order - undef test_order - end - def self.test_order - defined?(@test_order) ? @test_order : superclass.test_order - end - self.test_order = test_order - undef run_test - RUN_TEST_TRACE = "#{__FILE__}:#{__LINE__+3}:in `run_test'".freeze - def run_test(name) - progname, $0 = $0, "#{$0}: #{self.class}##{name}" - self.__send__(name) - ensure - $@.delete(RUN_TEST_TRACE) if $@ - $0 = progname - end -end - Test::Unit::Runner.autorun diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index 0c68a93b02..19581fc3ab 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -1,196 +1,566 @@ # frozen_string_literal: true -require 'minitest/unit' -require 'test/unit/core_assertions' require 'pp' module Test module Unit module Assertions - include Test::Unit::CoreAssertions - MINI_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), "minitest") #:nodoc: + ## + # Returns the diff command to use in #diff. Tries to intelligently + # figure out what diff to use. - # :call-seq: - # assert(test, [failure_message]) + def self.diff + unless defined? @diff + exe = RbConfig::CONFIG['EXEEXT'] + @diff = %W"gdiff#{exe} diff#{exe}".find do |diff| + if system(diff, "-u", __FILE__, __FILE__) + break "#{diff} -u" + end + end + end + + @diff + end + + ## + # Set the diff command to use in #diff. + + def self.diff= o + @diff = o + end + + ## + # Returns a diff between +exp+ and +act+. If there is no known + # diff command or if it doesn't make sense to diff the output + # (single line, short output), then it simply returns a basic + # comparison between the two. + + def diff exp, act + require "tempfile" + + expect = mu_pp_for_diff exp + butwas = mu_pp_for_diff act + result = nil + + need_to_diff = + self.class.diff && + (expect.include?("\n") || + butwas.include?("\n") || + expect.size > 30 || + butwas.size > 30 || + expect == butwas) + + return "Expected: #{mu_pp exp}\n Actual: #{mu_pp act}" unless + need_to_diff + + tempfile_a = nil + tempfile_b = nil + + Tempfile.open("expect") do |a| + tempfile_a = a + a.puts expect + a.flush + + Tempfile.open("butwas") do |b| + tempfile_b = b + b.puts butwas + b.flush + + result = `#{self.class.diff} #{a.path} #{b.path}` + result.sub!(/^\-\-\- .+/, "--- expected") + result.sub!(/^\+\+\+ .+/, "+++ actual") + + if result.empty? then + klass = exp.class + result = [ + "No visible difference in the #{klass}#inspect output.\n", + "You should look at the implementation of #== on ", + "#{klass} or its members.\n", + expect, + ].join + end + end + end + + result + ensure + tempfile_a.close! if tempfile_a + tempfile_b.close! if tempfile_b + end + + ## + # This returns a diff-able human-readable version of +obj+. This + # differs from the regular mu_pp because it expands escaped + # newlines and makes hex-values generic (like object_ids). This + # uses mu_pp to do the first pass and then cleans it up. + + def mu_pp_for_diff obj + mu_pp(obj).gsub(/(?<!\\)(?:\\\\)*\K\\n/, "\n").gsub(/:0x[a-fA-F0-9]{4,}/m, ':0xXXXXXX') + end + + ## + # Fails unless +test+ is a true value. + + def assert test, msg = nil + msg ||= "Failed assertion, no message given." + self._assertions += 1 + unless test then + msg = msg.call if Proc === msg + raise Test::Unit::AssertionFailedError, msg + end + true + end + + ## + # Fails unless +obj+ is empty. + + def assert_empty obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be empty" } + assert_respond_to obj, :empty? + assert obj.empty?, msg + end + + ## + # For comparing Floats. Fails unless +exp+ and +act+ are within +delta+ + # of each other. # - #Tests if +test+ is true. + # assert_in_delta Math::PI, (22.0 / 7.0), 0.01 + + def assert_in_delta exp, act, delta = 0.001, msg = nil + n = (exp - act).abs + msg = message(msg) { + "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}" + } + assert delta >= n, msg + end + + ## + # For comparing Floats. Fails unless +exp+ and +act+ have a relative + # error less than +epsilon+. + + def assert_in_epsilon a, b, epsilon = 0.001, msg = nil + assert_in_delta a, b, [a.abs, b.abs].min * epsilon, msg + end + + ## + # Fails unless +collection+ includes +obj+. + + def assert_includes collection, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(collection)} to include #{mu_pp(obj)}" + } + assert_respond_to collection, :include? + assert collection.include?(obj), msg + end + + ## + # Fails unless +obj+ is an instance of +cls+. + + def assert_instance_of cls, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} to be an instance of #{cls}, not #{obj.class}" + } + + assert obj.instance_of?(cls), msg + end + + ## + # Fails unless +obj+ is a kind of +cls+. + + def assert_kind_of cls, obj, msg = nil # TODO: merge with instance_of + msg = message(msg) { + "Expected #{mu_pp(obj)} to be a kind of #{cls}, not #{obj.class}" } + + assert obj.kind_of?(cls), msg + end + + ## + # Fails unless +matcher+ <tt>=~</tt> +obj+. + + def assert_match matcher, obj, msg = nil + msg = message(msg) { "Expected #{mu_pp matcher} to match #{mu_pp obj}" } + assert_respond_to matcher, :"=~" + matcher = Regexp.new Regexp.escape matcher if String === matcher + assert matcher =~ obj, msg + end + + ## + # Fails unless +obj+ is nil + + def assert_nil obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be nil" } + assert obj.nil?, msg + end + + ## + # Fails unless +obj+ is true + + def assert_true obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be true" } + assert obj == true, msg + end + + ## + # Fails unless +obj+ is false + + def assert_false obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be false" } + assert obj == false, msg + end + + ## + # For testing with binary operators. # - #+msg+ may be a String or a Proc. If +msg+ is a String, it will be used - #as the failure message. Otherwise, the result of calling +msg+ will be - #used as the message if the assertion fails. + # assert_operator 5, :<=, 4 + + def assert_operator o1, op, o2 = (predicate = true; nil), msg = nil + return assert_predicate o1, op, msg if predicate + msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op} #{mu_pp(o2)}" } + assert o1.__send__(op, o2), msg + end + + ## + # Fails if stdout or stderr do not output the expected results. + # Pass in nil if you don't care about that streams output. Pass in + # "" if you require it to be silent. Pass in a regexp if you want + # to pattern match. # - #If no +msg+ is given, a default message will be used. + # NOTE: this uses #capture_io, not #capture_subprocess_io. # - # assert(false, "This was expected to be true") - def assert(test, *msgs) - case msg = msgs.first - when String, Proc - when nil - msgs.shift - else - bt = caller.reject { |s| s.start_with?(MINI_DIR) } - raise ArgumentError, "assertion message must be String or Proc, but #{msg.class} was given.", bt - end unless msgs.empty? - super + # See also: #assert_silent + + def assert_output stdout = nil, stderr = nil + out, err = capture_output do + yield + end + + err_msg = Regexp === stderr ? :assert_match : :assert_equal if stderr + out_msg = Regexp === stdout ? :assert_match : :assert_equal if stdout + + y = send err_msg, stderr, err, "In stderr" if err_msg + x = send out_msg, stdout, out, "In stdout" if out_msg + + (!stdout || x) && (!stderr || y) end - # :call-seq: - # assert_block( failure_message = nil ) + ## + # For testing with predicates. # - #Tests the result of the given block. If the block does not return true, - #the assertion will fail. The optional +failure_message+ argument is the same as in - #Assertions#assert. + # assert_predicate str, :empty? # - # assert_block do - # [1, 2, 3].any? { |num| num < 1 } - # end - def assert_block(*msgs) - assert yield, *msgs + # This is really meant for specs and is front-ended by assert_operator: + # + # str.must_be :empty? + + def assert_predicate o1, op, msg = nil + msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op}" } + assert o1.__send__(op), msg end - # :call-seq: - # assert_raise( *args, &block ) + ## + # Fails unless +obj+ responds to +meth+. + + def assert_respond_to obj, meth, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}" + } + assert obj.respond_to?(meth), msg + end + + ## + # Fails unless +exp+ and +act+ are #equal? + + def assert_same exp, act, msg = nil + msg = message(msg) { + data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] + "Expected %s (oid=%d) to be the same as %s (oid=%d)" % data + } + assert exp.equal?(act), msg + end + + ## + # Fails if the block outputs anything to stderr or stdout. # - #Tests if the given block raises an exception. Acceptable exception - #types may be given as optional arguments. If the last argument is a - #String, it will be used as the error message. + # See also: #assert_output + + def assert_silent + assert_output "", "" do + yield + end + end + + ## + # Fails unless the block throws +sym+ + + def assert_throws sym, msg = nil + default = "Expected #{mu_pp(sym)} to have been thrown" + caught = true + catch(sym) do + begin + yield + rescue ThreadError => e # wtf?!? 1.8 + threads == suck + default += ", not \:#{e.message[/uncaught throw \`(\w+?)\'/, 1]}" + rescue ArgumentError => e # 1.9 exception + default += ", not #{e.message.split(/ /).last}" + rescue NameError => e # 1.8 exception + default += ", not #{e.name.inspect}" + end + caught = false + end + + assert caught, message(msg) { default } + end + + def assert_path_exists(path, msg = nil) + msg = message(msg) { "Expected path '#{path}' to exist" } + assert File.exist?(path), msg + end + alias assert_path_exist assert_path_exists + alias refute_path_not_exist assert_path_exists + + def refute_path_exists(path, msg = nil) + msg = message(msg) { "Expected path '#{path}' to not exist" } + refute File.exist?(path), msg + end + alias refute_path_exist refute_path_exists + alias assert_path_not_exist refute_path_exists + + ## + # Captures $stdout and $stderr into strings: # - # assert_raise do #Fails, no Exceptions are raised - # end + # out, err = capture_output do + # puts "Some info" + # warn "You did a bad thing" + # end # - # assert_raise NameError do - # puts x #Raises NameError, so assertion succeeds - # end - def assert_raise(*exp, &b) - case exp.last - when String, Proc - msg = exp.pop - end + # assert_match %r%info%, out + # assert_match %r%bad%, err - begin - yield - rescue MiniTest::Skip => e - return e if exp.include? MiniTest::Skip - raise e - rescue Exception => e - expected = exp.any? { |ex| - if ex.instance_of? Module then - e.kind_of? ex - else - e.instance_of? ex - end - } + def capture_output + require 'stringio' - assert expected, proc { - exception_details(e, message(msg) {"#{mu_pp(exp)} exception expected, not"}.call) - } + captured_stdout, captured_stderr = StringIO.new, StringIO.new - return e - ensure - unless e - exp = exp.first if exp.size == 1 + synchronize do + orig_stdout, orig_stderr = $stdout, $stderr + $stdout, $stderr = captured_stdout, captured_stderr - flunk(message(msg) {"#{mu_pp(exp)} expected but nothing was raised"}) + begin + yield + ensure + $stdout = orig_stdout + $stderr = orig_stderr end end + + return captured_stdout.string, captured_stderr.string end - def assert_raises(*exp, &b) - raise NoMethodError, "use assert_raise", caller + def capture_io + raise NoMethodError, "use capture_output" end - # :call-seq: - # assert_raise_with_message(exception, expected, msg = nil, &block) + ## + # Fails with +msg+ + + def flunk msg = nil + msg ||= "Epic Fail!" + assert false, msg + end + + ## + # used for counting assertions + + def pass msg = nil + assert true + end + + ## + # Fails if +test+ is a true value + + def refute test, msg = nil + msg ||= "Failed refutation, no message given" + not assert(! test, msg) + end + + ## + # Fails if +obj+ is empty. + + def refute_empty obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not be empty" } + assert_respond_to obj, :empty? + refute obj.empty?, msg + end + + ## + # Fails if <tt>exp == act</tt>. # - #Tests if the given block raises an exception with the expected - #message. + # For floats use refute_in_delta. + + def refute_equal exp, act, msg = nil + msg = message(msg) { + "Expected #{mu_pp(act)} to not be equal to #{mu_pp(exp)}" + } + refute exp == act, msg + end + + ## + # For comparing Floats. Fails if +exp+ is within +delta+ of +act+. # - # assert_raise_with_message(RuntimeError, "foo") do - # nil #Fails, no Exceptions are raised - # end + # refute_in_delta Math::PI, (22.0 / 7.0) + + def refute_in_delta exp, act, delta = 0.001, msg = nil + n = (exp - act).abs + msg = message(msg) { + "Expected |#{exp} - #{act}| (#{n}) to not be <= #{delta}" + } + refute delta >= n, msg + end + + ## + # For comparing Floats. Fails if +exp+ and +act+ have a relative error + # less than +epsilon+. + + def refute_in_epsilon a, b, epsilon = 0.001, msg = nil + refute_in_delta a, b, a * epsilon, msg + end + + ## + # Fails if +collection+ includes +obj+. + + def refute_includes collection, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(collection)} to not include #{mu_pp(obj)}" + } + assert_respond_to collection, :include? + refute collection.include?(obj), msg + end + + ## + # Fails if +obj+ is an instance of +cls+. + + def refute_instance_of cls, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} to not be an instance of #{cls}" + } + refute obj.instance_of?(cls), msg + end + + ## + # Fails if +obj+ is a kind of +cls+. + + def refute_kind_of cls, obj, msg = nil # TODO: merge with instance_of + msg = message(msg) { "Expected #{mu_pp(obj)} to not be a kind of #{cls}" } + refute obj.kind_of?(cls), msg + end + + ## + # Fails if +matcher+ <tt>=~</tt> +obj+. + + def refute_match matcher, obj, msg = nil + msg = message(msg) {"Expected #{mu_pp matcher} to not match #{mu_pp obj}"} + assert_respond_to matcher, :"=~" + matcher = Regexp.new Regexp.escape matcher if String === matcher + refute matcher =~ obj, msg + end + + ## + # Fails if +obj+ is nil. + + def refute_nil obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not be nil" } + refute obj.nil?, msg + end + + ## + # Fails if +o1+ is not +op+ +o2+. Eg: # - # assert_raise_with_message(RuntimeError, "foo") do - # raise ArgumentError, "foo" #Fails, different Exception is raised - # end + # refute_operator 1, :>, 2 #=> pass + # refute_operator 1, :<, 2 #=> fail + + def refute_operator o1, op, o2 = (predicate = true; nil), msg = nil + return refute_predicate o1, op, msg if predicate + msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op} #{mu_pp(o2)}"} + refute o1.__send__(op, o2), msg + end + + ## + # For testing with predicates. # - # assert_raise_with_message(RuntimeError, "foo") do - # raise "bar" #Fails, RuntimeError is raised but the message differs - # end + # refute_predicate str, :empty? # - # assert_raise_with_message(RuntimeError, "foo") do - # raise "foo" #Raises RuntimeError with the message, so assertion succeeds - # end - def assert_raise_with_message(exception, expected, msg = nil, &block) - case expected - when String - assert = :assert_equal - when Regexp - assert = :assert_match - else - raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" - end + # This is really meant for specs and is front-ended by refute_operator: + # + # str.wont_be :empty? - ex = m = nil - EnvUtil.with_default_internal(expected.encoding) do - ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do - yield - end - m = ex.message - end - msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} + def refute_predicate o1, op, msg = nil + msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op}" } + refute o1.__send__(op), msg + end - if assert == :assert_equal - assert_equal(expected, m, msg) - else - msg = message(msg) { "Expected #{mu_pp expected} to match #{mu_pp m}" } - assert expected =~ m, msg - block.binding.eval("proc{|_|$~=_}").call($~) + ## + # Fails if +obj+ responds to the message +meth+. + + def refute_respond_to obj, meth, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not respond to #{meth}" } + + refute obj.respond_to?(meth), msg + end + + ## + # Fails if +exp+ is the same (by object identity) as +act+. + + def refute_same exp, act, msg = nil + msg = message(msg) { + data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] + "Expected %s (oid=%d) to not be the same as %s (oid=%d)" % data + } + refute exp.equal?(act), msg + end + + ## + # Skips the current test. Gets listed at the end of the run but + # doesn't cause a failure exit code. + + def pend msg = nil, bt = caller, &_ + msg ||= "Skipped, no message given" + @skip = true + raise Test::Unit::PendedError, msg, bt + end + alias omit pend + + def skip(msg = nil, bt = caller) + raise NoMethodError, "use omit or pend", caller + end + + ## + # Was this testcase skipped? Meant for #teardown. + + def skipped? + defined?(@skip) and @skip + end + + ## + # Takes a block and wraps it with the runner's shared mutex. + + def synchronize + Test::Unit::Runner.runner.synchronize do + yield end - ex end # :call-seq: - # assert_nothing_raised( *args, &block ) - # - #If any exceptions are given as arguments, the assertion will - #fail if one of those exceptions are raised. Otherwise, the test fails - #if any exceptions are raised. - # - #The final argument may be a failure message. + # assert_block( failure_message = nil ) # - # assert_nothing_raised RuntimeError do - # raise Exception #Assertion passes, Exception is not a RuntimeError - # end + #Tests the result of the given block. If the block does not return true, + #the assertion will fail. The optional +failure_message+ argument is the same as in + #Assertions#assert. # - # assert_nothing_raised do - # raise Exception #Assertion fails + # assert_block do + # [1, 2, 3].any? { |num| num < 1 } # end - def assert_nothing_raised(*args) - self._assertions += 1 - if Module === args.last - msg = nil - else - msg = args.pop - end - begin - line = __LINE__; yield - rescue MiniTest::Skip - raise - rescue Exception => e - bt = e.backtrace - as = e.instance_of?(MiniTest::Assertion) - if as - ans = /\A#{Regexp.quote(__FILE__)}:#{line}:in /o - bt.reject! {|ln| ans =~ ln} - end - if ((args.empty? && !as) || - args.any? {|a| a.instance_of?(Module) ? e.is_a?(a) : e.class == a }) - msg = message(msg) { "Exception raised:\n<#{mu_pp(e)}>" } - raise MiniTest::Assertion, msg.call, bt - else - raise - end - end + def assert_block(*msgs) + assert yield, *msgs end # :call-seq: @@ -217,35 +587,6 @@ module Test end # :call-seq: - # assert_throw( tag, failure_message = nil, &block ) - # - #Fails unless the given block throws +tag+, returns the caught - #value otherwise. - # - #An optional failure message may be provided as the final argument. - # - # tag = Object.new - # assert_throw(tag, "#{tag} was not thrown!") do - # throw tag - # end - def assert_throw(tag, msg = nil) - ret = catch(tag) do - begin - yield(tag) - rescue UncaughtThrowError => e - thrown = e.tag - end - msg = message(msg) { - "Expected #{mu_pp(tag)} to have been thrown"\ - "#{%Q[, not #{thrown}] if thrown}" - } - assert(false, msg) - end - assert(true) - ret - end - - # :call-seq: # assert_equal( expected, actual, failure_message = nil ) # #Tests if +expected+ is equal to +actual+. @@ -352,52 +693,6 @@ EOT end # :call-seq: - # assert_respond_to( object, method, failure_message = nil ) - # - #Tests if the given Object responds to +method+. - # - #An optional failure message may be provided as the final argument. - # - # assert_respond_to("hello", :reverse) #Succeeds - # assert_respond_to("hello", :does_not_exist) #Fails - def assert_respond_to(obj, (meth, *priv), msg = nil) - unless priv.empty? - msg = message(msg) { - "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}#{" privately" if priv[0]}" - } - return assert obj.respond_to?(meth, *priv), msg - end - #get rid of overcounting - if caller_locations(1, 1)[0].path.start_with?(MINI_DIR) - return if obj.respond_to?(meth) - end - super(obj, meth, msg) - end - - # :call-seq: - # assert_not_respond_to( object, method, failure_message = nil ) - # - #Tests if the given Object does not respond to +method+. - # - #An optional failure message may be provided as the final argument. - # - # assert_not_respond_to("hello", :reverse) #Fails - # assert_not_respond_to("hello", :does_not_exist) #Succeeds - def assert_not_respond_to(obj, (meth, *priv), msg = nil) - unless priv.empty? - msg = message(msg) { - "Expected #{mu_pp(obj)} (#{obj.class}) to not respond to ##{meth}#{" privately" if priv[0]}" - } - return assert !obj.respond_to?(meth, *priv), msg - end - #get rid of overcounting - if caller_locations(1, 1)[0].path.start_with?(MINI_DIR) - return unless obj.respond_to?(meth) - end - refute_respond_to(obj, meth, msg) - end - - # :call-seq: # assert_send( +send_array+, failure_message = nil ) # # Passes if the method send returns a true value. @@ -457,16 +752,6 @@ EOT alias assert_include assert_includes alias assert_not_include assert_not_includes - def assert_all?(obj, m = nil, &blk) - failed = [] - obj.each do |*a, &b| - unless blk.call(*a, &b) - failed << (a.size > 1 ? a : a[0]) - end - end - assert(failed.empty?, message(m) {failed.pretty_inspect}) - end - def assert_not_all?(obj, m = nil, &blk) failed = [] obj.each do |*a, &b| @@ -477,102 +762,22 @@ EOT assert(failed.empty?, message(m) {failed.pretty_inspect}) end - # compatibility with test-unit - alias pend skip - - if defined?(RubyVM::InstructionSequence) - def syntax_check(code, fname, line) - code = code.dup.force_encoding(Encoding::UTF_8) - RubyVM::InstructionSequence.compile(code, fname, fname, line) - :ok - end - else - def syntax_check(code, fname, line) - code = code.b - code.sub!(/\A(?:\xef\xbb\xbf)?(\s*\#.*$)*(\n)?/n) { - "#$&#{"\n" if $1 && !$2}BEGIN{throw tag, :ok}\n" - } - code = code.force_encoding(Encoding::UTF_8) - catch {|tag| eval(code, binding, fname, line - 1)} - end - end - - def prepare_syntax_check(code, fname = nil, mesg = nil, verbose: nil) - fname ||= caller_locations(2, 1)[0] - mesg ||= fname.to_s - verbose, $VERBOSE = $VERBOSE, verbose - case - when Array === fname - fname, line = *fname - when defined?(fname.path) && defined?(fname.lineno) - fname, line = fname.path, fname.lineno - else - line = 1 - end - yield(code, fname, line, message(mesg) { - if code.end_with?("\n") - "```\n#{code}```\n" - else - "```\n#{code}\n```\n""no-newline" - end - }) - ensure - $VERBOSE = verbose - end - - def assert_valid_syntax(code, *args, **opt) + def assert_syntax_error(code, error, *args, **opt) prepare_syntax_check(code, *args, **opt) do |src, fname, line, mesg| yield if defined?(yield) - assert_nothing_raised(SyntaxError, mesg) do - assert_equal(:ok, syntax_check(src, fname, line), mesg) - end - end - end - - def assert_syntax_error(code, error, *args) - prepare_syntax_check(code, *args) do |src, fname, line, mesg| - yield if defined?(yield) e = assert_raise(SyntaxError, mesg) do syntax_check(src, fname, line) end - assert_match(error, e.message, mesg) - e - end - end - - def assert_normal_exit(testsrc, message = '', child_env: nil, **opt) - assert_valid_syntax(testsrc, caller_locations(1, 1)[0]) - if child_env - child_env = [child_env] - else - child_env = [] - end - out, _, status = EnvUtil.invoke_ruby(child_env + %W'-W0', testsrc, true, :merge_to_stdout, **opt) - assert !status.signaled?, FailDesc[status, message, out] - end - - def assert_ruby_status(args, test_stdin="", message=nil, **opt) - out, _, status = EnvUtil.invoke_ruby(args, test_stdin, true, :merge_to_stdout, **opt) - desc = FailDesc[status, message, out] - assert(!status.signaled?, desc) - message ||= "ruby exit status is not success:" - assert(status.success?, desc) - end - def assert_warning(pat, msg = nil) - result = nil - stderr = EnvUtil.with_default_internal(pat.encoding) { - EnvUtil.verbose_warning { - result = yield - } - } - msg = message(msg) {diff pat, stderr} - assert(pat === stderr, msg) - result - end + # Prism adds ANSI escape sequences to syntax error messages to + # colorize and format them. We strip them out here to make them easier + # to match against in tests. + message = e.message + message.gsub!(/\e\[.*?m/, "") - def assert_warn(*args) - assert_warning(*args) {$VERBOSE = false; yield} + assert_match(error, message, mesg) + e + end end def assert_no_warning(pat, msg = nil) @@ -587,195 +792,53 @@ EOT result end - def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) - # TODO: consider choosing some appropriate limit for MJIT and stop skipping this once it does not randomly fail - skip 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? - - require_relative '../../memory_status' - raise MiniTest::Skip, "unsupported platform" unless defined?(Memory::Status) - - token = "\e[7;1m#{$$.to_s}:#{Time.now.strftime('%s.%L')}:#{rand(0x10000).to_s(16)}:\e[m" - token_dump = token.dump - token_re = Regexp.quote(token) - envs = args.shift if Array === args and Hash === args.first - args = [ - "--disable=gems", - "-r", File.expand_path("../../../memory_status", __FILE__), - *args, - "-v", "-", - ] - if defined? Memory::NO_MEMORY_LEAK_ENVS then - envs ||= {} - newenvs = envs.merge(Memory::NO_MEMORY_LEAK_ENVS) { |_, _, _| break } - envs = newenvs if newenvs - end - args.unshift(envs) if envs - cmd = [ - 'END {STDERR.puts '"#{token_dump}"'"FINAL=#{Memory::Status.new}"}', - prepare, - 'STDERR.puts('"#{token_dump}"'"START=#{$initial_status = Memory::Status.new}")', - '$initial_size = $initial_status.size', - code, - 'GC.start', - ].join("\n") - _, err, status = EnvUtil.invoke_ruby(args, cmd, true, true, **opt) - before = err.sub!(/^#{token_re}START=(\{.*\})\n/, '') && Memory::Status.parse($1) - after = err.sub!(/^#{token_re}FINAL=(\{.*\})\n/, '') && Memory::Status.parse($1) - assert(status.success?, FailDesc[status, message, err]) - ([:size, (rss && :rss)] & after.members).each do |n| - b = before[n] - a = after[n] - next unless a > 0 and b > 0 - assert_operator(a.fdiv(b), :<, limit, message(message) {"#{n}: #{b} => #{a}"}) - end - rescue LoadError - skip - end - # kernel resolution can limit the minimum time we can measure # [ruby-core:81540] - MIN_HZ = MiniTest::Unit::TestCase.windows? ? 67 : 100 + MIN_HZ = /mswin|mingw/ =~ RUBY_PLATFORM ? 67 : 100 MIN_MEASURABLE = 1.0 / MIN_HZ def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) - require 'benchmark' - wait = EnvUtil.apply_timeout_scale(wait) if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" end - tms = Benchmark.measure(msg || '') do - if stop - th = Thread.start {sleep wait; stop.call} - yield - th.join - else - begin - Timeout.timeout(wait) {yield} - rescue Timeout::Error - end + + t0, r0 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error end end - max = pct * tms.real + t1, r1 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + total = t1.utime - t0.utime + t1.stime - t0.stime + t1.cutime - t0.cutime + t1.cstime - t0.cstime + real = r1 - r0 + + max = pct * real min_measurable = MIN_MEASURABLE min_measurable *= 1.30 # add a little (30%) to account for misc. overheads if max < min_measurable max = min_measurable end - assert_operator tms.total, :<=, max, msg + assert_operator total, :<=, max, msg end def assert_is_minus_zero(f) assert(1.0/f == -Float::INFINITY, "#{f} is not -0.0") end - # pattern_list is an array which contains regexp and :*. - # :* means any sequence. - # - # pattern_list is anchored. - # Use [:*, regexp, :*] for non-anchored match. - def assert_pattern_list(pattern_list, actual, message=nil) - rest = actual - anchored = true - pattern_list.each_with_index {|pattern, i| - if pattern == :* - anchored = false - else - if anchored - match = /\A#{pattern}/.match(rest) - else - match = pattern.match(rest) - end - unless match - msg = message(msg) { - expect_msg = "Expected #{mu_pp pattern}\n" - if /\n[^\n]/ =~ rest - actual_mesg = +"to match\n" - rest.scan(/.*\n+/) { - actual_mesg << ' ' << $&.inspect << "+\n" - } - actual_mesg.sub!(/\+\n\z/, '') - else - actual_mesg = "to match " + mu_pp(rest) - end - actual_mesg << "\nafter #{i} patterns with #{actual.length - rest.length} characters" - expect_msg + actual_mesg - } - assert false, msg - end - rest = match.post_match - anchored = true - end - } - if anchored - assert_equal("", rest) - end - end - - # threads should respond to shift method. - # Array can be used. - def assert_join_threads(threads, message = nil) - errs = [] - values = [] - while th = threads.shift - begin - values << th.value - rescue Exception - errs << [th, $!] - th = nil - end - end - values - ensure - if th&.alive? - th.raise(Timeout::Error.new) - th.join rescue errs << [th, $!] - end - if !errs.empty? - msg = "exceptions on #{errs.length} threads:\n" + - errs.map {|t, err| - "#{t.inspect}:\n" + - err.full_message(highlight: false, order: :top) - }.join("\n---\n") - if message - msg = "#{message}\n#{msg}" - end - raise MiniTest::Assertion, msg - end - end - - def assert_all_assertions_foreach(msg = nil, *keys, &block) - all = AllFailures.new - all.foreach(*keys, &block) - ensure - assert(all.pass?, message(msg) {all.message.chomp(".")}) - end - alias all_assertions_foreach assert_all_assertions_foreach - def build_message(head, template=nil, *arguments) #:nodoc: template &&= template.chomp template.gsub(/\G((?:[^\\]|\\.)*?)(\\)?\?/) { $1 + ($2 ? "?" : mu_pp(arguments.shift)) } end - - def message(msg = nil, *args, &default) # :nodoc: - if Proc === msg - super(nil, *args) do - ary = [msg.call, (default.call if default)].compact.reject(&:empty?) - if 1 < ary.length - ary[0...-1] = ary[0...-1].map {|str| str.sub(/(?<!\.)\z/, '.') } - end - begin - ary.join("\n") - rescue Encoding::CompatibilityError - ary.map(&:b).join("\n") - end - end - else - super - end - end end end end diff --git a/tool/lib/test/unit/core_assertions.rb b/tool/lib/test/unit/core_assertions.rb deleted file mode 100644 index 21be860176..0000000000 --- a/tool/lib/test/unit/core_assertions.rb +++ /dev/null @@ -1,216 +0,0 @@ -# frozen_string_literal: true - -require_relative '../../envutil' - -module Test - module Unit - module CoreAssertions - include MiniTest::Assertions - - def mu_pp(obj) #:nodoc: - obj.pretty_inspect.chomp - end - - def assert_file - AssertFile - end - - FailDesc = proc do |status, message = "", out = ""| - pid = status.pid - now = Time.now - faildesc = proc do - if signo = status.termsig - signame = Signal.signame(signo) - sigdesc = "signal #{signo}" - end - log = EnvUtil.diagnostic_reports(signame, pid, now) - if signame - sigdesc = "SIG#{signame} (#{sigdesc})" - end - if status.coredump? - sigdesc = "#{sigdesc} (core dumped)" - end - full_message = ''.dup - message = message.call if Proc === message - if message and !message.empty? - full_message << message << "\n" - end - full_message << "pid #{pid}" - full_message << " exit #{status.exitstatus}" if status.exited? - full_message << " killed by #{sigdesc}" if sigdesc - if out and !out.empty? - full_message << "\n" << out.b.gsub(/^/, '| ') - full_message.sub!(/(?<!\n)\z/, "\n") - end - if log - full_message << "Diagnostic reports:\n" << log.b.gsub(/^/, '| ') - end - full_message - end - faildesc - end - - def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, - success: nil, **opt) - args = Array(args).dup - args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') - stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) - if signo = status.termsig - EnvUtil.diagnostic_reports(Signal.signame(signo), status.pid, Time.now) - end - if block_given? - raise "test_stdout ignored, use block only or without block" if test_stdout != [] - raise "test_stderr ignored, use block only or without block" if test_stderr != [] - yield(stdout.lines.map {|l| l.chomp }, stderr.lines.map {|l| l.chomp }, status) - else - all_assertions(message) do |a| - [["stdout", test_stdout, stdout], ["stderr", test_stderr, stderr]].each do |key, exp, act| - a.for(key) do - if exp.is_a?(Regexp) - assert_match(exp, act) - elsif exp.all? {|e| String === e} - assert_equal(exp, act.lines.map {|l| l.chomp }) - else - assert_pattern_list(exp, act) - end - end - end - unless success.nil? - a.for("success?") do - if success - assert_predicate(status, :success?) - else - assert_not_predicate(status, :success?) - end - end - end - end - status - end - end - - ABORT_SIGNALS = Signal.list.values_at(*%w"ILL ABRT BUS SEGV TERM") - - def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **opt) - unless file and line - loc, = caller_locations(1,1) - file ||= loc.path - line ||= loc.lineno - end - src = <<eom -# -*- coding: #{line += __LINE__; src.encoding}; -*- - require #{__dir__.dump};include Test::Unit::Assertions - END { - puts [Marshal.dump($!)].pack('m'), "assertions=\#{self._assertions}" - } -#{line -= __LINE__; src} - class Test::Unit::Runner - @@stop_auto_run = true - end -eom - args = args.dup - args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) - stdout, stderr, status = EnvUtil.invoke_ruby(args, src, true, true, **opt) - abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) - assert(!abort, FailDesc[status, nil, stderr]) - self._assertions += stdout[/^assertions=(\d+)/, 1].to_i - begin - res = Marshal.load(stdout.unpack("m")[0]) - rescue => marshal_error - ignore_stderr = nil - end - if res - if bt = res.backtrace - bt.each do |l| - l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} - end - bt.concat(caller) - else - res.set_backtrace(caller) - end - raise res unless SystemExit === res - end - - # really is it succeed? - unless ignore_stderr - # the body of assert_separately must not output anything to detect error - assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) - end - assert(status.success?, FailDesc[status, "assert_separately failed", stderr]) - raise marshal_error if marshal_error - end - - class << (AssertFile = Struct.new(:failure_message).new) - include CoreAssertions - def assert_file_predicate(predicate, *args) - if /\Anot_/ =~ predicate - predicate = $' - neg = " not" - end - result = File.__send__(predicate, *args) - result = !result if neg - mesg = "Expected file ".dup << args.shift.inspect - mesg << "#{neg} to be #{predicate}" - mesg << mu_pp(args).sub(/\A\[(.*)\]\z/m, '(\1)') unless args.empty? - mesg << " #{failure_message}" if failure_message - assert(result, mesg) - end - alias method_missing assert_file_predicate - - def for(message) - clone.tap {|a| a.failure_message = message} - end - end - - class AllFailures - attr_reader :failures - - def initialize - @count = 0 - @failures = {} - end - - def for(key) - @count += 1 - yield - rescue Exception => e - @failures[key] = [@count, e] - end - - def foreach(*keys) - keys.each do |key| - @count += 1 - begin - yield key - rescue Exception => e - @failures[key] = [@count, e] - end - end - end - - def message - i = 0 - total = @count.to_s - fmt = "%#{total.size}d" - @failures.map {|k, (n, v)| - v = v.message - "\n#{i+=1}. [#{fmt%n}/#{total}] Assertion for #{k.inspect}\n#{v.b.gsub(/^/, ' | ').force_encoding(v.encoding)}" - }.join("\n") - end - - def pass? - @failures.empty? - end - end - - def assert_all_assertions(msg = nil) - all = AllFailures.new - yield all - ensure - assert(all.pass?, message(msg) {all.message.chomp(".")}) - end - alias all_assertions assert_all_assertions - - end - end -end diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb index d851326aca..188a0d1a19 100644 --- a/tool/lib/test/unit/parallel.rb +++ b/tool/lib/test/unit/parallel.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../.." -require 'test/unit' + +require_relative "../../../test/init" module Test module Unit @@ -9,7 +9,6 @@ module Test undef autorun end - alias orig_run_suite mini_run_suite undef _run_suite undef _run_suites undef run @@ -26,21 +25,24 @@ module Test end end + def _start_method(inst) + _report "start", Marshal.dump([inst.class.name, inst.__name__]) + end + def _run_suite(suite, type) # :nodoc: @partial_report = [] - orig_testout = MiniTest::Unit.output + orig_testout = Test::Unit::Runner.output i,o = IO.pipe - MiniTest::Unit.output = o + Test::Unit::Runner.output = o orig_stdin, orig_stdout = $stdin, $stdout th = Thread.new do begin while buf = (self.verbose ? i.gets : i.readpartial(1024)) - _report "p", buf + _report "p", buf or break end rescue IOError - rescue Errno::EPIPE end end @@ -53,7 +55,7 @@ module Test result = [nil,nil] end - MiniTest::Unit.output = orig_testout + Test::Unit::Runner.output = orig_testout $stdin = orig_stdin $stdout = orig_stdout @@ -71,12 +73,10 @@ module Test result << ($: - @old_loadpath) result << suite.name - begin - _report "done", Marshal.dump(result) - rescue Errno::EPIPE; end + _report "done", Marshal.dump(result) return result ensure - MiniTest::Unit.output = orig_stdout + Test::Unit::Runner.output = orig_stdout $stdin = orig_stdin if orig_stdin $stdout = orig_stdout if orig_stdout o.close if o && !o.closed? @@ -105,12 +105,12 @@ module Test case buf.chomp when /^loadpath (.+?)$/ @old_loadpath = $:.dup - $:.push(*Marshal.load($1.unpack("m")[0].force_encoding("ASCII-8BIT"))).uniq! + $:.push(*Marshal.load($1.unpack1("m").force_encoding("ASCII-8BIT"))).uniq! when /^run (.+?) (.+?)$/ _report "okay" @options = @opts.dup - suites = MiniTest::Unit::TestCase.test_suites + suites = Test::Unit::TestCase.test_suites begin require File.realpath($1) @@ -119,31 +119,39 @@ module Test _report "ready" next end - _run_suites MiniTest::Unit::TestCase.test_suites-suites, $2.to_sym + _run_suites Test::Unit::TestCase.test_suites-suites, $2.to_sym if @need_exit - begin - _report "bye" - rescue Errno::EPIPE; end + _report "bye" exit else _report "ready" end - when /^quit$/ - begin - _report "bye" - rescue Errno::EPIPE; end + when /^quit (.+?)$/, "quit" + if $1 == "timeout" + err = ["", "!!! worker #{$$} killed due to timeout:"] + Thread.list.each do |th| + err << "#{ th.inspect }:" + th.backtrace.each do |s| + err << " #{ s }" + end + end + err << "" + STDERR.puts err.join("\n") + end + _report "bye" exit end end - rescue Errno::EPIPE rescue Exception => e - begin - trace = e.backtrace || ['unknown method'] - err = ["#{trace.shift}: #{e.message} (#{e.class})"] + trace.map{|t| t.prepend("\t") } + trace = e.backtrace || ['unknown method'] + err = ["#{trace.shift}: #{e.message} (#{e.class})"] + trace.map{|t| "\t" + t } + if @stdout _report "bye", Marshal.dump(err.join("\n")) - rescue Errno::EPIPE;end + else + raise "failed to report a failure due to lack of @stdout" + end exit ensure @stdin.close if @stdin @@ -153,24 +161,28 @@ module Test def _report(res, *args) # :nodoc: @stdout.write(args.empty? ? "#{res}\n" : "#{res} #{args.pack("m0")}\n") + true + rescue Errno::EPIPE + rescue TypeError => e + abort("#{e.inspect} in _report(#{res.inspect}, #{args.inspect})\n#{e.backtrace.join("\n")}") end def puke(klass, meth, e) # :nodoc: - if e.is_a?(MiniTest::Skip) - new_e = MiniTest::Skip.new(e.message) + if e.is_a?(Test::Unit::PendedError) + new_e = Test::Unit::PendedError.new(e.message) new_e.set_backtrace(e.backtrace) e = new_e end - @partial_report << [klass.name, meth, e.is_a?(MiniTest::Assertion) ? e : ProxyError.new(e)] + @partial_report << [klass.name, meth, e.is_a?(Test::Unit::AssertionFailedError) ? e : ProxyError.new(e)] super end def record(suite, method, assertions, time, error) # :nodoc: case error when nil - when MiniTest::Assertion, MiniTest::Skip + when Test::Unit::AssertionFailedError, Test::Unit::PendedError case error.cause - when nil, MiniTest::Assertion, MiniTest::Skip + when nil, Test::Unit::AssertionFailedError, Test::Unit::PendedError else bt = error.backtrace error = error.class.new(error.message) @@ -179,7 +191,7 @@ module Test else error = ProxyError.new(error) end - _report "record", Marshal.dump([suite.name, method, assertions, time, error]) + _report "record", Marshal.dump([suite.name, method, assertions, time, error, suite.instance_method(method).source_location]) super end end @@ -189,20 +201,21 @@ end if $0 == __FILE__ module Test module Unit - class TestCase < MiniTest::Unit::TestCase # :nodoc: all + class TestCase # :nodoc: all undef on_parallel_worker? def on_parallel_worker? true end + def self.on_parallel_worker? + true + end end end end require 'rubygems' - module Gem # :nodoc: - end - class Gem::TestCase < MiniTest::Unit::TestCase # :nodoc: - @@project_dir = File.expand_path('../../../../..', __FILE__) + begin + require 'rake' + rescue LoadError end - Test::Unit::Worker.new.run(ARGV) end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb index 58cfbcab99..51ffff37eb 100644 --- a/tool/lib/test/unit/testcase.rb +++ b/tool/lib/test/unit/testcase.rb @@ -1,25 +1,287 @@ # frozen_string_literal: true -require 'test/unit/assertions' +require_relative 'assertions' +require_relative '../../core_assertions' module Test module Unit - # remove silly TestCase class - remove_const(:TestCase) if defined?(self::TestCase) - class TestCase < MiniTest::Unit::TestCase # :nodoc: all - include Assertions + ## + # Provides a simple set of guards that you can use in your tests + # to skip execution if it is not applicable. These methods are + # mixed into TestCase as both instance and class methods so you + # can use them inside or outside of the test methods. + # + # def test_something_for_mri + # skip "bug 1234" if jruby? + # # ... + # end + # + # if windows? then + # # ... lots of test methods ... + # end - def on_parallel_worker? - false + module Guard + + ## + # Is this running on jruby? + + def jruby? platform = RUBY_PLATFORM + "java" == platform end + ## + # Is this running on mri? + + def mri? platform = RUBY_DESCRIPTION + /^ruby/ =~ platform + end + + ## + # Is this running on windows? + + def windows? platform = RUBY_PLATFORM + /mswin|mingw/ =~ platform + end + + ## + # Is this running on mingw? + + def mingw? platform = RUBY_PLATFORM + /mingw/ =~ platform + end + + end + + ## + # Provides before/after hooks for setup and teardown. These are + # meant for library writers, NOT for regular test authors. See + # #before_setup for an example. + + module LifecycleHooks + ## + # Runs before every test, after setup. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def after_setup; end + + ## + # Runs before every test, before setup. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # As a simplistic example: + # + # module MyTestUnitPlugin + # def before_setup + # super + # # ... stuff to do before setup is run + # end + # + # def after_setup + # # ... stuff to do after setup is run + # super + # end + # + # def before_teardown + # super + # # ... stuff to do before teardown is run + # end + # + # def after_teardown + # # ... stuff to do after teardown is run + # super + # end + # end + # + # class Test::Unit::Runner::TestCase + # include MyTestUnitPlugin + # end + + def before_setup; end + + ## + # Runs after every test, before teardown. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def before_teardown; end + + ## + # Runs after every test, after teardown. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def after_teardown; end + end + + ## + # Subclass TestCase to create your own tests. Typically you'll want a + # TestCase subclass per implementation class. + # + # See <code>Test::Unit::AssertionFailedError</code>s + + class TestCase + include Assertions + include CoreAssertions + + include LifecycleHooks + include Guard + extend Guard + + attr_reader :__name__ # :nodoc: + + # Method name of this test. + alias method_name __name__ + + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, + Interrupt, SystemExit] # :nodoc: + + ## + # Runs the tests reporting the status to +runner+ + def run runner - @options = runner.options - super runner + @__runner_options__ = runner.options + trap "INFO" do + runner.report.each_with_index do |msg, i| + warn "\n%3d) %s" % [i + 1, msg] + end + warn '' + time = runner.start_time ? Time.now - runner.start_time : 0 + warn "Current Test: %s#%s %.2fs" % [self.class, self.__name__, time] + runner.status $stderr + end if runner.info_signal + + start_time = Time.now + + result = "" + + begin + @__passed__ = nil + self.before_setup + self.setup + self.after_setup + self.run_test self.__name__ + result = "." unless io? + time = Time.now - start_time + runner.record self.class, self.__name__, self._assertions, time, nil + @__passed__ = true + rescue *PASSTHROUGH_EXCEPTIONS + raise + rescue Exception => e + @__passed__ = Test::Unit::PendedError === e + time = Time.now - start_time + runner.record self.class, self.__name__, self._assertions, time, e + result = runner.puke self.class, self.__name__, e + ensure + %w{ before_teardown teardown after_teardown }.each do |hook| + begin + self.send hook + rescue *PASSTHROUGH_EXCEPTIONS + raise + rescue Exception => e + @__passed__ = false + runner.record self.class, self.__name__, self._assertions, time, e + result = runner.puke self.class, self.__name__, e + end + end + trap 'INFO', 'DEFAULT' if runner.info_signal + end + result + end + + RUN_TEST_TRACE = "#{__FILE__}:#{__LINE__+3}:in `run_test'".freeze + def run_test(name) + progname, $0 = $0, "#{$0}: #{self.class}##{name}" + self.__send__(name) + ensure + $@.delete(RUN_TEST_TRACE) if $@ + $0 = progname + end + + def initialize name # :nodoc: + @__name__ = name + @__io__ = nil + @__passed__ = nil + @@__current__ = self # FIX: make thread local + end + + def self.current # :nodoc: + @@__current__ # FIX: make thread local + end + + ## + # Return the output IO object + + def io + @__io__ = true + Test::Unit::Runner.output + end + + ## + # Have we hooked up the IO yet? + + def io? + @__io__ + end + + def self.reset # :nodoc: + @@test_suites = {} + @@test_suites[self] = true + end + + reset + + def self.inherited klass # :nodoc: + @@test_suites[klass] = true + super + end + + @test_order = :sorted + + class << self + attr_writer :test_order end def self.test_order - :sorted + defined?(@test_order) ? @test_order : superclass.test_order + end + + def self.test_suites # :nodoc: + @@test_suites.keys + end + + def self.test_methods # :nodoc: + public_instance_methods(true).grep(/^test/) + end + + ## + # Returns true if the test passed. + + def passed? + @__passed__ + end + + ## + # Runs before every test. Use this to set up before each test + # run. + + def setup; end + + ## + # Runs after every test. Use this to clean up after each test + # run. + + def teardown; end + + def on_parallel_worker? + false end def self.method_added(name) @@ -27,7 +289,7 @@ module Test return unless name.to_s.start_with?("test_") @test_methods ||= {} if @test_methods[name] - warn "test/unit warning: method #{ self }##{ name } is redefined" + raise AssertionFailedError, "test/unit: method #{ self }##{ name } is redefined" end @test_methods[name] = true end diff --git a/tool/lib/tracepointchecker.rb b/tool/lib/tracepointchecker.rb index 47822ecef5..3254e59357 100644 --- a/tool/lib/tracepointchecker.rb +++ b/tool/lib/tracepointchecker.rb @@ -80,7 +80,7 @@ module TracePointChecker call_stack.push method STATE[:count] += 1 - verbose_out :psuh, method if verbose + verbose_out :push, method if verbose } TRACES << TracePoint.new(*return_events){|tp| diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb index dd6162c8c2..26c9763c13 100644 --- a/tool/lib/vcs.rb +++ b/tool/lib/vcs.rb @@ -1,12 +1,34 @@ # vcs require 'fileutils' require 'optparse' +require 'pp' +require 'tempfile' # This library is used by several other tools/ scripts to detect the current # VCS in use (e.g. SVN, Git) or to interact with that VCS. ENV.delete('PWD') +class VCS + DEBUG_OUT = STDERR.dup + + def self.dump(obj, pre = nil) + out = DEBUG_OUT + @pp ||= PP.new(out) + @pp.guard_inspect_key do + if pre + @pp.group(pre.size, pre) { + obj.pretty_print(@pp) + } + else + obj.pretty_print(@pp) + end + @pp.flush + out << "\n" + end + end +end + unless File.respond_to? :realpath require 'pathname' def File.realpath(arg) @@ -15,106 +37,23 @@ unless File.respond_to? :realpath end def IO.pread(*args) - STDERR.puts(args.inspect) if $DEBUG + VCS.dump(args, "args: ") if $DEBUG popen(*args) {|f|f.read} end -if RUBY_VERSION < "2.0" - class IO - @orig_popen = method(:popen) - - if defined?(fork) - def self.popen(command, *rest, &block) - if command.kind_of?(Hash) - env = command - command = rest.shift - end - opts = rest.last - if opts.kind_of?(Hash) - dir = opts.delete(:chdir) - rest.pop if opts.empty? - opts.delete(:external_encoding) - end - - if block - @orig_popen.call("-", *rest) do |f| - if f - yield(f) - else - Dir.chdir(dir) if dir - ENV.replace(env) if env - exec(*command) - end - end - else - f = @orig_popen.call("-", *rest) - unless f - Dir.chdir(dir) if dir - ENV.replace(env) if env - exec(*command) - end - f - end - end - else - require 'shellwords' - def self.popen(command, *rest, &block) - if command.kind_of?(Hash) - env = command - oldenv = ENV.to_hash - command = rest.shift - end - opts = rest.last - if opts.kind_of?(Hash) - dir = opts.delete(:chdir) - rest.pop if opts.empty? - opts.delete(:external_encoding) - end - - command = command.shelljoin if Array === command - Dir.chdir(dir || ".") do - ENV.replace(env) if env - @orig_popen.call(command, *rest, &block) - ENV.replace(oldenv) if oldenv - end - end - end - end -else - module DebugPOpen - verbose, $VERBOSE = $VERBOSE, nil if RUBY_VERSION < "2.1" - refine IO.singleton_class do - def popen(*args) - STDERR.puts args.inspect if $DEBUG - super - end - end - ensure - $VERBOSE = verbose unless verbose.nil? - end - using DebugPOpen - module DebugSystem - def system(*args) - STDERR.puts args.inspect if $DEBUG - exception = false - opts = Hash.try_convert(args[-1]) - if RUBY_VERSION >= "2.6" - unless opts - opts = {} - args << opts - end - exception = opts.fetch(:exception) {opts[:exception] = true} - elsif opts - exception = opts.delete(:exception) {true} - args.pop if opts.empty? - end - ret = super(*args) - raise "Command failed with status (#$?): #{args[0]}" if exception and !ret - ret +module DebugPOpen + refine IO.singleton_class do + def popen(*args) + VCS.dump(args, "args: ") if $DEBUG + super end end - module Kernel - prepend(DebugSystem) +end +using DebugPOpen +module DebugSystem + def system(*args, exception: true, **opts) + VCS.dump(args, "args: ") if $DEBUG + super(*args, exception: exception, **opts) end end @@ -127,14 +66,19 @@ class VCS @@dirs << [dir, self, pred] end - def self.detect(path, options = {}, argv = ::ARGV) + def self.detect(path = '.', options = {}, parser = nil, **opts) + options.update(opts) uplevel_limit = options.fetch(:uplevel_limit, 0) curr = path begin @@dirs.each do |dir, klass, pred| if pred ? pred[curr, dir] : File.directory?(File.join(curr, dir)) + if klass.const_defined?(:COMMAND) + IO.pread([{'LANG' => 'C', 'LC_ALL' => 'C'}, klass::COMMAND, "--version"]) rescue next + end vcs = klass.new(curr) - vcs.parse_options(argv) + vcs.define_options(parser) if parser + vcs.set_options(options) return vcs end end @@ -151,6 +95,27 @@ class VCS String === path or path.respond_to?(:to_path) end + def self.define_options(parser, opts = {}) + parser.separator(" VCS common options:") + parser.define("--[no-]dryrun") {|v| opts[:dryrun] = v} + parser.define("--[no-]debug") {|v| opts[:debug] = v} + parser.define("-z", "--zone=OFFSET", /\A[-+]\d\d:\d\d\z/) {|v| opts[:zone] = v} + opts + end + + def release_date(time) + t = time.getlocal(@zone) + [ + t.strftime('#define RUBY_RELEASE_YEAR %Y'), + t.strftime('#define RUBY_RELEASE_MONTH %-m'), + t.strftime('#define RUBY_RELEASE_DAY %-d'), + ] + end + + def self.short_revision(rev) + rev + end + attr_reader :srcdir def initialize(path) @@ -158,39 +123,37 @@ class VCS super() end - def parse_options(opts, parser = OptionParser.new) - case opts - when Array - parser.on("--[no-]dryrun") {|v| @dryrun = v} - parser.on("--[no-]debug") {|v| @debug = v} - parser.parse(opts) - @debug = $DEBUG unless defined?(@debug) - @dryrun = @debug unless defined?(@dryrun) - when Hash - unless (keys = opts.keys - [:debug, :dryrun]).empty? - raise "Unknown options: #{keys.join(', ')}" - end - @debug = opts.fetch(:debug) {$DEBUG} - @dryrun = opts.fetch(:dryrun) {@debug} - end + def chdir(path) + @srcdir = path + end + + def define_options(parser) + end + + def set_options(opts) + @debug = opts.fetch(:debug) {$DEBUG} + @dryrun = opts.fetch(:dryrun) {@debug} + @zone = opts.fetch(:zone) {'+09:00'} end attr_reader :dryrun, :debug alias dryrun? dryrun alias debug? debug - NullDevice = defined?(IO::NULL) ? IO::NULL : - %w[/dev/null NUL NIL: NL:].find {|dev| File.exist?(dev)} + NullDevice = IO::NULL - # return a pair of strings, the last revision and the last revision in which - # +path+ was modified. + # returns + # * the last revision of the current branch + # * the last revision in which +path+ was modified + # * the last modified time of +path+ + # * the last commit title since the latest upstream def get_revisions(path) if self.class.local_path?(path) path = relative_to(path) end last, changed, modified, *rest = ( begin - if NullDevice + if NullDevice and !debug? save_stderr = STDERR.dup STDERR.reopen NullDevice, 'w' end @@ -217,6 +180,7 @@ class VCS rescue ArgumentError modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 end + modified = modified.getlocal(@zone) end return last, changed, modified, *rest end @@ -247,183 +211,76 @@ class VCS end def after_export(dir) + FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + FileUtils.rm_rf(Dir.glob("#{dir}/.mailmap")) + end + + def revision_handler(rev) + self.class end def revision_name(rev) - self.class.revision_name(rev) + revision_handler(rev).revision_name(rev) end def short_revision(rev) - self.class.short_revision(rev) + revision_handler(rev).short_revision(rev) end - class SVN < self - register(".svn") - COMMAND = ENV['SVN'] || 'svn' - - def self.revision_name(rev) - "r#{rev}" - end - - def self.short_revision(rev) - rev - end - - def _get_revisions(path, srcdir = nil) - if srcdir and local_path?(path) - path = File.join(srcdir, path) - end - if srcdir - info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") - info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s - end - info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") - _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) - modified = info_xml[/<date>([^<>]*)/, 1] - branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] - [last, changed, modified, branch] - end - - def self.search_root(path) - return unless local_path?(path) - parent = File.realpath(path) - begin - parent = File.dirname(wkdir = parent) - return wkdir if File.directory?(wkdir + "/.svn") - end until parent == wkdir - end - - def get_info - @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") - end - - def url - unless @url - url = get_info[/<root>(.*)<\/root>/, 1] - @url = URI.parse(url+"/") if url - end - @url - end - - def wcroot - unless @wcroot - info = get_info - @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] - @wcroot ||= self.class.search_root(@srcdir) - end - @wcroot - end - - def branch(name) - return trunk if name == "trunk" - url + "branches/#{name}" - end - - def tag(name) - url + "tags/#{name}" - end - - def trunk - url + "trunk" - end - alias master trunk - - def branch_list(pat) - IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| - f.each do |line| - line.chomp! - line.chomp!('/') - yield(line) if File.fnmatch?(pat, line) - end - end - end - - def grep(pat, tag, *files, &block) - cmd = %W"#{COMMAND} cat" - files.map! {|n| File.join(tag, n)} if tag - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files]) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end - - def export(revision, url, dir, keep_temp = false) - if @srcdir and (rootdir = wcroot) - srcdir = File.realpath(@srcdir) - rootdir << "/" - if srcdir.start_with?(rootdir) - subdir = srcdir[rootdir.size..-1] - subdir = nil if subdir.empty? - FileUtils.mkdir_p(svndir = dir+"/.svn") - FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) - system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false - FileUtils.rm_rf(svndir) unless keep_temp - if subdir - tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") - File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") - FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) - begin - Dir.rmdir("#{dir}/#{subdir}") - end until (subdir = File.dirname(subdir)) == '.' - FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) - Dir.rmdir(tmpdir) - end - return true - end - end - IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| - pipe.each {|line| /^A/ =~ line or yield line} - end - $?.success? - end - - def after_export(dir) - FileUtils.rm_rf(dir+"/.svn") - end + # make-snapshot generates only release_date whereas file2lastrev generates both release_date and release_datetime + def revision_header(last, release_date, release_datetime = nil, branch = nil, title = nil, limit: 20) + short = short_revision(last) + if /[^\x00-\x7f]/ =~ title and title.respond_to?(:force_encoding) + title = title.dup.force_encoding("US-ASCII") + end + code = [ + "#define RUBY_REVISION #{short.inspect}", + ] + unless short == last + code << "#define RUBY_FULL_REVISION #{last.inspect}" + end + if branch + e = '..' + name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} + name = name.dump.sub(/\\#/, '#') + code << "#define RUBY_BRANCH_NAME #{name}" + end + if title + title = title.dump.sub(/\\#/, '#') + code << "#define RUBY_LAST_COMMIT_TITLE #{title}" + end + if release_datetime + t = release_datetime.utc + code << t.strftime('#define RUBY_RELEASE_DATETIME "%FT%TZ"') + end + code += self.release_date(release_date) + code + end - def branch_beginning(url) - # `--limit` of svn-log is useless in this case, because it is - # applied before `--search`. - rev = IO.pread(%W[ #{COMMAND} log --xml - --search=matz --search-and=has\ started - -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] - rev.to_i if rev - end - - def export_changelog(url, from, to, path) - range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') - IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -r#{range} #{url}") do |r| - open(path, 'w') do |w| - IO.copy_stream(r, w) + class GIT < self + register(".git") do |path, dir| + SAFE_DIRECTORIES ||= + begin + command = ENV["GIT"] || 'git' + dirs = IO.popen(%W"#{command} config --global --get-all safe.directory", &:read).split("\n") + rescue + command = nil + dirs = [] + ensure + VCS.dump(dirs, "safe.directory: ") if $DEBUG + COMMAND = command end - end - end - def commit - args = %W"#{COMMAND} commit" - if dryrun? - STDERR.puts(args.inspect) - return true - end - system(*args) + COMMAND and File.exist?(File.join(path, dir)) end - end - - class GIT < self - register(".git") {|path, dir| File.exist?(File.join(path, dir))} - COMMAND = ENV["GIT"] || 'git' def cmd_args(cmds, srcdir = nil) (opts = cmds.last).kind_of?(Hash) or cmds << (opts = {}) opts[:external_encoding] ||= "UTF-8" - if srcdir and self.class.local_path?(srcdir) + if srcdir opts[:chdir] ||= srcdir end - STDERR.puts cmds.inspect if debug? + VCS.dump(cmds, "cmds: ") if debug? and !$DEBUG cmds end @@ -432,7 +289,9 @@ class VCS end def cmd_read_at(srcdir, cmds) - without_gitconfig { IO.pread(*cmd_args(cmds, srcdir)) } + result = without_gitconfig { IO.pread(*cmd_args(cmds, srcdir)) } + VCS.dump(result, "result: ") if debug? + result end def cmd_pipe(*cmds, &block) @@ -443,15 +302,41 @@ class VCS cmd_read_at(@srcdir, cmds) end + def svn_revision(log) + if /^ *git-svn-id: .*@(\d+) .*\n+\z/ =~ log + $1.to_i + end + end + def _get_revisions(path, srcdir = nil) + ref = Branch === path ? path.to_str : 'HEAD' gitcmd = [COMMAND] - last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', 'HEAD']]).rstrip + last = nil + IO.pipe do |r, w| + last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref, err: w]]).rstrip + w.close + unless r.eof? + raise VCS::NotFoundError, "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" + end + end log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) changed = log[/\Acommit (\h+)/, 1] modified = log[/^CommitDate:\s+(.*)/, 1] - branch = cmd_read_at(srcdir, [gitcmd + %W[symbolic-ref --short HEAD]]) + if rev = svn_revision(log) + if changed == last + last = rev + else + svn_rev = svn_revision(cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--format=%B', last]])) + last = svn_rev if svn_rev + end + changed = rev + end + branch = cmd_read_at(srcdir, [gitcmd + %W[symbolic-ref --short #{ref}]]) if branch.empty? - branch_list = cmd_read_at(srcdir, [gitcmd + %W[branch --list --contains HEAD]]).lines.to_a + branch = cmd_read_at(srcdir, [gitcmd + %W[describe --contains #{ref}]]).strip + end + if branch.empty? + branch_list = cmd_read_at(srcdir, [gitcmd + %W[branch --list --contains #{ref}]]).lines.to_a branch, = branch_list.grep(/\A\*/) case branch when /\A\* *\(\S+ detached at (.*)\)\Z/ @@ -472,7 +357,7 @@ class VCS branch = ":detached:" if branch.empty? upstream = cmd_read_at(srcdir, [gitcmd + %W[branch --list --format=%(upstream:short) #{branch}]]) upstream.chomp! - title = cmd_read_at(srcdir, [gitcmd + %W[log --format=%s -n1 #{upstream}..HEAD]]) + title = cmd_read_at(srcdir, [gitcmd + %W[log --format=%s -n1 #{upstream}..#{ref}]]) title = nil if title.empty? [last, changed, modified, branch, title] end @@ -486,17 +371,35 @@ class VCS end def without_gitconfig - home = ENV.delete('HOME') + envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| + h[v] = ENV.delete(v) + end + ENV['GIT_CONFIG_SYSTEM'] = NullDevice + ENV['GIT_CONFIG_GLOBAL'] = global_config yield ensure - ENV['HOME'] = home if home + ENV.update(envs) + end + + def global_config + return NullDevice if SAFE_DIRECTORIES.empty? + unless @gitconfig + @gitconfig = Tempfile.new(%w"vcs_ .gitconfig") + @gitconfig.close + ENV['GIT_CONFIG_GLOBAL'] = @gitconfig.path + SAFE_DIRECTORIES.each do |dir| + system(*%W[#{COMMAND} config --global --add safe.directory #{dir}]) + end + VCS.dump(`#{COMMAND} config --global --get-all safe.directory`, "safe.directory: ") if debug? + end + @gitconfig.path end def initialize(*) super - if srcdir = @srcdir and self.class.local_path?(srcdir) - @srcdir = File.realpath(srcdir) - end + @srcdir = File.realpath(@srcdir) + @gitconfig = nil + VCS.dump(@srcdir, "srcdir: ") if debug? self end @@ -540,47 +443,176 @@ class VCS end def export(revision, url, dir, keep_temp = false) - ret = system(COMMAND, "clone", "-s", (@srcdir || '.').to_s, "-b", url, dir) - ret - end - - def after_export(dir) - FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return + GIT.new(File.expand_path(dir)) end def branch_beginning(url) - cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H - --author=matz --committer=matz --grep=has\ started - -- version.h include/ruby/version.h]) + year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i + cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25 + --author=matz --committer=matz --grep=started\\.$ + #{url} -- version.h include/ruby/version.h])[/.*/] end - def export_changelog(url, from, to, path) + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true) from, to = [from, to].map do |rev| rev or next - if Integer === rev - rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -n1 --format=format:%H" << - "--grep=^ *git-svn-id: .*@#{rev} ") - end rev unless rev.empty? end - unless /./.match(from ||= branch_beginning(url)) - raise "cannot find the beginning revision of the branch" + unless from&.match?(/./) or (from = branch_beginning(url))&.match?(/./) + warn "no starting commit found", uplevel: 1 + from = nil end - range = [from, (to || 'HEAD')].join('^..') - cmd_pipe({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log --format=medium --no-notes --date=iso-local --topo-order #{range}", "rb") do |r| - format_changelog(r, path) + if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + chdir: @srcdir, exception: false) + system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", + chdir: @srcdir, exception: false) + else + warn "Could not fetch notes/commits tree", uplevel: 1 end - end + to ||= url.to_str + if from + arg = ["#{from}^..#{to}"] + else + arg = ["--since=25 Dec 00:00:00", to] + end + if base_url == true + env = CHANGELOG_ENV + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') + end + base_url = remote + end + writer = changelog_formatter(path, arg, base_url) + if !path or path == '-' + writer[$stdout] + else + File.open(path, 'wb', &writer) + end + end + + LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' + CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + + def changelog_formatter(path, arg, base_url = nil) + env = CHANGELOG_ENV + cmd = %W[#{COMMAND} log + --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges + --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] + ] + date = "--date=iso-local" + unless system(env, *cmd, date, "-1", chdir: @srcdir, out: NullDevice, exception: false) + date = "--date=iso" + end + cmd << date + cmd.concat(arg) + proc do |w| + w.print "-*- coding: utf-8 -*-\n" + w.print "\n""base-url = #{base_url}\n" if base_url + + begin + ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true) + .grep_v(/^ *(?:#|$)/) + .to_h {|v| [v, true]} + ignore_revs = nil if ignore_revs.empty? + rescue Errno::ENOENT + end + + cmd_pipe(env, cmd, chdir: @srcdir) do |r| + r.gets(sep = "commit ") + sep = "\n" + sep + while s = r.gets(sep, chomp: true) + h, s = s.split(/^$/, 2) + if ignore_revs&.key?(h[/\A\h{40}/]) + next + end + + next if /^Author: *dependabot\[bot\]/ =~ h + + h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') + if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') + fix = $1 + next if /\A *skip\Z/ =~ fix + s = s.lines + fix.each_line do |x| + next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') + b = ($2&.to_i || (s.size - 1 + $3.to_i)) + sp = $1 + if x.sub!(/^,(?:(\d+)|\$(?:-\d+)?)/, '') + range = b..($1&.to_i || (s.size - 1 + $2.to_i)) + else + range = b..b + end + case x + when %r[^s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\1(.*)\1([gr]+)?]o + wrong = $2 + correct = $3 + if opt = $4 and opt.include?("r") # regexp + wrong = Regexp.new(wrong) + correct.gsub!(/(?<!\\)(?:\\\\)*\K(?:\\n)+/) {"\n" * ($&.size / 2)} + sub = opt.include?("g") ? :gsub! : :sub! + else + sub = false + end + range.each do |n| + if sub + ss = s[n].sub(/^#{sp}/, "") # un-indent for /^/ + if ss.__send__(sub, wrong, correct) + s[n, 1] = ss.lines.map {|l| "#{sp}#{l}"} + next + end + else + begin + s[n][wrong] = correct + rescue IndexError + else + next + end + end + message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] + from = [1, n-2].max + to = [s.size-1, n+2].min + s.each_with_index do |e, i| + next if i < from + break if to < i + message << "#{i}:#{e}" + end + raise message.join('') + end + when %r[^i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\1]o + insert = "#{sp}#{$2}\n" + range.reverse_each do |n| + s[n, 0] = insert + end + when %r[^d] + s[range] = [] + end + end + s = s.join('') + end + + s.gsub!(%r[(?!<\w)([-\w]+/[-\w]+)(?:@(\h{8,40})|#(\d{5,}))\b]) do + path = defined?($2) ? "commit/#{$2}" : "pull/#{$3}" + "[#$&](https://github.com/#{$1}/#{path})" + end + if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s + issue = "#{$1}pull/" + s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} + end - def format_changelog(r, path) - IO.copy_stream(r, path) + s.gsub!(/ +\n/, "\n") + s.sub!(/^Notes:/, ' \&') + w.print sep, h, s + end + end + end end - def commit(opts = {}) - args = [COMMAND, "push"] - args << "-n" if dryrun + def upstream (branch = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD")).chomp! (upstream = cmd_read(%W"#{COMMAND} branch --list --format=%(upstream) #{branch}")).chomp! while ref = upstream[%r"\Arefs/heads/(.*)", 1] @@ -589,81 +621,36 @@ class VCS unless %r"\Arefs/remotes/([^/]+)/(.*)" =~ upstream raise "Upstream not found" end - args << $1 << "HEAD:#$2" + [$1, $2] + end + + def commit(opts = {}) + args = [COMMAND, "push"] + args << "-n" if dryrun? + remote, branch = upstream + args << remote + branches = %W[refs/notes/commits:refs/notes/commits HEAD:#{branch}] if dryrun? - STDERR.puts(args.inspect) + branches.each do |b| + VCS.dump(args + [b], "commit: ") + end return true end - system(*args) or return false + branches.each do |b| + system(*(args + [b])) or return false + end true end end - class GITSVN < GIT - def self.revision_name(rev) - SVN.revision_name(rev) + class Null < self + def get_revisions(path, srcdir = nil) + @modified ||= Time.now - 10 + return nil, nil, @modified end - def self.short_revision(rev) - SVN.short_revision(rev) - end - - def format_changelog(r, path) - open(path, 'w') do |w| - sep = "-"*72 - w.puts sep - while s = r.gets('') - author = s[/^Author:\s*(\S+)/, 1] - time = s[/^Date:\s*(.+)/, 1] - s = r.gets('') - s.gsub!(/^ {4}/, '') - s.sub!(/^git-svn-id: .*@(\d+) .*\n+\z/, '') - rev = $1 - s.gsub!(/^ {8}/, '') if /^(?! {8}|$)/ !~ s - s.sub!(/\n\n\z/, "\n") - if /\A(\d+)-(\d+)-(\d+)/ =~ time - date = Time.new($1.to_i, $2.to_i, $3.to_i).strftime("%a, %d %b %Y") - end - lines = s.count("\n") - lines = "#{lines} line#{lines == 1 ? '' : 's'}" - w.puts "r#{rev} | #{author} | #{time} (#{date}) | #{lines}\n\n" - w.puts s, sep - end - end - end - - def last_changed_revision - rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] - com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp - return rev, com - end - - def commit(opts = {}) - rev, com = last_changed_revision - head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp - - commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") - commits.each_with_index do |l, i| - r, a, c = l.split - dcommit = [COMMAND, "svn", "dcommit"] - dcommit.insert(-2, "-n") if dryrun - dcommit << "--add-author-from" unless a == c - dcommit << r - system(*dcommit) or return false - system(COMMAND, "checkout", head) or return false - system(COMMAND, "rebase") or return false - end - - if rev - old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] - old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] - 3.times do - sleep 2 - system(*%W"#{COMMAND} pull --no-edit --rebase") - break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) - end - end - true + def revision_header(last, release_date, release_datetime = nil, branch = nil, title = nil, limit: 20) + self.release_date(release_date) end end end diff --git a/tool/lib/vpath.rb b/tool/lib/vpath.rb index 48ab148405..fa819f3242 100644 --- a/tool/lib/vpath.rb +++ b/tool/lib/vpath.rb @@ -53,10 +53,11 @@ class VPath end def def_options(opt) + opt.separator(" VPath common options:") opt.on("-I", "--srcdir=DIR", "add a directory to search path") {|dir| @additional << dir } - opt.on("-L", "--vpath=PATH LIST", "add directories to search path") {|dirs| + opt.on("-L", "--vpath=PATH-LIST", "add directories to search path") {|dirs| @additional << [dirs] } opt.on("--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, "separator for vpath") {|sep, vsep| @@ -80,6 +81,10 @@ class VPath @list end + def add(path) + @additional << path + end + def strip(path) prefix = list.map {|dir| Regexp.quote(dir)} path.sub(/\A#{prefix.join('|')}(?:\/|\z)/, '') diff --git a/tool/ln_sr.rb b/tool/ln_sr.rb new file mode 100755 index 0000000000..e1b5b6f76b --- /dev/null +++ b/tool/ln_sr.rb @@ -0,0 +1,131 @@ +#!/usr/bin/ruby + +target_directory = true +noop = false +force = false +quiet = false + +until ARGV.empty? + case ARGV[0] + when '-n' + noop = true + when '-f' + force = true + when '-T' + target_directory = false + when '-q' + quiet = true + else + break + end + ARGV.shift +end + +unless ARGV.size == 2 + abort "usage: #{$0} src destdir" +end +src, dest = ARGV + +require 'fileutils' + +include FileUtils +unless respond_to?(:ln_sr) + def ln_sr(src, dest, target_directory: true, force: nil, noop: nil, verbose: nil) + options = "#{force ? 'f' : ''}#{target_directory ? '' : 'T'}" + dest = File.path(dest) + srcs = Array(src) + link = proc do |s, target_dir_p = true| + s = File.path(s) + if target_dir_p + d = File.join(destdirs = dest, File.basename(s)) + else + destdirs = File.dirname(d = dest) + end + destdirs = fu_split_path(File.realpath(destdirs)) + if fu_starting_path?(s) + srcdirs = fu_split_path((File.realdirpath(s) rescue File.expand_path(s))) + base = fu_relative_components_from(srcdirs, destdirs) + s = File.join(*base) + else + srcdirs = fu_clean_components(*fu_split_path(s)) + base = fu_relative_components_from(fu_split_path(Dir.pwd), destdirs) + while srcdirs.first&. == ".." and base.last&.!=("..") and !fu_starting_path?(base.last) + srcdirs.shift + base.pop + end + s = File.join(*base, *srcdirs) + end + fu_output_message "ln -s#{options} #{s} #{d}" if verbose + next if noop + remove_file d, true if force + File.symlink s, d + end + case srcs.size + when 0 + when 1 + link[srcs[0], target_directory && File.directory?(dest)] + else + srcs.each(&link) + end + end + + def fu_split_path(path) + path = File.path(path) + list = [] + until (parent, base = File.split(path); parent == path or parent == ".") + list << base + path = parent + end + list << path + list.reverse! + end + + def fu_relative_components_from(target, base) #:nodoc: + i = 0 + while target[i]&.== base[i] + i += 1 + end + Array.new(base.size-i, '..').concat(target[i..-1]) + end + + def fu_clean_components(*comp) + comp.shift while comp.first == "." + return comp if comp.empty? + clean = [comp.shift] + path = File.join(*clean, "") # ending with File::SEPARATOR + while c = comp.shift + if c == ".." and clean.last != ".." and !(fu_have_symlink? && File.symlink?(path)) + clean.pop + path.sub!(%r((?<=\A|/)[^/]+/\z), "") + else + clean << c + path << c << "/" + end + end + clean + end + + if fu_windows? + def fu_starting_path?(path) + path&.start_with?(%r(\w:|/)) + end + else + def fu_starting_path?(path) + path&.start_with?("/") + end + end +end + +if File.respond_to?(:symlink) + if quiet and File.identical?(src, dest) + exit + end + begin + ln_sr(src, dest, verbose: true, target_directory: target_directory, force: force, noop: noop) + rescue NotImplementedError, Errno::EPERM, Errno::EACCES + else + exit + end +end + +cp_r(src, dest) diff --git a/tool/lrama/LEGAL.md b/tool/lrama/LEGAL.md new file mode 100644 index 0000000000..a3ef848514 --- /dev/null +++ b/tool/lrama/LEGAL.md @@ -0,0 +1,12 @@ +# LEGAL NOTICE INFORMATION + +All the files in this distribution are covered under the MIT License except some files +mentioned below. + +## GNU General Public License version 3 + +These files are licensed under the GNU General Public License version 3 or later. See these files for more information. + +* template/bison/_yacc.h +* template/bison/yacc.c +* template/bison/yacc.h diff --git a/tool/lrama/MIT b/tool/lrama/MIT new file mode 100644 index 0000000000..b23d5210d5 --- /dev/null +++ b/tool/lrama/MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 Yuichiro Kaneko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md new file mode 100644 index 0000000000..f71118a913 --- /dev/null +++ b/tool/lrama/NEWS.md @@ -0,0 +1,1032 @@ +# NEWS for Lrama + +## Lrama 0.7.1 (2025-12-24) + +### Optimize IELR + +Optimized performance to a level that allows for IELR testing in practical applications. + +https://github.com/ruby/lrama/pull/595 +https://github.com/ruby/lrama/pull/605 +https://github.com/ruby/lrama/pull/685 +https://github.com/ruby/lrama/pull/700 + +### Introduce counterexamples timeout + +Counterexample searches can sometimes take a long time, so we've added a timeout to abort the process after a set period. The current limits are: + +* 10 seconds per case +* 120 seconds total (cumulative) + +Please note that these are hard-coded and cannot be modified by the user in the current version. + +https://github.com/ruby/lrama/pull/623 + +### Optimize Counterexamples + +Optimized counterexample search performance. + +https://github.com/ruby/lrama/pull/607 +https://github.com/ruby/lrama/pull/610 +https://github.com/ruby/lrama/pull/614 +https://github.com/ruby/lrama/pull/622 +https://github.com/ruby/lrama/pull/627 +https://github.com/ruby/lrama/pull/629 +https://github.com/ruby/lrama/pull/659 + +### Support parameterized rule's arguments include inline + +Allow to use %inline directive with Parameterized rules arguments. When an inline rule is used as an argument to a Parameterized rule, it expands inline at the point of use. + +```yacc +%rule %inline op : '+' + | '-' + ; +%% +operation : op? + ; +``` + +This expands to: + +```yacc +operation : /* empty */ + | '+' + | '-' + ; +``` + +https://github.com/ruby/lrama/pull/637 + +### Render conflicts of each state on output file + +Added token information for conflicts in the output file. +These information are useful when a state has many actions. + +``` +State 1 + + 4 class: keyword_class • tSTRING "end" + 5 $@1: ε • [tSTRING] + 7 class: keyword_class • $@1 tSTRING '!' "end" $@2 + 8 $@3: ε • [tSTRING] + 10 class: keyword_class • $@3 tSTRING '?' "end" $@4 + + Conflict on tSTRING. shift/reduce($@1) + Conflict on tSTRING. shift/reduce($@3) + Conflict on tSTRING. reduce($@1)/reduce($@3) + + tSTRING shift, and go to state 6 + + tSTRING reduce using rule 5 ($@1) + tSTRING reduce using rule 8 ($@3) + + $@1 go to state 7 + $@3 go to state 8 +``` + +https://github.com/ruby/lrama/pull/541 + +### Render the origin of conflicted tokens on output file + +For example, for the grammar file like below: + +``` +%% + +program: expr + ; + +expr: expr '+' expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where `"plus"` (`'+'`) look ahead tokens come from: + +``` +State 6 + + 2 expr: expr • "plus" expr + 2 | expr "plus" expr • ["end of file", "plus"] + + Conflict on "plus". shift/reduce(expr) + "plus" comes from state 0 goto by expr + "plus" comes from state 5 goto by expr +``` + +state 0 and state 5 look like below: + +``` +State 0 + + 0 $accept: • program "end of file" + 1 program: • expr + 2 expr: • expr "plus" expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + program go to state 2 + expr go to state 3 + +State 5 + + 2 expr: • expr "plus" expr + 2 | expr "plus" • expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + expr go to state 6 +``` + +https://github.com/ruby/lrama/pull/726 + +### Render precedences usage information on output file + +For example, for the grammar file like below: + +``` +%left tPLUS +%right tUPLUS + +%% + +program: expr ; + +expr: tUPLUS expr + | expr tPLUS expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where these precedences are used to resolve conflicts: + +``` +Precedences + precedence on "unary+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + precedence on "+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + state 8. Conflict between reduce by "expr -> expr tPLUS expr" and shift "+" resolved as reduce (%left "+"). +``` + +https://github.com/ruby/lrama/pull/741 + +### Add support for reporting Rule Usage Frequency + +Support to report rule usage frequency statistics for analyzing grammar characteristics. +Run `exe/lrama --report=rules` to show how frequently each terminal and non-terminal symbol is used in the grammar rules. + +```console +$ exe/lrama --report=rules sample/calc.y +Rule Usage Frequency + 0 tSTRING (4 times) + 1 keyword_class (3 times) + 2 keyword_end (3 times) + 3 '+' (2 times) + 4 string (2 times) + 5 string_1 (2 times) + 6 '!' (1 times) + 7 '-' (1 times) + 8 '?' (1 times) + 9 EOI (1 times) + 10 class (1 times) + 11 program (1 times) + 12 string_2 (1 times) + 13 strings_1 (1 times) + 14 strings_2 (1 times) + 15 tNUMBER (1 times) +``` + +This feature provides insights into the language characteristics by showing: +- Which symbols are most frequently used in the grammar +- The distribution of terminal and non-terminal usage +- Potential areas for grammar optimization or refactoring + +The frequency statistics help developers understand the grammar structure and can be useful for: +- Grammar complexity analysis +- Performance optimization hints +- Language design decisions +- Documentation and educational purposes + +https://github.com/ruby/lrama/pull/677 + +### Render Split States information on output file + +For example, for the grammar file like below: + +``` +%token a +%token b +%token c +%define lr.type ielr + +%precedence tLOWEST +%precedence a +%precedence tHIGHEST + +%% + +S: a A B a + | b A B b + ; + +A: a C D E + ; + +B: c + | // empty + ; + +C: D + ; + +D: a + ; + +E: a + | %prec tHIGHEST // empty + ; + +%% +``` + +Lrama generates output file which describes where which new states are created when IELR is enabled: + +``` +Split States + + State 19 is split from state 4 + State 20 is split from state 9 + State 21 is split from state 14 +``` + +https://github.com/ruby/lrama/pull/624 + +### Add ioption support to the Standard library + +Support `ioption` (inline option) rule, which is expanded inline without creating intermediate rules. + +Unlike the regular `option` rule that generates a separate rule, `ioption` directly expands at the point of use: + +```yacc +program: ioption(number) expr + +// Expanded inline to: + +program: expr + | number expr +``` + +This differs from the regular `option` which would generate: + +```yacc +program: option(number) expr + +// Expanded to: + +program: option_number expr +option_number: %empty + | number +``` + +The `ioption` rule provides more compact grammar generation by avoiding intermediate rule creation, which can be beneficial for reducing the parser's rule count and potentially improving performance. + +This feature is inspired by Menhir's standard library and maintains compatibility with [Menhir's `ioption` behavior](https://github.com/let-def/menhir/blob/e8ba7bef219acd355798072c42abbd11335ecf09/src/standard.mly#L33-L41). + +https://github.com/ruby/lrama/pull/666 + +### Syntax Diagrams + +Lrama provides an API for generating HTML syntax diagrams. These visual diagrams are highly useful as grammar development tools and can also serve as a form of automatic self-documentation. + + + +If you use syntax diagrams, you add `--diagram` option. + +```console +$ exe/lrama --diagram sample.y +``` + +https://github.com/ruby/lrama/pull/523 + +### Support `--profile` option + +You can profile parser generation process without modification for Lrama source code. +Currently `--profile=call-stack` and `--profile=memory` are supported. + +```console +$ exe/lrama --profile=call-stack sample/calc.y +``` + +Then "tmp/stackprof-cpu-myapp.dump" is generated. + +https://github.com/ruby/lrama/pull/525 + +### Add support Start-Symbol: `%start` + +https://github.com/ruby/lrama/pull/576 + +## Lrama 0.7.0 (2025-01-21) + +### [EXPERIMENTAL] Support the generation of the IELR(1) parser described in this paper + +Support the generation of the IELR(1) parser described in this paper. +https://www.sciencedirect.com/science/article/pii/S0167642309001191 + +If you use IELR(1) parser, you can write the following directive in your grammar file. + +```yacc +%define lr.type ielr +``` + +But, currently IELR(1) parser is experimental feature. If you find any bugs, please report it to us. Thank you. + +### Support `-t` option as same as `--debug` option + +Support to `-t` option as same as `--debug` option. +These options align with Bison behavior. So same as `--debug` option. + +### Trace only explicit rules + +Support to trace only explicit rules. +If you use `--trace=rules` option, it shows include mid-rule actions. If you want to show only explicit rules, you can use `--trace=only-explicit-rules` option. + +Example: + +```yacc +%{ +%} +%union { + int i; +} +%token <i> number +%type <i> program +%% +program : number { printf("%d", $1); } number { $$ = $1 + $3; } + ; +%% +``` + +Result of `--trace=rules`: + +```console +$ exe/lrama --trace=rules sample.y +Grammar rules: +$accept -> program YYEOF +$@1 -> ε +program -> number $@1 number +``` + +Result of `--trace=only-explicit-rules`: + +```console +$ exe/lrama --trace=explicit-rules sample.y +Grammar rules: +$accept -> program YYEOF +program -> number number +``` + +## Lrama 0.6.11 (2024-12-23) + +### Add support for %type declarations using %nterm in Nonterminal Symbols + +Allow to use `%nterm` in Nonterminal Symbols for `%type` declarations. + +```yacc +%nterm <type> nonterminal… +``` + +This directive is also supported for compatibility with Bison, and only non-terminal symbols are allowed. In other words, definitions like the following will result in an error: + +```yacc +%{ +// Prologue +%} + +%token EOI 0 "EOI" +%nterm EOI + +%% + +program: /* empty */ + ; +``` + +It show an error message like the following: + +```command +❯ exe/lrama nterm.y +nterm.y:6:7: symbol EOI redeclared as a nonterminal +%nterm EOI + ^^^ +``` + +## Lrama 0.6.10 (2024-09-11) + +### Aliased Named References for actions of RHS in Parameterizing rules + +Allow to use aliased named references for actions of RHS in Parameterizing rules. + +```yacc +%rule sum(X, Y): X[summand] '+' Y[addend] { $$ = $summand + $addend } + ; +``` + +https://github.com/ruby/lrama/pull/410 + + +### Named References for actions of RHS in Parameterizing rules caller side + +Allow to use named references for actions of RHS in Parameterizing rules caller side. + +```yacc +opt_nl: '\n'?[nl] <str> { $$ = $nl; } + ; +``` + +https://github.com/ruby/lrama/pull/414 + +### Widen the definable position of Parameterizing rules + +Allow to define Parameterizing rules in the middle of the grammar. + +```yacc +%rule defined_option(X): /* empty */ + | X + ; + +%% + +program : defined_option(number) <i> + | defined_list(number) <i> + ; + +%rule defined_list(X): /* empty */ /* <--- here */ + | defined_list(X) number + ; +``` + +https://github.com/ruby/lrama/pull/420 + +### Report unused terminal symbols + +Support to report unused terminal symbols. +Run `exe/lrama --report=terms` to show unused terminal symbols. + +```console +$ exe/lrama --report=terms sample/calc.y + 11 Unused Terms + 0 YYerror + 1 YYUNDEF + 2 '\\\\' + 3 '\\13' + 4 keyword_class2 + 5 tNUMBER + 6 tPLUS + 7 tMINUS + 8 tEQ + 9 tEQEQ + 10 '>' +``` +https://github.com/ruby/lrama/pull/439 + +### Report unused rules + +Support to report unused rules. +Run `exe/lrama --report=rules` to show unused rules. + +```console +$ exe/lrama --report=rules sample/calc.y + 3 Unused Rules + 0 unused_option + 1 unused_list + 2 unused_nonempty_list +``` + +https://github.com/ruby/lrama/pull/441 + +### Ensure compatibility with Bison for `%locations` directive + +Support `%locations` directive to ensure compatibility with Bison. +Change to `%locations` directive not set by default. + +https://github.com/ruby/lrama/pull/446 + +### Diagnostics report for parameterized rules redefine + +Support to warning redefined parameterized rules. +Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterized rules. + +```console +$ exe/lrama -W sample/calc.y +parameterized rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) +``` + +https://github.com/ruby/lrama/pull/448 + +### Support `-v` and `--verbose` option + +Support to `-v` and `--verbose` option. +These options align with Bison behavior. So same as '--report=state' option. + +https://github.com/ruby/lrama/pull/457 + +## Lrama 0.6.9 (2024-05-02) + +### Callee side tag specification of Parameterizing rules + +Allow to specify tag on callee side of Parameterizing rules. + +```yacc +%union { + int i; +} + +%rule with_tag(X) <i>: X { $$ = $1; } + ; +``` + +### Named References for actions of RHS in Parameterizing rules + +Allow to use named references for actions of RHS in Parameterizing rules. + +```yacc +%rule option(number): /* empty */ + | number { $$ = $number; } + ; +``` + +## Lrama 0.6.8 (2024-04-29) + +### Nested Parameterizing rules with tag + +Allow to nested Parameterizing rules with tag. + +```yacc +%union { + int i; +} + +%rule nested_nested_option(X): /* empty */ + | X + ; + +%rule nested_option(X): /* empty */ + | nested_nested_option(X) <i> + ; + +%rule option(Y): /* empty */ + | nested_option(Y) <i> + ; +``` + +## Lrama 0.6.7 (2024-04-28) + +### RHS of user defined Parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. + +User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined Parameterizing rules. + +``` +%rule with_word_seps(X): /* empty */ + | X ' '+ + ; +``` + +## Lrama 0.6.6 (2024-04-27) + +### Trace actions + +Support trace actions for debugging. +Run `exe/lrama --trace=actions` to show grammar rules with actions. + +```console +$ exe/lrama --trace=actions sample/calc.y +Grammar rules with actions: +$accept -> list, YYEOF {} +list -> ε {} +list -> list, LF {} +list -> list, expr, LF { printf("=> %d\n", $2); } +expr -> NUM {} +expr -> expr, '+', expr { $$ = $1 + $3; } +expr -> expr, '-', expr { $$ = $1 - $3; } +expr -> expr, '*', expr { $$ = $1 * $3; } +expr -> expr, '/', expr { $$ = $1 / $3; } +expr -> '(', expr, ')' { $$ = $2; } +``` + +### Inlining + +Support inlining for rules. +The `%inline` directive causes all references to symbols to be replaced with its definition. + +```yacc +%rule %inline op: PLUS { + } + | TIMES { * } + ; + +%% + +expr : number { $$ = $1; } + | expr op expr { $$ = $1 $2 $3; } + ; +``` + +as same as + +```yacc +expr : number { $$ = $1; } + | expr '+' expr { $$ = $1 + $3; } + | expr '*' expr { $$ = $1 * $3; } + ; +``` + +## Lrama 0.6.5 (2024-03-25) + +### Typed Midrule Actions + +User can specify the type of mid-rule action by tag (`<bar>`) instead of specifying it with in an action. + +```yacc +primary: k_case expr_value terms? + { + $<val>$ = p->case_labels; + p->case_labels = Qnil; + } + case_body + k_end + { + ... + } +``` + +can be written as + +```yacc +primary: k_case expr_value terms? + { + $$ = p->case_labels; + p->case_labels = Qnil; + }<val> + case_body + k_end + { + ... + } +``` + +`%destructor` for midrule action is invoked only when tag is specified by Typed Midrule Actions. + +Difference from Bison's Typed Midrule Actions is that tag is postposed in Lrama however it's preposed in Bison. + +Bison supports this feature from 3.1. + +## Lrama 0.6.4 (2024-03-22) + +### Parameterizing rules (preceded, terminated, delimited) + +Support `preceded`, `terminated` and `delimited` rules. + +```text +program: preceded(opening, X) + +// Expanded to + +program: preceded_opening_X +preceded_opening_X: opening X +``` + +``` +program: terminated(X, closing) + +// Expanded to + +program: terminated_X_closing +terminated_X_closing: X closing +``` + +``` +program: delimited(opening, X, closing) + +// Expanded to + +program: delimited_opening_X_closing +delimited_opening_X_closing: opening X closing +``` + +https://github.com/ruby/lrama/pull/382 + +### Support `%destructor` declaration + +User can set codes for freeing semantic value resources by using `%destructor`. +In general, these resources are freed by actions or after parsing. +However, if syntax error happens in parsing, these codes may not be executed. +Codes associated to `%destructor` are executed when semantic value is popped from the stack by an error. + +```yacc +%token <val1> NUM +%type <val2> expr2 +%type <val3> expr + +%destructor { + printf("destructor for val1: %d\n", $$); +} <val1> // printer for TAG + +%destructor { + printf("destructor for val2: %d\n", $$); +} <val2> + +%destructor { + printf("destructor for expr: %d\n", $$); +} expr // printer for symbol +``` + +Bison supports this feature from 1.75b. + +https://github.com/ruby/lrama/pull/385 + +## Lrama 0.6.3 (2024-02-15) + +### Bring Your Own Stack + +Provide functionalities for Bring Your Own Stack. + +Ruby’s Ripper library requires their own semantic value stack to manage Ruby Objects returned by user defined callback method. Currently Ripper uses semantic value stack (`yyvsa`) which is used by parser to manage Node. This hack introduces some limitation on Ripper. For example, Ripper can not execute semantic analysis depending on Node structure. + +Lrama introduces two features to support another semantic value stack by parser generator users. + +1. Callback entry points + +User can emulate semantic value stack by these callbacks. +Lrama provides these five callbacks. Registered functions are called when each event happens. For example %after-shift function is called when shift happens on original semantic value stack. + +* `%after-shift` function_name +* `%before-reduce` function_name +* `%after-reduce` function_name +* `%after-shift-error-token` function_name +* `%after-pop-stack` function_name + +2. `$:n` variable to access index of each grammar symbols + +User also needs to access semantic value of their stack in grammar action. `$:n` provides the way to access to it. `$:n` is translated to the minus index from the top of the stack. +For example + +```yacc +primary: k_if expr_value then compstmt if_tail k_end + { + /*% ripper: if!($:2, $:4, $:5) %*/ + /* $:2 = -5, $:4 = -3, $:5 = -2. */ + } +``` + +https://github.com/ruby/lrama/pull/367 + +## Lrama 0.6.2 (2024-01-27) + +### %no-stdlib directive + +If `%no-stdlib` directive is set, Lrama doesn't load Lrama standard library for +parameterized rules, stdlib.y. + +https://github.com/ruby/lrama/pull/344 + +## Lrama 0.6.1 (2024-01-13) + +### Nested Parameterizing rules + +Allow to pass an instantiated rule to other Parameterizing rules. + +```yacc +%rule constant(X) : X + ; + +%rule option(Y) : /* empty */ + | Y + ; + +%% + +program : option(constant(number)) // Nested rule + ; +%% +``` + +Allow to use nested Parameterizing rules when define Parameterizing rules. + +```yacc +%rule option(x) : /* empty */ + | X + ; + +%rule double(Y) : Y Y + ; + +%rule double_opt(A) : option(double(A)) // Nested rule + ; + +%% + +program : double_opt(number) + ; + +%% +``` + +https://github.com/ruby/lrama/pull/337 + +## Lrama 0.6.0 (2023-12-25) + +### User defined Parameterizing rules + +Allow to define Parameterizing rule by `%rule` directive. + +```yacc +%rule pair(X, Y): X Y { $$ = $1 + $2; } + ; + +%% + +program: stmt + ; + +stmt: pair(ODD, EVEN) <num> + | pair(EVEN, ODD) <num> + ; +``` + +https://github.com/ruby/lrama/pull/285 + +## Lrama 0.5.11 (2023-12-02) + +### Type specification of Parameterizing rules + +Allow to specify type of rules by specifying tag, `<i>` in below example. +Tag is post-modification style. + +```yacc +%union { + int i; +} + +%% + +program : option(number) <i> + | number_alias? <i> + ; +``` + +https://github.com/ruby/lrama/pull/272 + + +## Lrama 0.5.10 (2023-11-18) + +### Parameterizing rules (option, nonempty_list, list) + +Support function call style Parameterizing rules for `option`, `nonempty_list` and `list`. + +https://github.com/ruby/lrama/pull/197 + +### Parameterizing rules (separated_list) + +Support `separated_list` and `separated_nonempty_list` Parameterizing rules. + +```text +program: separated_list(',', number) + +// Expanded to + +program: separated_list_number +separated_list_number: ε +separated_list_number: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +``` +program: separated_nonempty_list(',', number) + +// Expanded to + +program: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +https://github.com/ruby/lrama/pull/204 + +## Lrama 0.5.9 (2023-11-05) + +### Parameterizing rules (suffix) + +Parameterizing rules are template of rules. +It's very common pattern to write "list" grammar rule like: + +```yacc +opt_args: /* none */ + | args + ; + +args: arg + | args arg +``` + +Lrama supports these suffixes: + +* `?`: option +* `+`: nonempty list +* `*`: list + +Idea of Parameterizing rules comes from Menhir LR(1) parser generator (https://gallium.inria.fr/~fpottier/menhir/manual.html#sec32). + +https://github.com/ruby/lrama/pull/181 + +## Lrama 0.5.7 (2023-10-23) + +### Racc parser + +Replace Lrama's parser from handwritten parser to LR parser generated by Racc. +Lrama uses `--embedded` option to generate LR parser because Racc is changed from default gem to bundled gem by Ruby 3.3 (https://github.com/ruby/lrama/pull/132). + +https://github.com/ruby/lrama/pull/62 + +## Lrama 0.5.4 (2023-08-17) + +### Runtime configuration for error recovery + +Make error recovery function configurable on runtime by two new macros. + +* `YYMAXREPAIR`: Expected to return max length of repair operations. `%parse-param` is passed to this function. +* `YYERROR_RECOVERY_ENABLED`: Expected to return bool value to determine error recovery is enabled or not. `%parse-param` is passed to this function. + +https://github.com/ruby/lrama/pull/74 + +## Lrama 0.5.3 (2023-08-05) + +### Error Recovery + +Support token insert base Error Recovery. +`-e` option is needed to generate parser with error recovery functions. + +https://github.com/ruby/lrama/pull/44 + +## Lrama 0.5.2 (2023-06-14) + +### Named References + +Instead of positional references like `$1` or `$$`, +named references allow to access to symbol by name. + +```yacc +primary: k_class cpath superclass bodystmt k_end + { + $primary = new_class($cpath, $bodystmt, $superclass); + } +``` + +Alias name can be declared. + +```yacc +expr[result]: expr[ex-left] '+' expr[ex.right] + { + $result = $[ex-left] + $[ex.right]; + } +``` + +Bison supports this feature from 2.5. + +### Add parse params to some macros and functions + +`%parse-param` are added to these macros and functions to remove ytab.sed hack from Ruby. + +* `YY_LOCATION_PRINT` +* `YY_SYMBOL_PRINT` +* `yy_stack_print` +* `YY_STACK_PRINT` +* `YY_REDUCE_PRINT` +* `yysyntax_error` + +https://github.com/ruby/lrama/pull/40 + +See also: https://github.com/ruby/ruby/pull/7807 + +## Lrama 0.5.0 (2023-05-17) + +### stdin mode + +When `-` is given as grammar file name, reads the grammar source from STDIN, and takes the next argument as the input file name. This mode helps pre-process a grammar source. + +https://github.com/ruby/lrama/pull/8 + +## Lrama 0.4.0 (2023-05-13) + +This is the first version migrated to Ruby. +This version generates "parse.c" compatible with Bison 3.8.2. diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama new file mode 100755 index 0000000000..710ac0cb96 --- /dev/null +++ b/tool/lrama/exe/lrama @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +$LOAD_PATH << File.join(__dir__, "../lib") +require "lrama" + +Lrama::Command.new(ARGV.dup).run diff --git a/tool/lrama/lib/lrama.rb b/tool/lrama/lib/lrama.rb new file mode 100644 index 0000000000..56ba0044d4 --- /dev/null +++ b/tool/lrama/lib/lrama.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative "lrama/bitmap" +require_relative "lrama/command" +require_relative "lrama/context" +require_relative "lrama/counterexamples" +require_relative "lrama/diagram" +require_relative "lrama/digraph" +require_relative "lrama/erb" +require_relative "lrama/grammar" +require_relative "lrama/lexer" +require_relative "lrama/logger" +require_relative "lrama/option_parser" +require_relative "lrama/options" +require_relative "lrama/output" +require_relative "lrama/parser" +require_relative "lrama/reporter" +require_relative "lrama/state" +require_relative "lrama/states" +require_relative "lrama/tracer" +require_relative "lrama/version" +require_relative "lrama/warnings" diff --git a/tool/lrama/lib/lrama/bitmap.rb b/tool/lrama/lib/lrama/bitmap.rb new file mode 100644 index 0000000000..88b255b012 --- /dev/null +++ b/tool/lrama/lib/lrama/bitmap.rb @@ -0,0 +1,47 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + module Bitmap + # @rbs! + # type bitmap = Integer + + # @rbs (Array[Integer] ary) -> bitmap + def self.from_array(ary) + bit = 0 + + ary.each do |int| + bit |= (1 << int) + end + + bit + end + + # @rbs (Integer int) -> bitmap + def self.from_integer(int) + 1 << int + end + + # @rbs (bitmap int) -> Array[Integer] + def self.to_array(int) + a = [] #: Array[Integer] + i = 0 + + len = int.bit_length + while i < len do + if int[i] == 1 + a << i + end + + i += 1 + end + + a + end + + # @rbs (bitmap int, Integer size) -> Array[bool] + def self.to_bool_array(int, size) + Array.new(size) { |i| int[i] == 1 } + end + end +end diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb new file mode 100644 index 0000000000..17aad1a1c1 --- /dev/null +++ b/tool/lrama/lib/lrama/command.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Lrama + class Command + LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__))) + STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y') + + def initialize(argv) + @logger = Lrama::Logger.new + @options = OptionParser.parse(argv) + @tracer = Tracer.new(STDERR, **@options.trace_opts) + @reporter = Reporter.new(**@options.report_opts) + @warnings = Warnings.new(@logger, @options.warnings) + rescue => e + abort format_error_message(e.message) + end + + def run + Lrama::Reporter::Profile::CallStack.report(@options.profile_opts[:call_stack]) do + Lrama::Reporter::Profile::Memory.report(@options.profile_opts[:memory]) do + execute_command_workflow + end + end + end + + private + + def execute_command_workflow + @tracer.enable_duration + text = read_input + grammar = build_grammar(text) + states, context = compute_status(grammar) + render_reports(states) if @options.report_file + @tracer.trace(grammar) + render_diagram(grammar) + render_output(context, grammar) + states.validate!(@logger) + @warnings.warn(grammar, states) + end + + def read_input + text = @options.y.read + @options.y.close unless @options.y == STDIN + text + end + + def build_grammar(text) + grammar = + Lrama::Parser.new(text, @options.grammar_file, @options.debug, @options.locations, @options.define).parse + merge_stdlib(grammar) + prepare_grammar(grammar) + grammar + rescue => e + raise e if @options.debug + abort format_error_message(e.message) + end + + def format_error_message(message) + return message unless Exception.to_tty? + + message.gsub(/.+/, "\e[1m\\&\e[m") + end + + def merge_stdlib(grammar) + return if grammar.no_stdlib + + stdlib_text = File.read(STDLIB_FILE_PATH) + stdlib_grammar = Lrama::Parser.new( + stdlib_text, + STDLIB_FILE_PATH, + @options.debug, + @options.locations, + @options.define, + ).parse + + grammar.prepend_parameterized_rules(stdlib_grammar.parameterized_rules) + end + + def prepare_grammar(grammar) + grammar.prepare + grammar.validate! + end + + def compute_status(grammar) + states = Lrama::States.new(grammar, @tracer) + states.compute + states.compute_ielr if grammar.ielr_defined? + [states, Lrama::Context.new(states)] + end + + def render_reports(states) + File.open(@options.report_file, "w+") do |f| + @reporter.report(f, states) + end + end + + def render_diagram(grammar) + return unless @options.diagram + + File.open(@options.diagram_file, "w+") do |f| + Lrama::Diagram.render(out: f, grammar: grammar) + end + end + + def render_output(context, grammar) + File.open(@options.outfile, "w+") do |f| + Lrama::Output.new( + out: f, + output_file_path: @options.outfile, + template_name: @options.skeleton, + grammar_file_path: @options.grammar_file, + header_file_path: @options.header_file, + context: context, + grammar: grammar, + error_recovery: @options.error_recovery, + ).render + end + end + end +end diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb new file mode 100644 index 0000000000..eb068c1b9e --- /dev/null +++ b/tool/lrama/lib/lrama/context.rb @@ -0,0 +1,497 @@ +# frozen_string_literal: true + +require_relative "tracer/duration" + +module Lrama + # This is passed to a template + class Context + include Tracer::Duration + + ErrorActionNumber = -Float::INFINITY + BaseMin = -Float::INFINITY + + # TODO: It might be better to pass `states` to Output directly? + attr_reader :states, :yylast, :yypact_ninf, :yytable_ninf, :yydefact, :yydefgoto + + def initialize(states) + @states = states + @yydefact = nil + @yydefgoto = nil + # Array of array + @_actions = [] + + compute_tables + end + + # enum yytokentype + def yytokentype + @states.terms.reject do |term| + 0 < term.token_id && term.token_id < 128 + end.map do |term| + [term.id.s_value, term.token_id, term.display_name] + end.unshift(["YYEMPTY", -2, nil]) + end + + # enum yysymbol_kind_t + def yysymbol_kind_t + @states.symbols.map do |sym| + [sym.enum_name, sym.number, sym.comment] + end.unshift(["YYSYMBOL_YYEMPTY", -2, nil]) + end + + # State number of final (accepted) state + def yyfinal + @states.states.find do |state| + state.items.find do |item| + item.lhs.accept_symbol? && item.end_of_rule? + end + end.id + end + + # Number of terms + def yyntokens + @states.terms.count + end + + # Number of nterms + def yynnts + @states.nterms.count + end + + # Number of rules + def yynrules + @states.rules.count + end + + # Number of states + def yynstates + @states.states.count + end + + # Last token number + def yymaxutok + @states.terms.map(&:token_id).max + end + + # YYTRANSLATE + # + # yytranslate is a mapping from token id to symbol number + def yytranslate + # 2 is YYSYMBOL_YYUNDEF + a = Array.new(yymaxutok, 2) + + @states.terms.each do |term| + a[term.token_id] = term.number + end + + return a + end + + def yytranslate_inverted + a = Array.new(@states.symbols.count, @states.undef_symbol.token_id) + + @states.terms.each do |term| + a[term.number] = term.token_id + end + + return a + end + + # Mapping from rule number to line number of the rule is defined. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyrline + a = [0] + + @states.rules.each do |rule| + a << rule.lineno + end + + return a + end + + # Mapping from symbol number to its name + def yytname + @states.symbols.sort_by(&:number).map do |sym| + sym.display_name + end + end + + def yypact + @base[0...yynstates] + end + + def yypgoto + @base[yynstates..-1] + end + + def yytable + @table + end + + def yycheck + @check + end + + def yystos + @states.states.map do |state| + state.accessing_symbol.number + end + end + + # Mapping from rule number to symbol number of LHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr1 + a = [0] + + @states.rules.each do |rule| + a << rule.lhs.number + end + + return a + end + + # Mapping from rule number to length of RHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr2 + a = [0] + + @states.rules.each do |rule| + a << rule.rhs.count + end + + return a + end + + private + + # Compute these + # + # See also: "src/tables.c" of Bison. + # + # * yydefact + # * yydefgoto + # * yypact and yypgoto + # * yytable + # * yycheck + # * yypact_ninf + # * yytable_ninf + def compute_tables + report_duration(:compute_yydefact) { compute_yydefact } + report_duration(:compute_yydefgoto) { compute_yydefgoto } + report_duration(:sort_actions) { sort_actions } + # debug_sorted_actions + report_duration(:compute_packed_table) { compute_packed_table } + end + + def vectors_count + @states.states.count + @states.nterms.count + end + + # In compressed table, rule 0 is appended as an error case + # and reduce is represented as minus number. + def rule_id_to_action_number(rule_id) + (rule_id + 1) * -1 + end + + # Symbol number is assigned to term first then nterm. + # This method calculates sequence_number for nterm. + def nterm_number_to_sequence_number(nterm_number) + nterm_number - @states.terms.count + end + + # Vector is states + nterms + def nterm_number_to_vector_number(nterm_number) + @states.states.count + (nterm_number - @states.terms.count) + end + + def compute_yydefact + # Default action (shift/reduce/error) for each state. + # Index is state id, value is `rule id + 1` of a default reduction. + @yydefact = Array.new(@states.states.count, 0) + + @states.states.each do |state| + # Action number means + # + # * number = 0, default action + # * number = -Float::INFINITY, error by %nonassoc + # * number > 0, shift then move to state "number" + # * number < 0, reduce by "-number" rule. Rule "number" is already added by 1. + actions = Array.new(@states.terms.count, 0) + + if state.reduces.map(&:selected_look_ahead).any? {|la| !la.empty? } + # Iterate reduces with reverse order so that first rule is used. + state.reduces.reverse_each do |reduce| + reduce.look_ahead.each do |term| + actions[term.number] = rule_id_to_action_number(reduce.rule.id) + end + end + end + + # Shift is selected when S/R conflict exists. + state.selected_term_transitions.each do |shift| + actions[shift.next_sym.number] = shift.to_state.id + end + + state.resolved_conflicts.select do |conflict| + conflict.which == :error + end.each do |conflict| + actions[conflict.symbol.number] = ErrorActionNumber + end + + # If default_reduction_rule, replace default_reduction_rule in + # actions with zero. + if state.default_reduction_rule + actions.map! do |e| + if e == rule_id_to_action_number(state.default_reduction_rule.id) + 0 + else + e + end + end + end + + # If no default_reduction_rule, default behavior is an + # error then replace ErrorActionNumber with zero. + unless state.default_reduction_rule + actions.map! do |e| + if e == ErrorActionNumber + 0 + else + e + end + end + end + + s = actions.each_with_index.map do |n, i| + [i, n] + end.reject do |i, n| + # Remove default_reduction_rule entries + n == 0 + end + + if s.count != 0 + # Entry of @_actions is an array of + # + # * State id + # * Array of tuple, [from, to] where from is term number and to is action. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [state.id, s, s.count, s.last[0] - s.first[0] + 1] + end + + @yydefact[state.id] = state.default_reduction_rule ? state.default_reduction_rule.id + 1 : 0 + end + end + + def compute_yydefgoto + # Default GOTO (nterm transition) for each nterm. + # Index is sequence number of nterm, value is state id + # of a default nterm transition destination. + @yydefgoto = Array.new(@states.nterms.count, 0) + # Mapping from nterm to next_states + nterm_to_to_states = {} + + @states.states.each do |state| + state.nterm_transitions.each do |goto| + key = goto.next_sym + nterm_to_to_states[key] ||= [] + nterm_to_to_states[key] << [state, goto.to_state] # [from_state, to_state] + end + end + + @states.nterms.each do |nterm| + if (states = nterm_to_to_states[nterm]) + default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first + default_goto = default_state.id + not_default_gotos = [] + states.each do |from_state, to_state| + next if to_state.id == default_goto + not_default_gotos << [from_state.id, to_state.id] + end + else + default_goto = 0 + not_default_gotos = [] + end + + k = nterm_number_to_sequence_number(nterm.number) + @yydefgoto[k] = default_goto + + if not_default_gotos.count != 0 + v = nterm_number_to_vector_number(nterm.number) + + # Entry of @_actions is an array of + # + # * Nterm number as vector number + # * Array of tuple, [from, to] where from is state number and to is state number. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [v, not_default_gotos, not_default_gotos.count, not_default_gotos.last[0] - not_default_gotos.first[0] + 1] + end + end + end + + def sort_actions + # This is not same with #sort_actions + # + # @sorted_actions = @_actions.sort_by do |_, _, count, width| + # [-width, -count] + # end + + @sorted_actions = [] + + @_actions.each do |action| + if @sorted_actions.empty? + @sorted_actions << action + next + end + + j = @sorted_actions.count - 1 + _state_id, _froms_and_tos, count, width = action + + while (j >= 0) do + case + when @sorted_actions[j][3] < width + j -= 1 + when @sorted_actions[j][3] == width && @sorted_actions[j][2] < count + j -= 1 + else + break + end + end + + @sorted_actions.insert(j + 1, action) + end + end + + def debug_sorted_actions + ary = Array.new + @sorted_actions.each do |state_id, froms_and_tos, count, width| + ary[state_id] = [state_id, froms_and_tos, count, width] + end + + print sprintf("table_print:\n\n") + + print sprintf("order [\n") + vectors_count.times do |i| + print sprintf("%d, ", @sorted_actions[i] ? @sorted_actions[i][0] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("width [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][3] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("tally [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][2] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + end + + def compute_packed_table + # yypact and yypgoto + @base = Array.new(vectors_count, BaseMin) + # yytable + @table = [] + # yycheck + @check = [] + # Key is froms_and_tos, value is index position + pushed = {} + used_res = {} + lowzero = 0 + high = 0 + + @sorted_actions.each do |state_id, froms_and_tos, _, _| + if (res = pushed[froms_and_tos]) + @base[state_id] = res + next + end + + res = lowzero - froms_and_tos.first[0] + + # Find the smallest `res` such that `@table[res + from]` is empty for all `from` in `froms_and_tos` + while true do + advanced = false + + while used_res[res] + res += 1 + advanced = true + end + + froms_and_tos.each do |from, to| + while @table[res + from] + res += 1 + advanced = true + end + end + + unless advanced + # no advance means that the current `res` satisfies the condition + break + end + end + + loc = 0 + + froms_and_tos.each do |from, to| + loc = res + from + + @table[loc] = to + @check[loc] = from + end + + while (@table[lowzero]) do + lowzero += 1 + end + + high = loc if high < loc + + @base[state_id] = res + pushed[froms_and_tos] = res + used_res[res] = true + end + + @yylast = high + + # replace_ninf + @yypact_ninf = (@base.reject {|i| i == BaseMin } + [0]).min - 1 + @base.map! do |i| + case i + when BaseMin + @yypact_ninf + else + i + end + end + + @yytable_ninf = (@table.compact.reject {|i| i == ErrorActionNumber } + [0]).min - 1 + @table.map! do |i| + case i + when nil + 0 + when ErrorActionNumber + @yytable_ninf + else + i + end + end + + @check.map! do |i| + case i + when nil + -1 + else + i + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb new file mode 100644 index 0000000000..60d830d048 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -0,0 +1,426 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "set" +require "timeout" + +require_relative "counterexamples/derivation" +require_relative "counterexamples/example" +require_relative "counterexamples/node" +require_relative "counterexamples/path" +require_relative "counterexamples/state_item" +require_relative "counterexamples/triple" + +module Lrama + # See: https://www.cs.cornell.edu/andru/papers/cupex/cupex.pdf + # 4. Constructing Nonunifying Counterexamples + class Counterexamples + PathSearchTimeLimit = 10 # 10 sec + CumulativeTimeLimit = 120 # 120 sec + + # @rbs! + # @states: States + # @iterate_count: Integer + # @total_duration: Float + # @exceed_cumulative_time_limit: bool + # @state_items: Hash[[State, State::Item], StateItem] + # @triples: Hash[Integer, Triple] + # @transitions: Hash[[StateItem, Grammar::Symbol], StateItem] + # @reverse_transitions: Hash[[StateItem, Grammar::Symbol], Set[StateItem]] + # @productions: Hash[StateItem, Set[StateItem]] + # @reverse_productions: Hash[[State, Grammar::Symbol], Set[StateItem]] # Grammar::Symbol is nterm + # @state_item_shift: Integer + + attr_reader :transitions #: Hash[[StateItem, Grammar::Symbol], StateItem] + attr_reader :productions #: Hash[StateItem, Set[StateItem]] + + # @rbs (States states) -> void + def initialize(states) + @states = states + @iterate_count = 0 + @total_duration = 0 + @exceed_cumulative_time_limit = false + @triples = {} + setup_state_items + setup_transitions + setup_productions + end + + # @rbs () -> "#<Counterexamples>" + def to_s + "#<Counterexamples>" + end + alias :inspect :to_s + + # @rbs (State conflict_state) -> Array[Example] + def compute(conflict_state) + conflict_state.conflicts.flat_map do |conflict| + # Check cumulative time limit for not each path search method call but each conflict + # to avoid one of example's path to be nil. + next if @exceed_cumulative_time_limit + + case conflict.type + when :shift_reduce + # @type var conflict: State::ShiftReduceConflict + shift_reduce_example(conflict_state, conflict) + when :reduce_reduce + # @type var conflict: State::ReduceReduceConflict + reduce_reduce_examples(conflict_state, conflict) + end + rescue Timeout::Error => e + STDERR.puts "Counterexamples calculation for state #{conflict_state.id} #{e.message} with #{@iterate_count} iteration" + increment_total_duration(PathSearchTimeLimit) + nil + end.compact + end + + private + + # @rbs (State state, State::Item item) -> StateItem + def get_state_item(state, item) + @state_items[[state, item]] + end + + # For optimization, create all StateItem in advance + # and use them by fetching an instance from `@state_items`. + # Do not create new StateItem instance in the shortest path search process + # to avoid miss hash lookup. + # + # @rbs () -> void + def setup_state_items + @state_items = {} + count = 0 + + @states.states.each do |state| + state.items.each do |item| + @state_items[[state, item]] = StateItem.new(count, state, item) + count += 1 + end + end + + @state_item_shift = Math.log(count, 2).ceil + end + + # @rbs () -> void + def setup_transitions + @transitions = {} + @reverse_transitions = {} + + @states.states.each do |src_state| + trans = {} #: Hash[Grammar::Symbol, State] + + src_state.transitions.each do |transition| + trans[transition.next_sym] = transition.to_state + end + + src_state.items.each do |src_item| + next if src_item.end_of_rule? + sym = src_item.next_sym + dest_state = trans[sym] + + dest_state.kernels.each do |dest_item| + next unless (src_item.rule == dest_item.rule) && (src_item.position + 1 == dest_item.position) + src_state_item = get_state_item(src_state, src_item) + dest_state_item = get_state_item(dest_state, dest_item) + + @transitions[[src_state_item, sym]] = dest_state_item + + # @type var key: [StateItem, Grammar::Symbol] + key = [dest_state_item, sym] + @reverse_transitions[key] ||= Set.new + @reverse_transitions[key] << src_state_item + end + end + end + end + + # @rbs () -> void + def setup_productions + @productions = {} + @reverse_productions = {} + + @states.states.each do |state| + # Grammar::Symbol is LHS + h = {} #: Hash[Grammar::Symbol, Set[StateItem]] + + state.closure.each do |item| + sym = item.lhs + + h[sym] ||= Set.new + h[sym] << get_state_item(state, item) + end + + state.items.each do |item| + next if item.end_of_rule? + next if item.next_sym.term? + + sym = item.next_sym + state_item = get_state_item(state, item) + @productions[state_item] = h[sym] + + # @type var key: [State, Grammar::Symbol] + key = [state, sym] + @reverse_productions[key] ||= Set.new + @reverse_productions[key] << state_item + end + end + end + + # For optimization, use same Triple if it's already created. + # Do not create new Triple instance anywhere else + # to avoid miss hash lookup. + # + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> Triple + def get_triple(state_item, precise_lookahead_set) + key = (precise_lookahead_set << @state_item_shift) | state_item.id + @triples[key] ||= Triple.new(state_item, precise_lookahead_set) + end + + # @rbs (State conflict_state, State::ShiftReduceConflict conflict) -> Example + def shift_reduce_example(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + # @type var shift_conflict_item: ::Lrama::State::Item + shift_conflict_item = conflict_state.items.find { |item| item.next_sym == conflict_symbol } + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) + end + path1 = with_timeout("#find_shift_conflict_shortest_path:") do + find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + end + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + # @rbs (State conflict_state, State::ReduceReduceConflict conflict) -> Example + def reduce_reduce_examples(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + path1 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) + end + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + end + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + # @rbs (Array[StateItem]? reduce_state_items, State conflict_state, State::Item conflict_item) -> Array[StateItem] + def find_shift_conflict_shortest_path(reduce_state_items, conflict_state, conflict_item) + time1 = Time.now.to_f + @iterate_count = 0 + + target_state_item = get_state_item(conflict_state, conflict_item) + result = [target_state_item] + reversed_state_items = reduce_state_items.to_a.reverse + # Index for state_item + i = 0 + + while (state_item = reversed_state_items[i]) + # Index for prev_state_item + j = i + 1 + _j = j + + while (prev_state_item = reversed_state_items[j]) + if prev_state_item.type == :production + j += 1 + else + break + end + end + + if target_state_item == state_item || target_state_item.item.start_item? + result.concat( + reversed_state_items[_j..-1] #: Array[StateItem] + ) + break + end + + if target_state_item.type == :production + queue = [] #: Array[Node[StateItem]] + queue << Node.new(target_state_item, nil) + + # Find reverse production + while (sis = queue.shift) + @iterate_count += 1 + si = sis.elem + + # Reach to start state + if si.item.start_item? + a = Node.to_a(sis).reverse + a.shift + result.concat(a) + target_state_item = si + break + end + + if si.type == :production + # @type var key: [State, Grammar::Symbol] + key = [si.state, si.item.lhs] + @reverse_productions[key].each do |state_item| + queue << Node.new(state_item, sis) + end + else + # @type var key: [StateItem, Grammar::Symbol] + key = [si, si.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item&.state + a = Node.to_a(sis).reverse + a.shift + result.concat(a) + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + queue.clear + break + end + end + end + else + # Find reverse transition + # @type var key: [StateItem, Grammar::Symbol] + key = [target_state_item, target_state_item.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item&.state + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + break + end + end + end + + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "find_shift_conflict_shortest_path #{@iterate_count} iteration", duration) + end + + result.reverse + end + + # @rbs (StateItem target) -> Set[StateItem] + def reachable_state_items(target) + result = Set.new + queue = [target] + + while (state_item = queue.shift) + next if result.include?(state_item) + result << state_item + + @reverse_transitions[[state_item, state_item.item.previous_sym]]&.each do |prev_state_item| + queue << prev_state_item + end + + if state_item.item.beginning_of_rule? + @reverse_productions[[state_item.state, state_item.item.lhs]]&.each do |si| + queue << si + end + end + end + + result + end + + # @rbs (State conflict_state, State::Item conflict_reduce_item, Grammar::Symbol conflict_term) -> ::Array[StateItem]? + def shortest_path(conflict_state, conflict_reduce_item, conflict_term) + time1 = Time.now.to_f + @iterate_count = 0 + + queue = [] #: Array[[Triple, Path]] + visited = {} #: Hash[Triple, true] + start_state = @states.states.first #: Lrama::State + conflict_term_bit = Bitmap::from_integer(conflict_term.number) + raise "BUG: Start state should be just one kernel." if start_state.kernels.count != 1 + reachable = reachable_state_items(get_state_item(conflict_state, conflict_reduce_item)) + start = get_triple(get_state_item(start_state, start_state.kernels.first), Bitmap::from_integer(@states.eof_symbol.number)) + + queue << [start, Path.new(start.state_item, nil)] + + while (triple, path = queue.shift) + @iterate_count += 1 + + # Found + if (triple.state == conflict_state) && (triple.item == conflict_reduce_item) && (triple.l & conflict_term_bit != 0) + state_items = [path.state_item] + + while (path = path.parent) + state_items << path.state_item + end + + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "shortest_path #{@iterate_count} iteration", duration) + end + + return state_items.reverse + end + + # transition + next_state_item = @transitions[[triple.state_item, triple.item.next_sym]] + if next_state_item && reachable.include?(next_state_item) + # @type var t: Triple + t = get_triple(next_state_item, triple.l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end + end + + # production step + @productions[triple.state_item]&.each do |si| + next unless reachable.include?(si) + + l = follow_l(triple.item, triple.l) + # @type var t: Triple + t = get_triple(si, l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end + end + end + + return nil + end + + # @rbs (State::Item item, Bitmap::bitmap current_l) -> Bitmap::bitmap + def follow_l(item, current_l) + # 1. follow_L (A -> X1 ... Xn-1 • Xn) = L + # 2. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = {Xk+2} if Xk+2 is a terminal + # 3. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) if Xk+2 is a nonnullable nonterminal + # 4. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) + follow_L (A -> X1 ... Xk+1 • Xk+2 ... Xn) if Xk+2 is a nullable nonterminal + case + when item.number_of_rest_symbols == 1 + current_l + when item.next_next_sym.term? + item.next_next_sym.number_bitmap + when !item.next_next_sym.nullable + item.next_next_sym.first_set_bitmap + else + item.next_next_sym.first_set_bitmap | follow_l(item.new_by_next_position, current_l) + end + end + + # @rbs [T] (String message) { -> T } -> T + def with_timeout(message) + Timeout.timeout(PathSearchTimeLimit, Timeout::Error, message + " timeout of #{PathSearchTimeLimit} sec exceeded") do + yield + end + end + + # @rbs (Float|Integer duration) -> void + def increment_total_duration(duration) + @total_duration += duration + + if !@exceed_cumulative_time_limit && @total_duration > CumulativeTimeLimit + @exceed_cumulative_time_limit = true + STDERR.puts "CumulativeTimeLimit #{CumulativeTimeLimit} sec exceeded then skip following Counterexamples calculation" + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/derivation.rb b/tool/lrama/lib/lrama/counterexamples/derivation.rb new file mode 100644 index 0000000000..a2b74767a9 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/derivation.rb @@ -0,0 +1,76 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Derivation + # @rbs! + # @item: State::Item + # @left: Derivation? + + attr_reader :item #: State::Item + attr_reader :left #: Derivation? + attr_accessor :right #: Derivation? + + # @rbs (State::Item item, Derivation? left) -> void + def initialize(item, left) + @item = item + @left = left + end + + # @rbs () -> ::String + def to_s + "#<Derivation(#{item.display_name})>" + end + alias :inspect :to_s + + # @rbs () -> Array[String] + def render_strings_for_report + result = [] #: Array[String] + _render_for_report(self, 0, result, 0) + result.map(&:rstrip) + end + + # @rbs () -> String + def render_for_report + render_strings_for_report.join("\n") + end + + private + + # @rbs (Derivation derivation, Integer offset, Array[String] strings, Integer index) -> Integer + def _render_for_report(derivation, offset, strings, index) + item = derivation.item + if strings[index] + strings[index] << " " * (offset - strings[index].length) + else + strings[index] = " " * offset + end + str = strings[index] + str << "#{item.rule_id}: #{item.symbols_before_dot.map(&:display_name).join(" ")} " + + if derivation.left + len = str.length + str << "#{item.next_sym.display_name}" + length = _render_for_report(derivation.left, len, strings, index + 1) + # I want String#ljust! + str << " " * (length - str.length) if length > str.length + else + str << " • #{item.symbols_after_dot.map(&:display_name).join(" ")} " + return str.length + end + + if derivation.right&.left + left = derivation.right&.left #: Derivation + length = _render_for_report(left, str.length, strings, index + 1) + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore + str << " " * (length - str.length) if length > str.length + elsif item.next_next_sym + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore + end + + return str.length + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/example.rb b/tool/lrama/lib/lrama/counterexamples/example.rb new file mode 100644 index 0000000000..c007f45af4 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/example.rb @@ -0,0 +1,154 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Example + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @path1: ::Array[StateItem] + # @path2: ::Array[StateItem] + # @conflict: State::conflict + # @conflict_symbol: Grammar::Symbol + # @counterexamples: Counterexamples + # @derivations1: Derivation + # @derivations2: Derivation + + attr_reader :path1 #: ::Array[StateItem] + attr_reader :path2 #: ::Array[StateItem] + attr_reader :conflict #: State::conflict + attr_reader :conflict_symbol #: Grammar::Symbol + + # path1 is shift conflict when S/R conflict + # path2 is always reduce conflict + # + # @rbs (Array[StateItem]? path1, Array[StateItem]? path2, State::conflict conflict, Grammar::Symbol conflict_symbol, Counterexamples counterexamples) -> void + def initialize(path1, path2, conflict, conflict_symbol, counterexamples) + @path1 = path1 + @path2 = path2 + @conflict = conflict + @conflict_symbol = conflict_symbol + @counterexamples = counterexamples + end + + # @rbs () -> (:shift_reduce | :reduce_reduce) + def type + @conflict.type + end + + # @rbs () -> State::Item + def path1_item + @path1.last.item + end + + # @rbs () -> State::Item + def path2_item + @path2.last.item + end + + # @rbs () -> Derivation + def derivations1 + @derivations1 ||= _derivations(path1) + end + + # @rbs () -> Derivation + def derivations2 + @derivations2 ||= _derivations(path2) + end + + private + + # @rbs (Array[StateItem] state_items) -> Derivation + def _derivations(state_items) + derivation = nil #: Derivation + current = :production + last_state_item = state_items.last #: StateItem + lookahead_sym = last_state_item.item.end_of_rule? ? @conflict_symbol : nil + + state_items.reverse_each do |si| + item = si.item + + case current + when :production + case si.type + when :start + derivation = Derivation.new(item, derivation) + current = :start + when :transition + derivation = Derivation.new(item, derivation) + current = :transition + when :production + derivation = Derivation.new(item, derivation) + current = :production + else + raise "Unexpected. #{si}" + end + + if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym) + si2 = @counterexamples.transitions[[si, item.next_sym]] + derivation2 = find_derivation_for_symbol(si2, lookahead_sym) + derivation.right = derivation2 # steep:ignore + lookahead_sym = nil + end + + when :transition + case si.type + when :start + derivation = Derivation.new(item, derivation) + current = :start + when :transition + # ignore + current = :transition + when :production + # ignore + current = :production + end + else + raise "BUG: Unknown #{current}" + end + + break if current == :start + end + + derivation + end + + # @rbs (StateItem state_item, Grammar::Symbol sym) -> Derivation? + def find_derivation_for_symbol(state_item, sym) + queue = [] #: Array[Array[StateItem]] + queue << [state_item] + + while (sis = queue.shift) + si = sis.last + next_sym = si.item.next_sym + + if next_sym == sym + derivation = nil + + sis.reverse_each do |si| + derivation = Derivation.new(si.item, derivation) + end + + return derivation + end + + if next_sym.nterm? && next_sym.first_set.include?(sym) + @counterexamples.productions[si].each do |next_si| + next if next_si.item.empty_rule? + next if sis.include?(next_si) + queue << (sis + [next_si]) + end + + if next_sym.nullable + next_si = @counterexamples.transitions[[si, next_sym]] + queue << (sis + [next_si]) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/node.rb b/tool/lrama/lib/lrama/counterexamples/node.rb new file mode 100644 index 0000000000..9214a0e7f1 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/node.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + # @rbs generic E < Object -- Type of an element + class Node + attr_reader :elem #: E + attr_reader :next_node #: Node[E]? + + # @rbs [E < Object] (Node[E] node) -> Array[E] + def self.to_a(node) + a = [] # steep:ignore UnannotatedEmptyCollection + + while (node) + a << node.elem + node = node.next_node + end + + a + end + + # @rbs (E elem, Node[E]? next_node) -> void + def initialize(elem, next_node) + @elem = elem + @next_node = next_node + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/path.rb b/tool/lrama/lib/lrama/counterexamples/path.rb new file mode 100644 index 0000000000..6b1325f73b --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/path.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Path + # @rbs! + # @state_item: StateItem + # @parent: Path? + + attr_reader :state_item #: StateItem + attr_reader :parent #: Path? + + # @rbs (StateItem state_item, Path? parent) -> void + def initialize(state_item, parent) + @state_item = state_item + @parent = parent + end + + # @rbs () -> ::String + def to_s + "#<Path>" + end + alias :inspect :to_s + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/state_item.rb b/tool/lrama/lib/lrama/counterexamples/state_item.rb new file mode 100644 index 0000000000..8c2481d793 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/state_item.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class StateItem + attr_reader :id #: Integer + attr_reader :state #: State + attr_reader :item #: State::Item + + # @rbs (Integer id, State state, State::Item item) -> void + def initialize(id, state, item) + @id = id + @state = state + @item = item + end + + # @rbs () -> (:start | :transition | :production) + def type + case + when item.start_item? + :start + when item.beginning_of_rule? + :production + else + :transition + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/triple.rb b/tool/lrama/lib/lrama/counterexamples/triple.rb new file mode 100644 index 0000000000..98fe051f53 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/triple.rb @@ -0,0 +1,41 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Triple + attr_reader :precise_lookahead_set #: Bitmap::bitmap + + alias :l :precise_lookahead_set + + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> void + def initialize(state_item, precise_lookahead_set) + @state_item = state_item + @precise_lookahead_set = precise_lookahead_set + end + + # @rbs () -> State + def state + @state_item.state + end + alias :s :state + + # @rbs () -> State::Item + def item + @state_item.item + end + alias :itm :item + + # @rbs () -> StateItem + def state_item + @state_item + end + + # @rbs () -> ::String + def inspect + "#{state.inspect}. #{item.display_name}. #{l.to_s(2)}" + end + alias :to_s :inspect + end + end +end diff --git a/tool/lrama/lib/lrama/diagram.rb b/tool/lrama/lib/lrama/diagram.rb new file mode 100644 index 0000000000..985808933f --- /dev/null +++ b/tool/lrama/lib/lrama/diagram.rb @@ -0,0 +1,77 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Diagram + class << self + # @rbs (IO out, Grammar grammar, String template_name) -> void + def render(out:, grammar:, template_name: 'diagram/diagram.html') + return unless require_railroad_diagrams + new(out: out, grammar: grammar, template_name: template_name).render + end + + # @rbs () -> bool + def require_railroad_diagrams + require "railroad_diagrams" + true + rescue LoadError + warn "railroad_diagrams is not installed. Please run `bundle install`." + false + end + end + + # @rbs (IO out, Grammar grammar, String template_name) -> void + def initialize(out:, grammar:, template_name: 'diagram/diagram.html') + @grammar = grammar + @out = out + @template_name = template_name + end + + # @rbs () -> void + def render + RailroadDiagrams::TextDiagram.set_formatting(RailroadDiagrams::TextDiagram::PARTS_UNICODE) + @out << ERB.render(template_file, output: self) + end + + # @rbs () -> string + def default_style + RailroadDiagrams::Style::default_style + end + + # @rbs () -> string + def diagrams + result = +'' + @grammar.unique_rule_s_values.each do |s_value| + diagrams = + @grammar.select_rules_by_s_value(s_value).map { |r| r.to_diagrams } + add_diagram( + s_value, + RailroadDiagrams::Diagram.new( + RailroadDiagrams::Choice.new(0, *diagrams), + ), + result + ) + end + result + end + + private + + # @rbs () -> string + def template_dir + File.expand_path('../../template', __dir__) + end + + # @rbs () -> string + def template_file + File.join(template_dir, @template_name) + end + + # @rbs (String name, RailroadDiagrams::Diagram diagram, String result) -> void + def add_diagram(name, diagram, result) + result << "\n<h2 class=\"diagram-header\">#{RailroadDiagrams.escape_html(name)}</h2>" + diagram.write_svg(result.method(:<<)) + result << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/digraph.rb b/tool/lrama/lib/lrama/digraph.rb new file mode 100644 index 0000000000..52865f52dd --- /dev/null +++ b/tool/lrama/lib/lrama/digraph.rb @@ -0,0 +1,104 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Digraph Algorithm of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # + # Digraph is an algorithm for graph data structure. + # The algorithm efficiently traverses SCC (Strongly Connected Component) of graph + # and merges nodes attributes within the same SCC. + # + # `compute_read_sets` and `compute_follow_sets` have the same structure. + # Graph of gotos and attributes of gotos are given then compute propagated attributes for each node. + # + # In the case of `compute_read_sets`: + # + # * Set of gotos is nodes of graph + # * `reads_relation` is edges of graph + # * `direct_read_sets` is nodes attributes + # + # In the case of `compute_follow_sets`: + # + # * Set of gotos is nodes of graph + # * `includes_relation` is edges of graph + # * `read_sets` is nodes attributes + # + # + # @rbs generic X < Object -- Type of a node + # @rbs generic Y < _Or -- Type of attribute sets assigned to a node which should support merge operation (#| method) + class Digraph + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # interface _Or + # def |: (self) -> self + # end + # @sets: Array[X] + # @relation: Hash[X, Array[X]] + # @base_function: Hash[X, Y] + # @stack: Array[X] + # @h: Hash[X, (Integer|Float)?] + # @result: Hash[X, Y] + + # @rbs sets: Array[X] -- Nodes of graph + # @rbs relation: Hash[X, Array[X]] -- Edges of graph + # @rbs base_function: Hash[X, Y] -- Attributes of nodes + # @rbs return: void + def initialize(sets, relation, base_function) + + # X in the paper + @sets = sets + + # R in the paper + @relation = relation + + # F' in the paper + @base_function = base_function + + # S in the paper + @stack = [] + + # N in the paper + @h = Hash.new(0) + + # F in the paper + @result = {} + end + + # @rbs () -> Hash[X, Y] + def compute + @sets.each do |x| + next if @h[x] != 0 + traverse(x) + end + + return @result + end + + private + + # @rbs (X x) -> void + def traverse(x) + @stack.push(x) + d = @stack.count + @h[x] = d + @result[x] = @base_function[x] # F x = F' x + + @relation[x]&.each do |y| + traverse(y) if @h[y] == 0 + @h[x] = [@h[x], @h[y]].min + @result[x] |= @result[y] # F x = F x + F y + end + + if @h[x] == d + while (z = @stack.pop) do + @h[z] = Float::INFINITY + break if z == x + @result[z] = @result[x] # F (Top of S) = F x + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/erb.rb b/tool/lrama/lib/lrama/erb.rb new file mode 100644 index 0000000000..8f8be54811 --- /dev/null +++ b/tool/lrama/lib/lrama/erb.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "erb" + +module Lrama + class ERB + # @rbs (String file, **untyped kwargs) -> String + def self.render(file, **kwargs) + new(file).render(**kwargs) + end + + # @rbs (String file) -> void + def initialize(file) + input = File.read(file) + if ::ERB.instance_method(:initialize).parameters.last.first == :key + @erb = ::ERB.new(input, trim_mode: '-') + else + @erb = ::ERB.new(input, nil, '-') # steep:ignore UnexpectedPositionalArgument + end + @erb.filename = file + end + + # @rbs (**untyped kwargs) -> String + def render(**kwargs) + @erb.result_with_hash(kwargs) + end + end +end diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb new file mode 100644 index 0000000000..95a80bb01c --- /dev/null +++ b/tool/lrama/lib/lrama/grammar.rb @@ -0,0 +1,603 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "grammar/auxiliary" +require_relative "grammar/binding" +require_relative "grammar/code" +require_relative "grammar/counter" +require_relative "grammar/destructor" +require_relative "grammar/error_token" +require_relative "grammar/inline" +require_relative "grammar/parameterized" +require_relative "grammar/percent_code" +require_relative "grammar/precedence" +require_relative "grammar/printer" +require_relative "grammar/reference" +require_relative "grammar/rule" +require_relative "grammar/rule_builder" +require_relative "grammar/symbol" +require_relative "grammar/symbols" +require_relative "grammar/type" +require_relative "grammar/union" +require_relative "lexer" + +module Lrama + # Grammar is the result of parsing an input grammar file + class Grammar + # @rbs! + # + # interface _DelegatedMethods + # def rules: () -> Array[Rule] + # def accept_symbol: () -> Grammar::Symbol + # def eof_symbol: () -> Grammar::Symbol + # def undef_symbol: () -> Grammar::Symbol + # def precedences: () -> Array[Precedence] + # + # # delegate to @symbols_resolver + # def symbols: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def ielr_defined?: () -> bool + # end + # + # include Symbols::Resolver::_DelegatedMethods + # + # @rule_counter: Counter + # @percent_codes: Array[PercentCode] + # @printers: Array[Printer] + # @destructors: Array[Destructor] + # @error_tokens: Array[ErrorToken] + # @symbols_resolver: Symbols::Resolver + # @types: Array[Type] + # @rule_builders: Array[RuleBuilder] + # @rules: Array[Rule] + # @sym_to_rules: Hash[Integer, Array[Rule]] + # @parameterized_resolver: Parameterized::Resolver + # @empty_symbol: Grammar::Symbol + # @eof_symbol: Grammar::Symbol + # @error_symbol: Grammar::Symbol + # @undef_symbol: Grammar::Symbol + # @accept_symbol: Grammar::Symbol + # @aux: Auxiliary + # @no_stdlib: bool + # @locations: bool + # @define: Hash[String, String] + # @required: bool + # @union: Union + # @precedences: Array[Precedence] + # @start_nterm: Lrama::Lexer::Token::Base? + + extend Forwardable + + attr_reader :percent_codes #: Array[PercentCode] + attr_reader :eof_symbol #: Grammar::Symbol + attr_reader :error_symbol #: Grammar::Symbol + attr_reader :undef_symbol #: Grammar::Symbol + attr_reader :accept_symbol #: Grammar::Symbol + attr_reader :aux #: Auxiliary + attr_reader :parameterized_resolver #: Parameterized::Resolver + attr_reader :precedences #: Array[Precedence] + attr_accessor :union #: Union + attr_accessor :expect #: Integer + attr_accessor :printers #: Array[Printer] + attr_accessor :error_tokens #: Array[ErrorToken] + attr_accessor :lex_param #: String + attr_accessor :parse_param #: String + attr_accessor :initial_action #: Grammar::Code::InitialActionCode + attr_accessor :after_shift #: Lexer::Token::Base + attr_accessor :before_reduce #: Lexer::Token::Base + attr_accessor :after_reduce #: Lexer::Token::Base + attr_accessor :after_shift_error_token #: Lexer::Token::Base + attr_accessor :after_pop_stack #: Lexer::Token::Base + attr_accessor :symbols_resolver #: Symbols::Resolver + attr_accessor :types #: Array[Type] + attr_accessor :rules #: Array[Rule] + attr_accessor :rule_builders #: Array[RuleBuilder] + attr_accessor :sym_to_rules #: Hash[Integer, Array[Rule]] + attr_accessor :no_stdlib #: bool + attr_accessor :locations #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :required #: bool + + def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value, + :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, + :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type, + :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number! + + # @rbs (Counter rule_counter, bool locations, Hash[String, String] define) -> void + def initialize(rule_counter, locations, define = {}) + @rule_counter = rule_counter + + # Code defined by "%code" + @percent_codes = [] + @printers = [] + @destructors = [] + @error_tokens = [] + @symbols_resolver = Grammar::Symbols::Resolver.new + @types = [] + @rule_builders = [] + @rules = [] + @sym_to_rules = {} + @parameterized_resolver = Parameterized::Resolver.new + @empty_symbol = nil + @eof_symbol = nil + @error_symbol = nil + @undef_symbol = nil + @accept_symbol = nil + @aux = Auxiliary.new + @no_stdlib = false + @locations = locations + @define = define + @required = false + @precedences = [] + @start_nterm = nil + + append_special_symbols + end + + # @rbs (Counter rule_counter, Counter midrule_action_counter) -> RuleBuilder + def create_rule_builder(rule_counter, midrule_action_counter) + RuleBuilder.new(rule_counter, midrule_action_counter, @parameterized_resolver) + end + + # @rbs (id: Lexer::Token::Base, code: Lexer::Token::UserCode) -> Array[PercentCode] + def add_percent_code(id:, code:) + @percent_codes << PercentCode.new(id.s_value, code.s_value) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Destructor] + def add_destructor(ident_or_tags:, token_code:, lineno:) + @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Printer] + def add_printer(ident_or_tags:, token_code:, lineno:) + @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[ErrorToken] + def add_error_token(ident_or_tags:, token_code:, lineno:) + @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> Array[Type] + def add_type(id:, tag:) + @types << Type.new(id: id, tag: tag) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_nonassoc(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :nonassoc, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_left(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :left, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_right(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :right, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_precedence(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :precedence, precedence: precedence, lineno: lineno)) + end + + # @rbs (Lrama::Lexer::Token::Base id) -> Lrama::Lexer::Token::Base + def set_start_nterm(id) + # When multiple `%start` directives are defined, Bison does not generate an error, + # whereas Lrama does generate an error. + # Related Bison's specification are + # refs: https://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html + if @start_nterm.nil? + @start_nterm = id + else + start = @start_nterm #: Lrama::Lexer::Token::Base + raise "Start non-terminal is already set to #{start.s_value} (line: #{start.first_line}). Cannot set to #{id.s_value} (line: #{id.first_line})." + end + end + + # @rbs (Grammar::Symbol sym, Precedence precedence) -> (Precedence | bot) + def set_precedence(sym, precedence) + @precedences << precedence + sym.precedence = precedence + end + + # @rbs (Grammar::Code::NoReferenceCode code, Integer lineno) -> Union + def set_union(code, lineno) + @union = Union.new(code: code, lineno: lineno) + end + + # @rbs (RuleBuilder builder) -> Array[RuleBuilder] + def add_rule_builder(builder) + @rule_builders << builder + end + + # @rbs (Parameterized::Rule rule) -> Array[Parameterized::Rule] + def add_parameterized_rule(rule) + @parameterized_resolver.add_rule(rule) + end + + # @rbs () -> Array[Parameterized::Rule] + def parameterized_rules + @parameterized_resolver.rules + end + + # @rbs (Array[Parameterized::Rule] rules) -> Array[Parameterized::Rule] + def prepend_parameterized_rules(rules) + @parameterized_resolver.rules = rules + @parameterized_resolver.rules + end + + # @rbs (Integer prologue_first_lineno) -> Integer + def prologue_first_lineno=(prologue_first_lineno) + @aux.prologue_first_lineno = prologue_first_lineno + end + + # @rbs (String prologue) -> String + def prologue=(prologue) + @aux.prologue = prologue + end + + # @rbs (Integer epilogue_first_lineno) -> Integer + def epilogue_first_lineno=(epilogue_first_lineno) + @aux.epilogue_first_lineno = epilogue_first_lineno + end + + # @rbs (String epilogue) -> String + def epilogue=(epilogue) + @aux.epilogue = epilogue + end + + # @rbs () -> void + def prepare + resolve_inline_rules + normalize_rules + collect_symbols + set_lhs_and_rhs + fill_default_precedence + fill_symbols + fill_sym_to_rules + sort_precedence + compute_nullable + compute_first_set + set_locations + end + + # TODO: More validation methods + # + # * Validation for no_declared_type_reference + # + # @rbs () -> void + def validate! + @symbols_resolver.validate! + validate_no_precedence_for_nterm! + validate_rule_lhs_is_nterm! + validate_duplicated_precedence! + end + + # @rbs (Grammar::Symbol sym) -> Array[Rule] + def find_rules_by_symbol!(sym) + find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found") + end + + # @rbs (Grammar::Symbol sym) -> Array[Rule]? + def find_rules_by_symbol(sym) + @sym_to_rules[sym.number] + end + + # @rbs (String s_value) -> Array[Rule] + def select_rules_by_s_value(s_value) + @rules.select {|rule| rule.lhs.id.s_value == s_value } + end + + # @rbs () -> Array[String] + def unique_rule_s_values + @rules.map {|rule| rule.lhs.id.s_value }.uniq + end + + # @rbs () -> bool + def ielr_defined? + @define.key?('lr.type') && @define['lr.type'] == 'ielr' + end + + private + + # @rbs () -> void + def sort_precedence + @precedences.sort_by! do |prec| + prec.symbol.number + end + @precedences.freeze + end + + # @rbs () -> Array[Grammar::Symbol] + def compute_nullable + @rules.each do |rule| + case + when rule.empty_rule? + rule.nullable = true + when rule.rhs.any?(&:term) + rule.nullable = false + else + # noop + end + end + + while true do + rs = @rules.select {|e| e.nullable.nil? } + nts = nterms.select {|e| e.nullable.nil? } + rule_count_1 = rs.count + nterm_count_1 = nts.count + + rs.each do |rule| + if rule.rhs.all?(&:nullable) + rule.nullable = true + end + end + + nts.each do |nterm| + find_rules_by_symbol!(nterm).each do |rule| + if rule.nullable + nterm.nullable = true + end + end + end + + rule_count_2 = @rules.count {|e| e.nullable.nil? } + nterm_count_2 = nterms.count {|e| e.nullable.nil? } + + if (rule_count_1 == rule_count_2) && (nterm_count_1 == nterm_count_2) + break + end + end + + rules.select {|r| r.nullable.nil? }.each do |rule| + rule.nullable = false + end + + nterms.select {|e| e.nullable.nil? }.each do |nterm| + nterm.nullable = false + end + end + + # @rbs () -> Array[Grammar::Symbol] + def compute_first_set + terms.each do |term| + term.first_set = Set.new([term]).freeze + term.first_set_bitmap = Lrama::Bitmap.from_array([term.number]) + end + + nterms.each do |nterm| + nterm.first_set = Set.new([]).freeze + nterm.first_set_bitmap = Lrama::Bitmap.from_array([]) + end + + while true do + changed = false + + @rules.each do |rule| + rule.rhs.each do |r| + if rule.lhs.first_set_bitmap | r.first_set_bitmap != rule.lhs.first_set_bitmap + changed = true + rule.lhs.first_set_bitmap = rule.lhs.first_set_bitmap | r.first_set_bitmap + end + + break unless r.nullable + end + end + + break unless changed + end + + nterms.each do |nterm| + nterm.first_set = Lrama::Bitmap.to_array(nterm.first_set_bitmap).map do |number| + find_symbol_by_number!(number) + end.to_set + end + end + + # @rbs () -> Array[RuleBuilder] + def setup_rules + @rule_builders.each do |builder| + builder.setup_rules + end + end + + # @rbs () -> Grammar::Symbol + def append_special_symbols + # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated + # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) + # term.number = -2 + # @empty_symbol = term + + # YYEOF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYEOF"), alias_name: "\"end of file\"", token_id: 0) + term.number = 0 + term.eof_symbol = true + @eof_symbol = term + + # YYerror + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYerror"), alias_name: "error") + term.number = 1 + term.error_symbol = true + @error_symbol = term + + # YYUNDEF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYUNDEF"), alias_name: "\"invalid token\"") + term.number = 2 + term.undef_symbol = true + @undef_symbol = term + + # $accept + term = add_nterm(id: Lrama::Lexer::Token::Ident.new(s_value: "$accept")) + term.accept_symbol = true + @accept_symbol = term + end + + # @rbs () -> void + def resolve_inline_rules + while @rule_builders.any?(&:has_inline_rules?) do + @rule_builders = @rule_builders.flat_map do |builder| + if builder.has_inline_rules? + Inline::Resolver.new(builder).resolve + else + builder + end + end + end + end + + # @rbs () -> void + def normalize_rules + add_accept_rule + setup_rules + @rule_builders.each do |builder| + builder.rules.each do |rule| + add_nterm(id: rule._lhs, tag: rule.lhs_tag) + @rules << rule + end + end + + nterms.freeze + @rules.sort_by!(&:id).freeze + end + + # Add $accept rule to the top of rules + def add_accept_rule + if @start_nterm + start = @start_nterm #: Lrama::Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [start, @eof_symbol.id], token_code: nil, lineno: start.line) + else + rule_builder = @rule_builders.first #: RuleBuilder + lineno = rule_builder ? rule_builder.line : 0 + lhs = rule_builder.lhs #: Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + end + end + + # Collect symbols from rules + # + # @rbs () -> void + def collect_symbols + @rules.flat_map(&:_rhs).each do |s| + case s + when Lrama::Lexer::Token::Char + add_term(id: s) + when Lrama::Lexer::Token::Base + # skip + else + raise "Unknown class: #{s}" + end + end + + terms.freeze + end + + # @rbs () -> void + def set_lhs_and_rhs + @rules.each do |rule| + rule.lhs = token_to_symbol(rule._lhs) if rule._lhs + + rule.rhs = rule._rhs.map do |t| + token_to_symbol(t) + end + end + end + + # Rule inherits precedence from the last term in RHS. + # + # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html + # + # @rbs () -> void + def fill_default_precedence + @rules.each do |rule| + # Explicitly specified precedence has the highest priority + next if rule.precedence_sym + + precedence_sym = nil + rule.rhs.each do |sym| + precedence_sym = sym if sym.term? + end + + rule.precedence_sym = precedence_sym + end + end + + # @rbs () -> Array[Grammar::Symbol] + def fill_symbols + fill_symbol_number + fill_nterm_type(@types) + fill_printer(@printers) + fill_destructor(@destructors) + fill_error_token(@error_tokens) + sort_by_number! + end + + # @rbs () -> Array[Rule] + def fill_sym_to_rules + @rules.each do |rule| + key = rule.lhs.number + @sym_to_rules[key] ||= [] + @sym_to_rules[key] << rule + end + end + + # @rbs () -> void + def validate_no_precedence_for_nterm! + errors = [] #: Array[String] + + nterms.each do |nterm| + next if nterm.precedence.nil? + + errors << "[BUG] Precedence #{nterm.name} (line: #{nterm.precedence.lineno}) is defined for nonterminal symbol (line: #{nterm.id.first_line}). Precedence can be defined for only terminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void + def validate_rule_lhs_is_nterm! + errors = [] #: Array[String] + + rules.each do |rule| + next if rule.lhs.nterm? + + errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is terminal symbol. It should be nonterminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # # @rbs () -> void + def validate_duplicated_precedence! + errors = [] #: Array[String] + seen = {} #: Hash[String, Precedence] + + precedences.each do |prec| + s_value = prec.s_value + if first = seen[s_value] + errors << "%#{prec.type} redeclaration for #{s_value} (line: #{prec.lineno}) previous declaration was %#{first.type} (line: #{first.lineno})" + else + seen[s_value] = prec + end + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void + def set_locations + @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? } + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/auxiliary.rb b/tool/lrama/lib/lrama/grammar/auxiliary.rb new file mode 100644 index 0000000000..76cfb74d4d --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/auxiliary.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # Grammar file information not used by States but by Output + class Auxiliary + attr_accessor :prologue_first_lineno #: Integer? + attr_accessor :prologue #: String? + attr_accessor :epilogue_first_lineno #: Integer? + attr_accessor :epilogue #: String? + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/binding.rb b/tool/lrama/lib/lrama/grammar/binding.rb new file mode 100644 index 0000000000..5940d153a9 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/binding.rb @@ -0,0 +1,79 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Binding + # @rbs @actual_args: Array[Lexer::Token::Base] + # @rbs @param_to_arg: Hash[String, Lexer::Token::Base] + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> void + def initialize(params, actual_args) + @actual_args = actual_args + @param_to_arg = build_param_to_arg(params, @actual_args) + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def resolve_symbol(sym) + return create_instantiate_rule(sym) if sym.is_a?(Lexer::Token::InstantiateRule) + find_arg_for_param(sym) + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def concatenated_args_str(token) + "#{token.rule_name}_#{format_args(token)}" + end + + private + + # @rbs (Lexer::Token::InstantiateRule sym) -> Lexer::Token::InstantiateRule + def create_instantiate_rule(sym) + Lrama::Lexer::Token::InstantiateRule.new( + s_value: sym.s_value, + location: sym.location, + args: resolve_args(sym.args), + lhs_tag: sym.lhs_tag + ) + end + + # @rbs (Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def resolve_args(args) + args.map { |arg| resolve_symbol(arg) } + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def find_arg_for_param(sym) + if (arg = @param_to_arg[sym.s_value]&.dup) + arg.alias_name = sym.alias_name + arg + else + sym + end + end + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> Hash[String, Lexer::Token::Base?] + def build_param_to_arg(params, actual_args) + params.zip(actual_args).map do |param, arg| + [param.s_value, arg] + end.to_h + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def format_args(token) + token_to_args_s_values(token).join('_') + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Array[String] + def token_to_args_s_values(token) + token.args.flat_map do |arg| + resolved = resolve_symbol(arg) + if resolved.is_a?(Lexer::Token::InstantiateRule) + [resolved.s_value] + resolved.args.map(&:s_value) + else + [resolved.s_value] + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb new file mode 100644 index 0000000000..f1b860eeba --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -0,0 +1,68 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "code/destructor_code" +require_relative "code/initial_action_code" +require_relative "code/no_reference_code" +require_relative "code/printer_code" +require_relative "code/rule_action" + +module Lrama + class Grammar + class Code + # @rbs! + # + # # delegated + # def s_value: -> String + # def line: -> Integer + # def column: -> Integer + # def references: -> Array[Lrama::Grammar::Reference] + + extend Forwardable + + def_delegators "token_code", :s_value, :line, :column, :references + + attr_reader :type #: ::Symbol + attr_reader :token_code #: Lexer::Token::UserCode + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode) -> void + def initialize(type:, token_code:) + @type = type + @token_code = token_code + end + + # @rbs (Code other) -> bool + def ==(other) + self.class == other.class && + self.type == other.type && + self.token_code == other.token_code + end + + # $$, $n, @$, @n are translated to C code + # + # @rbs () -> String + def translated_code + t_code = s_value.dup + + references.reverse_each do |ref| + first_column = ref.first_column + last_column = ref.last_column + + str = reference_to_c(ref) + + t_code[first_column...last_column] = str + end + + return t_code + end + + private + + # @rbs (Lrama::Grammar::Reference ref) -> bot + def reference_to_c(ref) + raise NotImplementedError.new("#reference_to_c is not implemented") + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb new file mode 100644 index 0000000000..d71b62e513 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class DestructorCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb new file mode 100644 index 0000000000..cb36041524 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class InitialActionCode < Code + private + + # * ($$) yylval + # * (@$) yylloc + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + "yylval" + when ref.type == :at && ref.name == "$" # @$ + "yylloc" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in initial_action." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in initial_action." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in initial_action." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in initial_action." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb new file mode 100644 index 0000000000..1d39919979 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class NoReferenceCode < Code + private + + # * ($$) error + # * (@$) error + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> bot + def reference_to_c(ref) + case + when ref.type == :dollar # $$, $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @$, @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:$, $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/printer_code.rb b/tool/lrama/lib/lrama/grammar/code/printer_code.rb new file mode 100644 index 0000000000..c6e25d5235 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/printer_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class PrinterCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/rule_action.rb b/tool/lrama/lib/lrama/grammar/code/rule_action.rb new file mode 100644 index 0000000000..e71e93e5a5 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/rule_action.rb @@ -0,0 +1,109 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class RuleAction < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @rule: Rule + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, rule: Rule) -> void + def initialize(type:, token_code:, rule:) + super(type: type, token_code: token_code) + @rule = rule + end + + private + + # * ($$) yyval + # * (@$) yyloc + # * ($:$) error + # * ($1) yyvsp[i] + # * (@1) yylsp[i] + # * ($:1) i - 1 + # + # + # Consider a rule like + # + # class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # + # For the semantic action of original rule: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 $2 $3 $4 $5 + # "Index for yyvsp" -4 -3 -2 -1 0 + # "$:n" $:1 $:2 $:3 $:4 $:5 + # "index of $:n" -5 -4 -3 -2 -1 + # + # + # For the first midrule action: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 + # "Index for yyvsp" 0 + # "$:n" $:1 + # + # @rbs (Reference ref) -> String + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + tag = ref.ex_tag || lhs.tag + raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag + "(yyval.#{tag.member})" + when ref.type == :at && ref.name == "$" # @$ + "(yyloc)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:$ is not supported" + when ref.type == :dollar # $n + i = -position_in_rhs + ref.index + tag = ref.ex_tag || rhs[ref.index - 1].tag + raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag + "(yyvsp[#{i}].#{tag.member})" + when ref.type == :at # @n + i = -position_in_rhs + ref.index + "(yylsp[#{i}])" + when ref.type == :index # $:n + i = -position_in_rhs + ref.index + "(#{i} - 1)" + else + raise "Unexpected. #{self}, #{ref}" + end + end + + # @rbs () -> Integer + def position_in_rhs + # If rule is not derived rule, User Code is only action at + # the end of rule RHS. In such case, the action is located on + # `@rule.rhs.count`. + @rule.position_in_original_rule_rhs || @rule.rhs.count + end + + # If this is midrule action, RHS is an RHS of the original rule. + # + # @rbs () -> Array[Grammar::Symbol] + def rhs + (@rule.original_rule || @rule).rhs + end + + # Unlike `rhs`, LHS is always an LHS of the rule. + # + # @rbs () -> Grammar::Symbol + def lhs + @rule.lhs + end + + # @rbs (Reference ref) -> bot + def raise_tag_not_found_error(ref) + raise "Tag is not specified for '$#{ref.value}' in '#{@rule.display_name}'" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/counter.rb b/tool/lrama/lib/lrama/grammar/counter.rb new file mode 100644 index 0000000000..ced934309d --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/counter.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Counter + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @number: Integer + + # @rbs (Integer number) -> void + def initialize(number) + @number = number + end + + # @rbs () -> Integer + def increment + n = @number + @number += 1 + n + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/destructor.rb b/tool/lrama/lib/lrama/grammar/destructor.rb new file mode 100644 index 0000000000..0ce8611e77 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/destructor.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Destructor + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident|Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::DestructorCode.new(type: :destructor, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/error_token.rb b/tool/lrama/lib/lrama/grammar/error_token.rb new file mode 100644 index 0000000000..9d9ed54ae2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/error_token.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class ErrorToken + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident | Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::PrinterCode.new(type: :error_token, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/inline.rb b/tool/lrama/lib/lrama/grammar/inline.rb new file mode 100644 index 0000000000..c02ab6002b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative 'inline/resolver' diff --git a/tool/lrama/lib/lrama/grammar/inline/resolver.rb b/tool/lrama/lib/lrama/grammar/inline/resolver.rb new file mode 100644 index 0000000000..aca689ccfb --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline/resolver.rb @@ -0,0 +1,80 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Inline + class Resolver + # @rbs (Lrama::Grammar::RuleBuilder rule_builder) -> void + def initialize(rule_builder) + @rule_builder = rule_builder + end + + # @rbs () -> Array[Lrama::Grammar::RuleBuilder] + def resolve + resolved_builders = [] #: Array[Lrama::Grammar::RuleBuilder] + @rule_builder.rhs.each_with_index do |token, i| + if (rule = @rule_builder.parameterized_resolver.find_inline(token)) + rule.rhs.each do |rhs| + builder = build_rule(rhs, token, i, rule) + resolved_builders << builder + end + break + end + end + resolved_builders + end + + private + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Lrama::Lexer::Token token, Integer index, Lrama::Grammar::Parameterized::Rule rule) -> Lrama::Grammar::RuleBuilder + def build_rule(rhs, token, index, rule) + builder = RuleBuilder.new( + @rule_builder.rule_counter, + @rule_builder.midrule_action_counter, + @rule_builder.parameterized_resolver, + lhs_tag: @rule_builder.lhs_tag + ) + resolve_rhs(builder, rhs, index, token, rule) + builder.lhs = @rule_builder.lhs + builder.line = @rule_builder.line + builder.precedence_sym = @rule_builder.precedence_sym + builder.user_code = replace_user_code(rhs, index) + builder + end + + # @rbs (Lrama::Grammar::RuleBuilder builder, Lrama::Grammar::Parameterized::Rhs rhs, Integer index, Lrama::Lexer::Token token, Lrama::Grammar::Parameterized::Rule rule) -> void + def resolve_rhs(builder, rhs, index, token, rule) + @rule_builder.rhs.each_with_index do |tok, i| + if i == index + rhs.symbols.each do |sym| + if token.is_a?(Lexer::Token::InstantiateRule) + bindings = Binding.new(rule.parameters, token.args) + builder.add_rhs(bindings.resolve_symbol(sym)) + else + builder.add_rhs(sym) + end + end + else + builder.add_rhs(tok) + end + end + end + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Integer index) -> Lrama::Lexer::Token::UserCode + def replace_user_code(rhs, index) + user_code = @rule_builder.user_code + return user_code if rhs.user_code.nil? || user_code.nil? + + code = user_code.s_value.gsub(/\$#{index + 1}/, rhs.user_code.s_value) + user_code.references.each do |ref| + next if ref.index.nil? || ref.index <= index # nil は $$ の場合 + code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (rhs.symbols.count - 1)}") + code = code.gsub(/@#{ref.index}/, "@#{ref.index + (rhs.symbols.count - 1)}") + end + Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized.rb b/tool/lrama/lib/lrama/grammar/parameterized.rb new file mode 100644 index 0000000000..48db3433f3 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'parameterized/resolver' +require_relative 'parameterized/rhs' +require_relative 'parameterized/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb new file mode 100644 index 0000000000..558f308190 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Resolver + attr_accessor :rules #: Array[Rule] + attr_accessor :created_lhs_list #: Array[Lexer::Token::Base] + + # @rbs () -> void + def initialize + @rules = [] + @created_lhs_list = [] + end + + # @rbs (Rule rule) -> Array[Rule] + def add_rule(rule) + @rules << rule + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Rule? + def find_rule(token) + select_rules(@rules, token).last + end + + # @rbs (Lexer::Token::Base token) -> Rule? + def find_inline(token) + @rules.reverse.find { |rule| rule.name == token.s_value && rule.inline? } + end + + # @rbs (String lhs_s_value) -> Lexer::Token::Base? + def created_lhs(lhs_s_value) + @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } + end + + # @rbs () -> Array[Rule] + def redefined_rules + @rules.select { |rule| @rules.count { |r| r.name == rule.name && r.required_parameters_count == rule.required_parameters_count } > 1 } + end + + private + + # @rbs (Array[Rule] rules, Lexer::Token::InstantiateRule token) -> Array[Rule] + def select_rules(rules, token) + rules = reject_inline_rules(rules) + rules = select_rules_by_name(rules, token.rule_name) + rules = rules.select { |rule| rule.required_parameters_count == token.args_count } + if rules.empty? + raise "Invalid number of arguments. `#{token.rule_name}`" + else + rules + end + end + + # @rbs (Array[Rule] rules) -> Array[Rule] + def reject_inline_rules(rules) + rules.reject(&:inline?) + end + + # @rbs (Array[Rule] rules, String rule_name) -> Array[Rule] + def select_rules_by_name(rules, rule_name) + rules = rules.select { |rule| rule.name == rule_name } + if rules.empty? + raise "Parameterized rule does not exist. `#{rule_name}`" + else + rules + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb new file mode 100644 index 0000000000..663de49100 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rhs + attr_accessor :symbols #: Array[Lexer::Token::Base] + attr_accessor :user_code #: Lexer::Token::UserCode? + attr_accessor :precedence_sym #: Grammar::Symbol? + + # @rbs () -> void + def initialize + @symbols = [] + @user_code = nil + @precedence_sym = nil + end + + # @rbs (Grammar::Binding bindings) -> Lexer::Token::UserCode? + def resolve_user_code(bindings) + return unless user_code + + resolved = Lexer::Token::UserCode.new(s_value: user_code.s_value, location: user_code.location) + var_to_arg = {} #: Hash[String, String] + symbols.each do |sym| + resolved_sym = bindings.resolve_symbol(sym) + if resolved_sym != sym + var_to_arg[sym.s_value] = resolved_sym.s_value + end + end + + var_to_arg.each do |var, arg| + resolved.references.each do |ref| + if ref.name == var + ref.name = arg + end + end + end + + return resolved + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rule.rb b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb new file mode 100644 index 0000000000..7048be3cff --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb @@ -0,0 +1,36 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rule + attr_reader :name #: String + attr_reader :parameters #: Array[Lexer::Token::Base] + attr_reader :rhs #: Array[Rhs] + attr_reader :required_parameters_count #: Integer + attr_reader :tag #: Lexer::Token::Tag? + + # @rbs (String name, Array[Lexer::Token::Base] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void + def initialize(name, parameters, rhs, tag: nil, is_inline: false) + @name = name + @parameters = parameters + @rhs = rhs + @tag = tag + @is_inline = is_inline + @required_parameters_count = parameters.count + end + + # @rbs () -> String + def to_s + "#{@name}(#{@parameters.map(&:s_value).join(', ')})" + end + + # @rbs () -> bool + def inline? + @is_inline + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/percent_code.rb b/tool/lrama/lib/lrama/grammar/percent_code.rb new file mode 100644 index 0000000000..9afb903056 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/percent_code.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class PercentCode + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @name: String + # @code: String + + attr_reader :name #: String + attr_reader :code #: String + + # @rbs (String name, String code) -> void + def initialize(name, code) + @name = name + @code = code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/precedence.rb b/tool/lrama/lib/lrama/grammar/precedence.rb new file mode 100644 index 0000000000..b4c6403372 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/precedence.rb @@ -0,0 +1,55 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Precedence < Struct.new(:type, :symbol, :precedence, :s_value, :lineno, keyword_init: true) + include Comparable + # @rbs! + # type type_enum = :left | :right | :nonassoc | :precedence + # + # attr_accessor type: type_enum + # attr_accessor symbol: Grammar::Symbol + # attr_accessor precedence: Integer + # attr_accessor s_value: String + # attr_accessor lineno: Integer + # + # def initialize: (?type: type_enum, ?symbol: Grammar::Symbol, ?precedence: Integer, ?s_value: ::String, ?lineno: Integer) -> void + + attr_reader :used_by_lalr #: Array[State::ResolvedConflict] + attr_reader :used_by_ielr #: Array[State::ResolvedConflict] + + # @rbs (Precedence other) -> Integer + def <=>(other) + self.precedence <=> other.precedence + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_lalr(resolved_conflict) + @used_by_lalr ||= [] #: Array[State::ResolvedConflict] + @used_by_lalr << resolved_conflict + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_ielr(resolved_conflict) + @used_by_ielr ||= [] #: Array[State::ResolvedConflict] + @used_by_ielr << resolved_conflict + end + + # @rbs () -> bool + def used_by? + used_by_lalr? || used_by_ielr? + end + + # @rbs () -> bool + def used_by_lalr? + !@used_by_lalr.nil? && !@used_by_lalr.empty? + end + + # @rbs () -> bool + def used_by_ielr? + !@used_by_ielr.nil? && !@used_by_ielr.empty? + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/printer.rb b/tool/lrama/lib/lrama/grammar/printer.rb new file mode 100644 index 0000000000..490fe701db --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/printer.rb @@ -0,0 +1,20 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Printer < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + # @rbs! + # attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag] + # attr_accessor token_code: Lexer::Token::UserCode + # attr_accessor lineno: Integer + # + # def initialize: (?ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], ?token_code: Lexer::Token::UserCode, ?lineno: Integer) -> void + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::PrinterCode.new(type: :printer, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/reference.rb b/tool/lrama/lib/lrama/grammar/reference.rb new file mode 100644 index 0000000000..7e3badfecc --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/reference.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # type: :dollar or :at + # name: String (e.g. $$, $foo, $expr.right) + # number: Integer (e.g. $1) + # index: Integer + # ex_tag: "$<tag>1" (Optional) + class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + # @rbs! + # attr_accessor type: ::Symbol + # attr_accessor name: String + # attr_accessor number: Integer + # attr_accessor index: Integer + # attr_accessor ex_tag: Lexer::Token::Base? + # attr_accessor first_column: Integer + # attr_accessor last_column: Integer + # + # def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token::Base?, first_column: Integer, last_column: Integer) -> void + + # @rbs () -> (String|Integer) + def value + name || number + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule.rb b/tool/lrama/lib/lrama/grammar/rule.rb new file mode 100644 index 0000000000..d00d6a8883 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule.rb @@ -0,0 +1,135 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # _rhs holds original RHS element. Use rhs to refer to Symbol. + class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true) + # @rbs! + # + # interface _DelegatedMethods + # def lhs: -> Grammar::Symbol + # def rhs: -> Array[Grammar::Symbol] + # end + # + # attr_accessor id: Integer + # attr_accessor _lhs: Lexer::Token::Base + # attr_accessor lhs: Grammar::Symbol + # attr_accessor lhs_tag: Lexer::Token::Tag? + # attr_accessor _rhs: Array[Lexer::Token::Base] + # attr_accessor rhs: Array[Grammar::Symbol] + # attr_accessor token_code: Lexer::Token::UserCode? + # attr_accessor position_in_original_rule_rhs: Integer + # attr_accessor nullable: bool + # attr_accessor precedence_sym: Grammar::Symbol? + # attr_accessor lineno: Integer? + # + # def initialize: ( + # ?id: Integer, ?_lhs: Lexer::Token::Base?, ?lhs: Lexer::Token::Base, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token::Base], ?rhs: Array[Grammar::Symbol], + # ?token_code: Lexer::Token::UserCode?, ?position_in_original_rule_rhs: Integer?, ?nullable: bool, + # ?precedence_sym: Grammar::Symbol?, ?lineno: Integer? + # ) -> void + + attr_accessor :original_rule #: Rule + + # @rbs (Rule other) -> bool + def ==(other) + self.class == other.class && + self.lhs == other.lhs && + self.lhs_tag == other.lhs_tag && + self.rhs == other.rhs && + self.token_code == other.token_code && + self.position_in_original_rule_rhs == other.position_in_original_rule_rhs && + self.nullable == other.nullable && + self.precedence_sym == other.precedence_sym && + self.lineno == other.lineno + end + + # @rbs () -> String + def display_name + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map {|r| r.id.s_value }.join(" ") + "#{l} -> #{r}" + end + + # @rbs () -> String + def display_name_without_action + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map do |r| + r.id.s_value if r.first_set.any? + end.compact.join(" ") + + "#{l} -> #{r}" + end + + # @rbs () -> (RailroadDiagrams::Skip | RailroadDiagrams::Sequence) + def to_diagrams + if rhs.empty? + RailroadDiagrams::Skip.new + else + RailroadDiagrams::Sequence.new(*rhs_to_diagram) + end + end + + # Used by #user_actions + # + # @rbs () -> String + def as_comment + l = lhs.id.s_value + r = empty_rule? ? "%empty" : rhs.map(&:display_name).join(" ") + + "#{l}: #{r}" + end + + # @rbs () -> String + def with_actions + "#{display_name} {#{token_code&.s_value}}" + end + + # opt_nl: ε <-- empty_rule + # | '\n' <-- not empty_rule + # + # @rbs () -> bool + def empty_rule? + rhs.empty? + end + + # @rbs () -> Precedence? + def precedence + precedence_sym&.precedence + end + + # @rbs () -> bool + def initial_rule? + id == 0 + end + + # @rbs () -> String? + def translated_code + return nil unless token_code + + Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self).translated_code + end + + # @rbs () -> bool + def contains_at_reference? + return false unless token_code + + token_code.references.any? {|r| r.type == :at } + end + + private + + # @rbs () -> Array[(RailroadDiagrams::Terminal | RailroadDiagrams::NonTerminal)] + def rhs_to_diagram + rhs.map do |r| + if r.term + RailroadDiagrams::Terminal.new(r.id.s_value) + else + RailroadDiagrams::NonTerminal.new(r.id.s_value) + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb new file mode 100644 index 0000000000..34fdca6c86 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -0,0 +1,270 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class RuleBuilder + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @position_in_original_rule_rhs: Integer? + # @skip_preprocess_references: bool + # @rules: Array[Rule] + # @rule_builders_for_parameterized: Array[RuleBuilder] + # @rule_builders_for_derived_rules: Array[RuleBuilder] + # @parameterized_rules: Array[Rule] + # @midrule_action_rules: Array[Rule] + # @replaced_rhs: Array[Lexer::Token::Base]? + + attr_accessor :lhs #: Lexer::Token::Base? + attr_accessor :line #: Integer? + attr_reader :rule_counter #: Counter + attr_reader :midrule_action_counter #: Counter + attr_reader :parameterized_resolver #: Grammar::Parameterized::Resolver + attr_reader :lhs_tag #: Lexer::Token::Tag? + attr_reader :rhs #: Array[Lexer::Token::Base] + attr_reader :user_code #: Lexer::Token::UserCode? + attr_reader :precedence_sym #: Grammar::Symbol? + + # @rbs (Counter rule_counter, Counter midrule_action_counter, Grammar::Parameterized::Resolver parameterized_resolver, ?Integer position_in_original_rule_rhs, ?lhs_tag: Lexer::Token::Tag?, ?skip_preprocess_references: bool) -> void + def initialize(rule_counter, midrule_action_counter, parameterized_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) + @rule_counter = rule_counter + @midrule_action_counter = midrule_action_counter + @parameterized_resolver = parameterized_resolver + @position_in_original_rule_rhs = position_in_original_rule_rhs + @skip_preprocess_references = skip_preprocess_references + + @lhs = nil + @lhs_tag = lhs_tag + @rhs = [] + @user_code = nil + @precedence_sym = nil + @line = nil + @rules = [] + @rule_builders_for_parameterized = [] + @rule_builders_for_derived_rules = [] + @parameterized_rules = [] + @midrule_action_rules = [] + end + + # @rbs (Lexer::Token::Base rhs) -> void + def add_rhs(rhs) + @line ||= rhs.line + + flush_user_code + + @rhs << rhs + end + + # @rbs (Lexer::Token::UserCode? user_code) -> void + def user_code=(user_code) + @line ||= user_code&.line + + flush_user_code + + @user_code = user_code + end + + # @rbs (Grammar::Symbol? precedence_sym) -> void + def precedence_sym=(precedence_sym) + flush_user_code + + @precedence_sym = precedence_sym + end + + # @rbs () -> void + def complete_input + freeze_rhs + end + + # @rbs () -> void + def setup_rules + preprocess_references unless @skip_preprocess_references + process_rhs + resolve_inline_rules + build_rules + end + + # @rbs () -> Array[Grammar::Rule] + def rules + @parameterized_rules + @midrule_action_rules + @rules + end + + # @rbs () -> bool + def has_inline_rules? + rhs.any? { |token| @parameterized_resolver.find_inline(token) } + end + + private + + # @rbs () -> void + def freeze_rhs + @rhs.freeze + end + + # @rbs () -> void + def preprocess_references + numberize_references + end + + # @rbs () -> void + def build_rules + tokens = @replaced_rhs #: Array[Lexer::Token::Base] + return if tokens.any? { |t| @parameterized_resolver.find_inline(t) } + + rule = Rule.new( + id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code, + position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line + ) + @rules = [rule] + @parameterized_rules = @rule_builders_for_parameterized.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules.each do |r| + r.original_rule = rule + end + end + + # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on. + # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`. + # + # @rbs () -> void + def process_rhs + return if @replaced_rhs + + replaced_rhs = [] #: Array[Lexer::Token::Base] + + rhs.each_with_index do |token, i| + case token + when Lrama::Lexer::Token::Char + replaced_rhs << token + when Lrama::Lexer::Token::Ident + replaced_rhs << token + when Lrama::Lexer::Token::InstantiateRule + parameterized_rule = @parameterized_resolver.find_rule(token) + raise "Unexpected token. #{token}" unless parameterized_rule + + bindings = Binding.new(parameterized_rule.parameters, token.args) + lhs_s_value = bindings.concatenated_args_str(token) + if (created_lhs = @parameterized_resolver.created_lhs(lhs_s_value)) + replaced_rhs << created_lhs + else + lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) + replaced_rhs << lhs_token + @parameterized_resolver.created_lhs_list << lhs_token + parameterized_rule.rhs.each do |r| + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, lhs_tag: token.lhs_tag || parameterized_rule.tag) + rule_builder.lhs = lhs_token + r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } + rule_builder.line = line + rule_builder.precedence_sym = r.precedence_sym + rule_builder.user_code = r.resolve_user_code(bindings) + rule_builder.complete_input + rule_builder.setup_rules + @rule_builders_for_parameterized << rule_builder + end + end + when Lrama::Lexer::Token::UserCode + prefix = token.referred ? "@" : "$@" + tag = token.tag || lhs_tag + new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) + replaced_rhs << new_token + + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, i, lhs_tag: tag, skip_preprocess_references: true) + rule_builder.lhs = new_token + rule_builder.user_code = token + rule_builder.complete_input + rule_builder.setup_rules + + @rule_builders_for_derived_rules << rule_builder + when Lrama::Lexer::Token::Empty + # Noop + else + raise "Unexpected token. #{token}" + end + end + + @replaced_rhs = replaced_rhs + end + + # @rbs () -> void + def resolve_inline_rules + while @rule_builders_for_parameterized.any?(&:has_inline_rules?) do + @rule_builders_for_parameterized = @rule_builders_for_parameterized.flat_map do |rule_builder| + if rule_builder.has_inline_rules? + inlined_builders = Inline::Resolver.new(rule_builder).resolve + inlined_builders.each { |builder| builder.setup_rules } + inlined_builders + else + rule_builder + end + end + end + end + + # @rbs () -> void + def numberize_references + # Bison n'th component is 1-origin + (rhs + [user_code]).compact.each.with_index(1) do |token, i| + next unless token.is_a?(Lrama::Lexer::Token::UserCode) + + token.references.each do |ref| + ref_name = ref.name + + if ref_name + if ref_name == '$' + ref.name = '$' + else + candidates = ([lhs] + rhs).each_with_index.select do |token, _i| + # @type var token: Lexer::Token::Base + token.referred_by?(ref_name) + end + + if candidates.size >= 2 + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is duplicated.") + end + + unless (referring_symbol = candidates.first) + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is not found.") + end + + if referring_symbol[1] == 0 # Refers to LHS + ref.name = '$' + else + ref.number = referring_symbol[1] + end + end + end + + if ref.number + ref.index = ref.number + end + + # TODO: Need to check index of @ too? + next if ref.type == :at + + if ref.index + # TODO: Prohibit $0 even so Bison allows it? + # See: https://www.gnu.org/software/bison/manual/html_node/Actions.html + token.invalid_ref(ref, "Can not refer following component. #{ref.index} >= #{i}.") if ref.index >= i + rhs[ref.index - 1].referred = true + end + end + end + end + + # @rbs () -> void + def flush_user_code + if (c = @user_code) + @rhs << c + @user_code = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/stdlib.y b/tool/lrama/lib/lrama/grammar/stdlib.y new file mode 100644 index 0000000000..dd397c9e08 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/stdlib.y @@ -0,0 +1,142 @@ +/********************************************************************** + + stdlib.y + + This is lrama's standard library. It provides a number of + parameterized rule definitions, such as options and lists, + that should be useful in a number of situations. + +**********************************************************************/ + +%% + +// ------------------------------------------------------------------- +// Options + +/* + * program: option(X) + * + * => + * + * program: option_X + * option_X: %empty + * option_X: X + */ +%rule option(X) + : /* empty */ + | X + ; + + +/* + * program: ioption(X) + * + * => + * + * program: %empty + * program: X + */ +%rule %inline ioption(X) + : /* empty */ + | X + ; + +// ------------------------------------------------------------------- +// Sequences + +/* + * program: preceded(opening, X) + * + * => + * + * program: preceded_opening_X + * preceded_opening_X: opening X + */ +%rule preceded(opening, X) + : opening X { $$ = $2; } + ; + +/* + * program: terminated(X, closing) + * + * => + * + * program: terminated_X_closing + * terminated_X_closing: X closing + */ +%rule terminated(X, closing) + : X closing { $$ = $1; } + ; + +/* + * program: delimited(opening, X, closing) + * + * => + * + * program: delimited_opening_X_closing + * delimited_opening_X_closing: opening X closing + */ +%rule delimited(opening, X, closing) + : opening X closing { $$ = $2; } + ; + +// ------------------------------------------------------------------- +// Lists + +/* + * program: list(X) + * + * => + * + * program: list_X + * list_X: %empty + * list_X: list_X X + */ +%rule list(X) + : /* empty */ + | list(X) X + ; + +/* + * program: nonempty_list(X) + * + * => + * + * program: nonempty_list_X + * nonempty_list_X: X + * nonempty_list_X: nonempty_list_X X + */ +%rule nonempty_list(X) + : X + | nonempty_list(X) X + ; + +/* + * program: separated_nonempty_list(separator, X) + * + * => + * + * program: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separated_nonempty_list_separator_X separator X + */ +%rule separated_nonempty_list(separator, X) + : X + | separated_nonempty_list(separator, X) separator X + ; + +/* + * program: separated_list(separator, X) + * + * => + * + * program: separated_list_separator_X + * separated_list_separator_X: option_separated_nonempty_list_separator_X + * option_separated_nonempty_list_separator_X: %empty + * option_separated_nonempty_list_separator_X: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separator separated_nonempty_list_separator_X X + */ +%rule separated_list(separator, X) + : option(separated_nonempty_list(separator, X)) + ; diff --git a/tool/lrama/lib/lrama/grammar/symbol.rb b/tool/lrama/lib/lrama/grammar/symbol.rb new file mode 100644 index 0000000000..07aee0c0a2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbol.rb @@ -0,0 +1,149 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# Symbol is both of nterm and term +# `number` is both for nterm and term +# `token_id` is tokentype for term, internal sequence number for nterm +# + +module Lrama + class Grammar + class Symbol + attr_accessor :id #: Lexer::Token::Base + attr_accessor :alias_name #: String? + attr_reader :number #: Integer + attr_accessor :number_bitmap #: Bitmap::bitmap + attr_accessor :tag #: Lexer::Token::Tag? + attr_accessor :token_id #: Integer + attr_accessor :nullable #: bool + attr_accessor :precedence #: Precedence? + attr_accessor :printer #: Printer? + attr_accessor :destructor #: Destructor? + attr_accessor :error_token #: ErrorToken + attr_accessor :first_set #: Set[Grammar::Symbol] + attr_accessor :first_set_bitmap #: Bitmap::bitmap + attr_reader :term #: bool + attr_writer :eof_symbol #: bool + attr_writer :error_symbol #: bool + attr_writer :undef_symbol #: bool + attr_writer :accept_symbol #: bool + + # @rbs (id: Lexer::Token::Base, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token::Tag?, + # ?token_id: Integer?, ?nullable: bool?, ?precedence: Precedence?, ?printer: Printer?) -> void + def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil) + @id = id + @alias_name = alias_name + @number = number + @tag = tag + @term = term + @token_id = token_id + @nullable = nullable + @precedence = precedence + @printer = printer + @destructor = destructor + end + + # @rbs (Integer) -> void + def number=(number) + @number = number + @number_bitmap = Bitmap::from_integer(number) + end + + # @rbs () -> bool + def term? + term + end + + # @rbs () -> bool + def nterm? + !term + end + + # @rbs () -> bool + def eof_symbol? + !!@eof_symbol + end + + # @rbs () -> bool + def error_symbol? + !!@error_symbol + end + + # @rbs () -> bool + def undef_symbol? + !!@undef_symbol + end + + # @rbs () -> bool + def accept_symbol? + !!@accept_symbol + end + + # @rbs () -> bool + def midrule? + return false if term? + + name.include?("$") || name.include?("@") + end + + # @rbs () -> String + def name + id.s_value + end + + # @rbs () -> String + def display_name + alias_name || name + end + + # name for yysymbol_kind_t + # + # See: b4_symbol_kind_base + # @type var name: String + # @rbs () -> String + def enum_name + case + when accept_symbol? + res = "YYACCEPT" + when eof_symbol? + res = "YYEOF" + when term? && id.is_a?(Lrama::Lexer::Token::Char) + res = number.to_s + display_name + when term? && id.is_a?(Lrama::Lexer::Token::Ident) + res = name + when midrule? + res = number.to_s + name + when nterm? + res = name + else + raise "Unexpected #{self}" + end + + "YYSYMBOL_" + res.gsub(/\W+/, "_") + end + + # comment for yysymbol_kind_t + # + # @rbs () -> String? + def comment + case + when accept_symbol? + # YYSYMBOL_YYACCEPT + name + when eof_symbol? + # YYEOF + alias_name + when (term? && 0 < token_id && token_id < 128) + # YYSYMBOL_3_backslash_, YYSYMBOL_14_ + display_name + when midrule? + # YYSYMBOL_21_1 + name + else + # YYSYMBOL_keyword_class, YYSYMBOL_strings_1 + display_name + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/symbols.rb b/tool/lrama/lib/lrama/grammar/symbols.rb new file mode 100644 index 0000000000..337241d1b2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative "symbols/resolver" diff --git a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb new file mode 100644 index 0000000000..085a835d28 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb @@ -0,0 +1,362 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Symbols + class Resolver + # @rbs! + # + # interface _DelegatedMethods + # def symbols: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def add_nterm: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + # def add_term: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + # def find_symbol_by_number!: (Integer number) -> Grammar::Symbol + # def find_symbol_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol + # def token_to_symbol: (Lexer::Token::Base token) -> Grammar::Symbol + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def fill_nterm_type: (Array[Grammar::Type] types) -> void + # def fill_symbol_number: () -> void + # def fill_printer: (Array[Grammar::Printer] printers) -> void + # def fill_destructor: (Array[Destructor] destructors) -> (Destructor | bot) + # def fill_error_token: (Array[Grammar::ErrorToken] error_tokens) -> void + # def sort_by_number!: () -> Array[Grammar::Symbol] + # end + # + # @symbols: Array[Grammar::Symbol]? + # @number: Integer + # @used_numbers: Hash[Integer, bool] + + attr_reader :terms #: Array[Grammar::Symbol] + attr_reader :nterms #: Array[Grammar::Symbol] + + # @rbs () -> void + def initialize + @terms = [] + @nterms = [] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols + @symbols ||= (@terms + @nterms) + end + + # @rbs () -> Array[Grammar::Symbol] + def sort_by_number! + symbols.sort_by!(&:number) + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) + if token_id && (sym = find_symbol_by_token_id(token_id)) + if replace + sym.id = id + sym.alias_name = alias_name + sym.tag = tag + end + + return sym + end + + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + term = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: true, token_id: token_id, nullable: false + ) + @terms << term + term + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + def add_nterm(id:, alias_name: nil, tag: nil) + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + nterm = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: false, token_id: nil, nullable: nil, + ) + @nterms << nterm + nterm + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_term_by_s_value(s_value) + terms.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_symbol_by_s_value(s_value) + symbols.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol + def find_symbol_by_s_value!(s_value) + find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`") + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol? + def find_symbol_by_id(id) + symbols.find do |s| + s.id == id || s.alias_name == id.s_value + end + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_symbol_by_id!(id) + find_symbol_by_id(id) || (raise "Symbol not found. #{id}") + end + + # @rbs (Integer token_id) -> Grammar::Symbol? + def find_symbol_by_token_id(token_id) + symbols.find {|s| s.token_id == token_id } + end + + # @rbs (Integer number) -> Grammar::Symbol + def find_symbol_by_number!(number) + sym = symbols[number] + + raise "Symbol not found. number: `#{number}`" unless sym + raise "[BUG] Symbol number mismatch. #{number}, #{sym}" if sym.number != number + + sym + end + + # @rbs () -> void + def fill_symbol_number + # YYEMPTY = -2 + # YYEOF = 0 + # YYerror = 1 + # YYUNDEF = 2 + @number = 3 + fill_terms_number + fill_nterms_number + end + + # @rbs (Array[Grammar::Type] types) -> void + def fill_nterm_type(types) + types.each do |type| + nterm = find_nterm_by_id!(type.id) + nterm.tag = type.tag + end + end + + # @rbs (Array[Grammar::Printer] printers) -> void + def fill_printer(printers) + symbols.each do |sym| + printers.each do |printer| + printer.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.printer = printer if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.printer = printer if sym.tag == ident_or_tag + else + raise "Unknown token type. #{printer}" + end + end + end + end + end + + # @rbs (Array[Destructor] destructors) -> (Array[Grammar::Symbol] | bot) + def fill_destructor(destructors) + symbols.each do |sym| + destructors.each do |destructor| + destructor.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.destructor = destructor if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.destructor = destructor if sym.tag == ident_or_tag + else + raise "Unknown token type. #{destructor}" + end + end + end + end + end + + # @rbs (Array[Grammar::ErrorToken] error_tokens) -> void + def fill_error_token(error_tokens) + symbols.each do |sym| + error_tokens.each do |token| + token.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.error_token = token if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.error_token = token if sym.tag == ident_or_tag + else + raise "Unknown token type. #{token}" + end + end + end + end + end + + # @rbs (Lexer::Token::Base token) -> Grammar::Symbol + def token_to_symbol(token) + case token + when Lrama::Lexer::Token::Base + find_symbol_by_id!(token) + else + raise "Unknown class: #{token}" + end + end + + # @rbs () -> void + def validate! + validate_number_uniqueness! + validate_alias_name_uniqueness! + validate_symbols! + end + + private + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_nterm_by_id!(id) + @nterms.find do |s| + s.id == id + end || (raise "Symbol not found. #{id}") + end + + # @rbs () -> void + def fill_terms_number + # Character literal in grammar file has + # token id corresponding to ASCII code by default, + # so start token_id from 256. + token_id = 256 + + @terms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + # If id is Token::Char, it uses ASCII code + if sym.token_id.nil? + if sym.id.is_a?(Lrama::Lexer::Token::Char) + # Ignore ' on the both sides + case sym.id.s_value[1..-2] + when "\\b" + sym.token_id = 8 + when "\\f" + sym.token_id = 12 + when "\\n" + sym.token_id = 10 + when "\\r" + sym.token_id = 13 + when "\\t" + sym.token_id = 9 + when "\\v" + sym.token_id = 11 + when "\"" + sym.token_id = 34 + when "'" + sym.token_id = 39 + when "\\\\" + sym.token_id = 92 + when /\A\\(\d+)\z/ + unless (id = Integer($1, 8)).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + when /\A(.)\z/ + unless (id = $1&.bytes&.first).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + else + raise "Unknown Char s_value #{sym}" + end + else + sym.token_id = token_id + token_id += 1 + end + end + end + end + + # @rbs () -> void + def fill_nterms_number + token_id = 0 + + @nterms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + if sym.token_id.nil? + sym.token_id = token_id + token_id += 1 + end + end + end + + # @rbs () -> Hash[Integer, bool] + def used_numbers + return @used_numbers if defined?(@used_numbers) + + @used_numbers = {} + symbols.map(&:number).each do |n| + @used_numbers[n] = true + end + @used_numbers + end + + # @rbs () -> void + def validate_number_uniqueness! + invalid = symbols.group_by(&:number).select do |number, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol number is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_alias_name_uniqueness! + invalid = symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol alias name is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_symbols! + symbols.each { |sym| sym.id.validate } + errors = symbols.map { |sym| sym.id.errors }.flatten.compact + return if errors.empty? + + raise errors.join("\n") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/type.rb b/tool/lrama/lib/lrama/grammar/type.rb new file mode 100644 index 0000000000..c631769447 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/type.rb @@ -0,0 +1,32 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Type + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @id: Lexer::Token::Base + # @tag: Lexer::Token::Tag + + attr_reader :id #: Lexer::Token::Base + attr_reader :tag #: Lexer::Token::Tag + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> void + def initialize(id:, tag:) + @id = id + @tag = tag + end + + # @rbs (Grammar::Type other) -> bool + def ==(other) + self.class == other.class && + self.id == other.id && + self.tag == other.tag + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/union.rb b/tool/lrama/lib/lrama/grammar/union.rb new file mode 100644 index 0000000000..774cc66fc6 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/union.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Union + attr_reader :code #: Grammar::Code::NoReferenceCode + attr_reader :lineno #: Integer + + # @rbs (code: Grammar::Code::NoReferenceCode, lineno: Integer) -> void + def initialize(code:, lineno:) + @code = code + @lineno = lineno + end + + # @rbs () -> String + def braces_less_code + # Braces is already removed by lexer + code.s_value + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb new file mode 100644 index 0000000000..ce98b505a7 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer.rb @@ -0,0 +1,219 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "strscan" + +require_relative "lexer/grammar_file" +require_relative "lexer/location" +require_relative "lexer/token" + +module Lrama + class Lexer + # @rbs! + # + # type token = lexer_token | c_token + # + # type lexer_token = [String, Token::Token] | + # [::Symbol, Token::Tag] | + # [::Symbol, Token::Char] | + # [::Symbol, Token::Str] | + # [::Symbol, Token::Int] | + # [::Symbol, Token::Ident] + # + # type c_token = [:C_DECLARATION, Token::UserCode] + + attr_reader :head_line #: Integer + attr_reader :head_column #: Integer + attr_reader :line #: Integer + attr_accessor :status #: :initial | :c_declaration + attr_accessor :end_symbol #: String? + + SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze #: Array[String] + PERCENT_TOKENS = %w( + %union + %token + %type + %nterm + %left + %right + %nonassoc + %expect + %define + %require + %printer + %destructor + %lex-param + %parse-param + %initial-action + %precedence + %prec + %error-token + %before-reduce + %after-reduce + %after-shift-error-token + %after-shift + %after-pop-stack + %empty + %code + %rule + %no-stdlib + %inline + %locations + %categories + %start + ).freeze #: Array[String] + + # @rbs (GrammarFile grammar_file) -> void + def initialize(grammar_file) + @grammar_file = grammar_file + @scanner = StringScanner.new(grammar_file.text) + @head_column = @head = @scanner.pos + @head_line = @line = 1 + @status = :initial + @end_symbol = nil + end + + # @rbs () -> token? + def next_token + case @status + when :initial + lex_token + when :c_declaration + lex_c_code + end + end + + # @rbs () -> Integer + def column + @scanner.pos - @head + end + + # @rbs () -> Location + def location + Location.new( + grammar_file: @grammar_file, + first_line: @head_line, first_column: @head_column, + last_line: line, last_column: column + ) + end + + # @rbs () -> lexer_token? + def lex_token + until @scanner.eos? do + case + when @scanner.scan(/\n/) + newline + when @scanner.scan(/\s+/) + @scanner.matched.count("\n").times { newline } + when @scanner.scan(/\/\*/) + lex_comment + when @scanner.scan(/\/\/.*(?<newline>\n)?/) + newline if @scanner[:newline] + else + break + end + end + + reset_first_position + + case + when @scanner.eos? + return + when @scanner.scan(/#{SYMBOLS.join('|')}/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/[\?\+\*]/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/<\w+>/) + return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'.'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/".*?"/) + return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] + when @scanner.scan(/\d+/) + return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)] + when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) + token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location) + type = + if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/) + :IDENT_COLON + else + :IDENTIFIER + end + return [type, token] + else + raise ParseError, location.generate_error_message("Unexpected token") # steep:ignore UnknownConstant + end + end + + # @rbs () -> c_token + def lex_c_code + nested = 0 + code = +'' + reset_first_position + + until @scanner.eos? do + case + when @scanner.scan(/{/) + code << @scanner.matched + nested += 1 + when @scanner.scan(/}/) + if nested == 0 && @end_symbol == '}' + @scanner.unscan + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + else + code << @scanner.matched + nested -= 1 + end + when @scanner.check(/#{@end_symbol}/) + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + when @scanner.scan(/\n/) + code << @scanner.matched + newline + when @scanner.scan(/".*?"/) + code << %Q(#{@scanner.matched}) + @line += @scanner.matched.count("\n") + when @scanner.scan(/'.*?'/) + code << %Q(#{@scanner.matched}) + when @scanner.scan(/[^\"'\{\}\n]+/) + code << @scanner.matched + when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) # steep:ignore + code << @scanner.matched + else + code << @scanner.getch + end + end + raise ParseError, location.generate_error_message("Unexpected code: #{code}") # steep:ignore UnknownConstant + end + + private + + # @rbs () -> void + def lex_comment + until @scanner.eos? do + case + when @scanner.scan_until(/[\s\S]*?\*\//) + @scanner.matched.count("\n").times { newline } + return + when @scanner.scan_until(/\n/) + newline + end + end + end + + # @rbs () -> void + def reset_first_position + @head_line = line + @head_column = column + end + + # @rbs () -> void + def newline + @line += 1 + @head = @scanner.pos + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/grammar_file.rb b/tool/lrama/lib/lrama/lexer/grammar_file.rb new file mode 100644 index 0000000000..37e82ff18d --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/grammar_file.rb @@ -0,0 +1,40 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + class GrammarFile + class Text < String + # @rbs () -> String + def inspect + length <= 50 ? super : "#{self[0..47]}...".inspect + end + end + + attr_reader :path #: String + attr_reader :text #: String + + # @rbs (String path, String text) -> void + def initialize(path, text) + @path = path + @text = Text.new(text).freeze + end + + # @rbs () -> String + def inspect + "<#{self.class}: @path=#{path}, @text=#{text.inspect}>" + end + + # @rbs (GrammarFile other) -> bool + def ==(other) + self.class == other.class && + self.path == other.path + end + + # @rbs () -> Array[String] + def lines + @lines ||= text.split("\n") + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/location.rb b/tool/lrama/lib/lrama/lexer/location.rb new file mode 100644 index 0000000000..4465576d53 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/location.rb @@ -0,0 +1,132 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + class Location + attr_reader :grammar_file #: GrammarFile + attr_reader :first_line #: Integer + attr_reader :first_column #: Integer + attr_reader :last_line #: Integer + attr_reader :last_column #: Integer + + # @rbs (grammar_file: GrammarFile, first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void + def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:) + @grammar_file = grammar_file + @first_line = first_line + @first_column = first_column + @last_line = last_line + @last_column = last_column + end + + # @rbs (Location other) -> bool + def ==(other) + self.class == other.class && + self.grammar_file == other.grammar_file && + self.first_line == other.first_line && + self.first_column == other.first_column && + self.last_line == other.last_line && + self.last_column == other.last_column + end + + # @rbs (Integer left, Integer right) -> Location + def partial_location(left, right) + offset = -first_column + new_first_line = -1 + new_first_column = -1 + new_last_line = -1 + new_last_column = -1 + + _text.each.with_index do |line, index| + new_offset = offset + line.length + 1 + + if offset <= left && left <= new_offset + new_first_line = first_line + index + new_first_column = left - offset + end + + if offset <= right && right <= new_offset + new_last_line = first_line + index + new_last_column = right - offset + end + + offset = new_offset + end + + Location.new( + grammar_file: grammar_file, + first_line: new_first_line, first_column: new_first_column, + last_line: new_last_line, last_column: new_last_column + ) + end + + # @rbs () -> String + def to_s + "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})" + end + + # @rbs (String error_message) -> String + def generate_error_message(error_message) + <<~ERROR.chomp + #{path}:#{first_line}:#{first_column}: #{error_message} + #{error_with_carets} + ERROR + end + + # @rbs () -> String + def error_with_carets + <<~TEXT + #{formatted_first_lineno} | #{text} + #{line_number_padding} | #{carets_line} + TEXT + end + + private + + # @rbs () -> String + def path + grammar_file.path + end + + # @rbs () -> String + def carets_line + leading_whitespace + highlight_marker + end + + # @rbs () -> String + def leading_whitespace + (text[0...first_column] or raise "Invalid first_column: #{first_column}") + .gsub(/[^\t]/, ' ') + end + + # @rbs () -> String + def highlight_marker + length = last_column - first_column + '^' + '~' * [0, length - 1].max + end + + # @rbs () -> String + def formatted_first_lineno + first_line.to_s.rjust(4) + end + + # @rbs () -> String + def line_number_padding + ' ' * formatted_first_lineno.length + end + + # @rbs () -> String + def text + @text ||= _text.join("\n") + end + + # @rbs () -> Array[String] + def _text + @_text ||=begin + range = (first_line - 1)...last_line + grammar_file.lines[range] or raise "#{range} is invalid" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb new file mode 100644 index 0000000000..37f77aa069 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -0,0 +1,20 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'token/base' +require_relative 'token/char' +require_relative 'token/empty' +require_relative 'token/ident' +require_relative 'token/instantiate_rule' +require_relative 'token/int' +require_relative 'token/str' +require_relative 'token/tag' +require_relative 'token/token' +require_relative 'token/user_code' + +module Lrama + class Lexer + module Token + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/base.rb b/tool/lrama/lib/lrama/lexer/token/base.rb new file mode 100644 index 0000000000..3df93bbc73 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/base.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Base + attr_reader :s_value #: String + attr_reader :location #: Location + attr_accessor :alias_name #: String + attr_accessor :referred #: bool + attr_reader :errors #: Array[String] + + # @rbs (s_value: String, ?alias_name: String, ?location: Location) -> void + def initialize(s_value:, alias_name: nil, location: nil) + s_value.freeze + @s_value = s_value + @alias_name = alias_name + @location = location + @errors = [] + end + + # @rbs () -> String + def to_s + "value: `#{s_value}`, location: #{location}" + end + + # @rbs (String string) -> bool + def referred_by?(string) + [self.s_value, self.alias_name].compact.include?(string) + end + + # @rbs (Lexer::Token::Base other) -> bool + def ==(other) + self.class == other.class && self.s_value == other.s_value + end + + # @rbs () -> Integer + def first_line + location.first_line + end + alias :line :first_line + + # @rbs () -> Integer + def first_column + location.first_column + end + alias :column :first_column + + # @rbs () -> Integer + def last_line + location.last_line + end + + # @rbs () -> Integer + def last_column + location.last_column + end + + # @rbs (Lrama::Grammar::Reference ref, String message) -> bot + def invalid_ref(ref, message) + location = self.location.partial_location(ref.first_column, ref.last_column) + raise location.generate_error_message(message) + end + + # @rbs () -> bool + def validate + true + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/char.rb b/tool/lrama/lib/lrama/lexer/token/char.rb new file mode 100644 index 0000000000..f4ef7c9fbc --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/char.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Char < Base + # @rbs () -> void + def validate + validate_ascii_code_range + end + + private + + # @rbs () -> void + def validate_ascii_code_range + unless s_value.ascii_only? + errors << "Invalid character: `#{s_value}`. Only ASCII characters are allowed." + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/empty.rb b/tool/lrama/lib/lrama/lexer/token/empty.rb new file mode 100644 index 0000000000..375e256493 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/empty.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Empty < Base + def initialize(location: nil) + super(s_value: '%empty', location: location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/ident.rb b/tool/lrama/lib/lrama/lexer/token/ident.rb new file mode 100644 index 0000000000..4880be9073 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/ident.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Ident < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb new file mode 100644 index 0000000000..7051ba75a4 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class InstantiateRule < Base + attr_reader :args #: Array[Lexer::Token::Base] + attr_reader :lhs_tag #: Lexer::Token::Tag? + + # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token::Base], ?lhs_tag: Lexer::Token::Tag?) -> void + def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil) + super s_value: s_value, alias_name: alias_name, location: location + @args = args + @lhs_tag = lhs_tag + end + + # @rbs () -> String + def rule_name + s_value + end + + # @rbs () -> Integer + def args_count + args.count + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/int.rb b/tool/lrama/lib/lrama/lexer/token/int.rb new file mode 100644 index 0000000000..7daf48d4d3 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/int.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Int < Base + # @rbs! + # def initialize: (s_value: Integer, ?alias_name: String, ?location: Location) -> void + # def s_value: () -> Integer + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/str.rb b/tool/lrama/lib/lrama/lexer/token/str.rb new file mode 100644 index 0000000000..cf9de6cf0f --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/str.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Str < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/tag.rb b/tool/lrama/lib/lrama/lexer/token/tag.rb new file mode 100644 index 0000000000..68c6268219 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/tag.rb @@ -0,0 +1,16 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Tag < Base + # @rbs () -> String + def member + # Omit "<>" + s_value[1..-2] or raise "Unexpected Tag format (#{s_value})" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/token.rb b/tool/lrama/lib/lrama/lexer/token/token.rb new file mode 100644 index 0000000000..935797efc6 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/token.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Token < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb new file mode 100644 index 0000000000..166f04954a --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -0,0 +1,109 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "strscan" + +module Lrama + class Lexer + module Token + class UserCode < Base + attr_accessor :tag #: Lexer::Token::Tag + + # @rbs () -> Array[Lrama::Grammar::Reference] + def references + @references ||= _references + end + + private + + # @rbs () -> Array[Lrama::Grammar::Reference] + def _references + scanner = StringScanner.new(s_value) + references = [] #: Array[Grammar::Reference] + + until scanner.eos? do + case + when reference = scan_reference(scanner) + references << reference + when scanner.scan(/\/\*/) + scanner.scan_until(/\*\//) + else + scanner.getch + end + end + + references + end + + # @rbs (StringScanner scanner) -> Lrama::Grammar::Reference? + def scan_reference(scanner) + start = scanner.pos + if scanner.scan(/ + # $ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + \$(<[a-zA-Z0-9_]+>)?(?: + (\$) # $$, $<long>$ + | (\d+) # $1, $2, $<long>1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $foo, $expr, $<long>program (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + ) + | + # @ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + @(?: + (\$) # @$ + | (\d+) # @1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # @foo, @expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # @[expr.right], @[expr-right] (named reference with brackets) + ) + | + # $: references + \$: + (?: + (\$) # $:$ + | (\d+) # $:1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $:foo, $:expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $:[expr.right], $:[expr-right] (named reference with brackets) + ) + /x) + case + # $ references + when scanner[2] # $$, $<long>$ + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[3] # $1, $2, $<long>1 + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[3]), index: Integer(scanner[3]), ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[4] # $foo, $expr, $<long>program (named reference without brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[4], ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[5] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[5], ex_tag: tag, first_column: start, last_column: scanner.pos) + + # @ references + when scanner[6] # @$ + return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) + when scanner[7] # @1 + return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[7]), index: Integer(scanner[7]), first_column: start, last_column: scanner.pos) + when scanner[8] # @foo, @expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[8], first_column: start, last_column: scanner.pos) + when scanner[9] # @[expr.right], @[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[9], first_column: start, last_column: scanner.pos) + + # $: references + when scanner[10] # $:$ + return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) + when scanner[11] # $:1 + return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[11]), index: Integer(scanner[11]), first_column: start, last_column: scanner.pos) + when scanner[12] # $:foo, $:expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[12], first_column: start, last_column: scanner.pos) + when scanner[13] # $:[expr.right], $:[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[13], first_column: start, last_column: scanner.pos) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/logger.rb b/tool/lrama/lib/lrama/logger.rb new file mode 100644 index 0000000000..291eea5296 --- /dev/null +++ b/tool/lrama/lib/lrama/logger.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Logger + # @rbs (IO out) -> void + def initialize(out = STDERR) + @out = out + end + + # @rbs () -> void + def line_break + @out << "\n" + end + + # @rbs (String message) -> void + def trace(message) + @out << message << "\n" + end + + # @rbs (String message) -> void + def warn(message) + @out << 'warning: ' << message << "\n" + end + + # @rbs (String message) -> void + def error(message) + @out << 'error: ' << message << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/option_parser.rb b/tool/lrama/lib/lrama/option_parser.rb new file mode 100644 index 0000000000..5a15d59c7b --- /dev/null +++ b/tool/lrama/lib/lrama/option_parser.rb @@ -0,0 +1,223 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require 'optparse' + +module Lrama + # Handle option parsing for the command line interface. + class OptionParser + # @rbs! + # @options: Lrama::Options + # @trace: Array[String] + # @report: Array[String] + # @profile: Array[String] + + # @rbs (Array[String]) -> Lrama::Options + def self.parse(argv) + new.parse(argv) + end + + # @rbs () -> void + def initialize + @options = Options.new + @trace = [] + @report = [] + @profile = [] + end + + # @rbs (Array[String]) -> Lrama::Options + def parse(argv) + parse_by_option_parser(argv) + + @options.trace_opts = validate_trace(@trace) + @options.report_opts = validate_report(@report) + @options.profile_opts = validate_profile(@profile) + @options.grammar_file = argv.shift + + unless @options.grammar_file + abort "File should be specified\n" + end + + if @options.grammar_file == '-' + @options.grammar_file = argv.shift or abort "File name for STDIN should be specified\n" + else + @options.y = File.open(@options.grammar_file, 'r') + end + + if !@report.empty? && @options.report_file.nil? && @options.grammar_file + @options.report_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".output" + end + + if !@options.header_file && @options.header + case + when @options.outfile + @options.header_file = File.dirname(@options.outfile) + "/" + File.basename(@options.outfile, ".*") + ".h" + when @options.grammar_file + @options.header_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".h" + end + end + + @options + end + + private + + # @rbs (Array[String]) -> void + def parse_by_option_parser(argv) + ::OptionParser.new do |o| + o.banner = <<~BANNER + Lrama is LALR (1) parser generator written by Ruby. + + Usage: lrama [options] FILE + BANNER + o.separator '' + o.separator 'STDIN mode:' + o.separator 'lrama [options] - FILE read grammar from STDIN' + o.separator '' + o.separator 'Tuning the Parser:' + o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v } + o.on('-t', '--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.separator " same as '-Dparse.trace'" + o.on('--locations', 'enable location support') {|v| @options.locations = true } + o.on('-D', '--define=NAME[=VALUE]', Array, "similar to '%define NAME VALUE'") do |v| + @options.define = v.each_with_object({}) do |item, hash| # steep:ignore UnannotatedEmptyCollection + key, value = item.split('=', 2) + hash[key] = value + end + end + o.separator '' + o.separator 'Output:' + o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v } + o.on('-d', 'also produce a header file') { @options.header = true } + o.on('-r', '--report=REPORTS', Array, 'also produce details on the automaton') {|v| @report = v } + o.on_tail '' + o.on_tail 'REPORTS is a list of comma-separated words that can include:' + o.on_tail ' states describe the states' + o.on_tail ' itemsets complete the core item sets with their closure' + o.on_tail ' lookaheads explicitly associate lookahead tokens to items' + o.on_tail ' solved describe shift/reduce conflicts solving' + o.on_tail ' counterexamples, cex generate conflict counterexamples' + o.on_tail ' rules list unused rules' + o.on_tail ' terms list unused terminals' + o.on_tail ' verbose report detailed internal state and analysis results' + o.on_tail ' all include all the above reports' + o.on_tail ' none disable all reports' + o.on('--report-file=FILE', 'also produce details on the automaton output to a file named FILE') {|v| @options.report_file = v } + o.on('-o', '--output=FILE', 'leave output to FILE') {|v| @options.outfile = v } + o.on('--trace=TRACES', Array, 'also output trace logs at runtime') {|v| @trace = v } + o.on_tail '' + o.on_tail 'TRACES is a list of comma-separated words that can include:' + o.on_tail ' automaton display states' + o.on_tail ' closure display states' + o.on_tail ' rules display grammar rules' + o.on_tail ' only-explicit-rules display only explicit grammar rules' + o.on_tail ' actions display grammar rules with actions' + o.on_tail ' time display generation time' + o.on_tail ' all include all the above traces' + o.on_tail ' none disable all traces' + o.on('--diagram=[FILE]', 'generate a diagram of the rules') do |v| + @options.diagram = true + @options.diagram_file = v if v + end + o.on('--profile=PROFILES', Array, 'profiles parser generation parts') {|v| @profile = v } + o.on_tail '' + o.on_tail 'PROFILES is a list of comma-separated words that can include:' + o.on_tail ' call-stack use sampling call-stack profiler (stackprof gem)' + o.on_tail ' memory use memory profiler (memory_profiler gem)' + o.on('-v', '--verbose', "same as '--report=state'") {|_v| @report << 'states' } + o.separator '' + o.separator 'Diagnostics:' + o.on('-W', '--warnings', 'report the warnings') {|v| @options.warnings = true } + o.separator '' + o.separator 'Error Recovery:' + o.on('-e', 'enable error recovery') {|v| @options.error_recovery = true } + o.separator '' + o.separator 'Other options:' + o.on('-V', '--version', "output version information and exit") {|v| puts "lrama #{Lrama::VERSION}"; exit 0 } + o.on('-h', '--help', "display this help and exit") {|v| puts o; exit 0 } + o.on_tail + o.parse!(argv) + end + end + + ALIASED_REPORTS = { cex: :counterexamples }.freeze #: Hash[Symbol, Symbol] + VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze #: Array[Symbol] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_report(report) + h = { grammar: true } + return h if report.empty? + return {} if report == ['none'] + if report == ['all'] + VALID_REPORTS.each { |r| h[r] = true } + return h + end + + report.each do |r| + aliased = aliased_report_option(r) + if VALID_REPORTS.include?(aliased) + h[aliased] = true + else + raise "Invalid report option \"#{r}\"." + end + end + + return h + end + + # @rbs (String) -> Symbol + def aliased_report_option(opt) + (ALIASED_REPORTS[opt.to_sym] || opt).to_sym + end + + VALID_TRACES = %w[ + locations scan parse automaton bitsets closure + grammar rules only-explicit-rules actions resource + sets muscles tools m4-early m4 skeleton time ielr cex + ].freeze #: Array[String] + NOT_SUPPORTED_TRACES = %w[ + locations scan parse bitsets grammar resource + sets muscles tools m4-early m4 skeleton ielr cex + ].freeze #: Array[String] + SUPPORTED_TRACES = VALID_TRACES - NOT_SUPPORTED_TRACES #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_trace(trace) + h = {} #: Hash[Symbol, bool] + return h if trace.empty? || trace == ['none'] + all_traces = SUPPORTED_TRACES - %w[only-explicit-rules] + if trace == ['all'] + all_traces.each { |t| h[t.gsub(/-/, '_').to_sym] = true } + return h + end + + trace.each do |t| + if SUPPORTED_TRACES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid trace option \"#{t}\".\nValid options are [#{SUPPORTED_TRACES.join(", ")}]." + end + end + + return h + end + + VALID_PROFILES = %w[call-stack memory].freeze #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_profile(profile) + h = {} #: Hash[Symbol, bool] + return h if profile.empty? + + profile.each do |t| + if VALID_PROFILES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid profile option \"#{t}\".\nValid options are [#{VALID_PROFILES.join(", ")}]." + end + end + + return h + end + end +end diff --git a/tool/lrama/lib/lrama/options.rb b/tool/lrama/lib/lrama/options.rb new file mode 100644 index 0000000000..87aec62448 --- /dev/null +++ b/tool/lrama/lib/lrama/options.rb @@ -0,0 +1,46 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Command line options. + class Options + attr_accessor :skeleton #: String + attr_accessor :locations #: bool + attr_accessor :header #: bool + attr_accessor :header_file #: String? + attr_accessor :report_file #: String? + attr_accessor :outfile #: String + attr_accessor :error_recovery #: bool + attr_accessor :grammar_file #: String + attr_accessor :trace_opts #: Hash[Symbol, bool]? + attr_accessor :report_opts #: Hash[Symbol, bool]? + attr_accessor :warnings #: bool + attr_accessor :y #: IO + attr_accessor :debug #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :diagram #: bool + attr_accessor :diagram_file #: String + attr_accessor :profile_opts #: Hash[Symbol, bool]? + + # @rbs () -> void + def initialize + @skeleton = "bison/yacc.c" + @locations = false + @define = {} + @header = false + @header_file = nil + @report_file = nil + @outfile = "y.tab.c" + @error_recovery = false + @grammar_file = '' + @trace_opts = nil + @report_opts = nil + @warnings = false + @y = STDIN + @debug = false + @diagram = false + @diagram_file = "diagram.html" + @profile_opts = nil + end + end +end diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb new file mode 100644 index 0000000000..d527be8bd4 --- /dev/null +++ b/tool/lrama/lib/lrama/output.rb @@ -0,0 +1,452 @@ +# frozen_string_literal: true + +require "forwardable" +require_relative "tracer/duration" + +module Lrama + class Output + extend Forwardable + include Tracer::Duration + + attr_reader :grammar_file_path, :context, :grammar, :error_recovery, :include_header + + def_delegators "@context", :yyfinal, :yylast, :yyntokens, :yynnts, :yynrules, :yynstates, + :yymaxutok, :yypact_ninf, :yytable_ninf + + def_delegators "@grammar", :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + + def initialize( + out:, output_file_path:, template_name:, grammar_file_path:, + context:, grammar:, header_out: nil, header_file_path: nil, error_recovery: false + ) + @out = out + @output_file_path = output_file_path + @template_name = template_name + @grammar_file_path = grammar_file_path + @header_out = header_out + @header_file_path = header_file_path + @context = context + @grammar = grammar + @error_recovery = error_recovery + @include_header = header_file_path ? header_file_path.sub("./", "") : nil + end + + if ERB.instance_method(:initialize).parameters.last.first == :key + def self.erb(input) + ERB.new(input, trim_mode: '-') + end + else + def self.erb(input) + ERB.new(input, nil, '-') + end + end + + def render_partial(file) + ERB.render(partial_file(file), context: @context, output: self) + end + + def render + report_duration(:render) do + tmp = eval_template(template_file, @output_file_path) + @out << tmp + + if @header_file_path + tmp = eval_template(header_template_file, @header_file_path) + + if @header_out + @header_out << tmp + else + File.write(@header_file_path, tmp) + end + end + end + end + + # A part of b4_token_enums + def token_enums + @context.yytokentype.map do |s_value, token_id, display_name| + s = sprintf("%s = %d%s", s_value, token_id, token_id == yymaxutok ? "" : ",") + + if display_name + sprintf(" %-30s /* %s */\n", s, display_name) + else + sprintf(" %s\n", s) + end + end.join + end + + # b4_symbol_enum + def symbol_enum + last_sym_number = @context.yysymbol_kind_t.last[1] + @context.yysymbol_kind_t.map do |s_value, sym_number, display_name| + s = sprintf("%s = %d%s", s_value, sym_number, (sym_number == last_sym_number) ? "" : ",") + + if display_name + sprintf(" %-40s /* %s */\n", s, display_name) + else + sprintf(" %s\n", s) + end + end.join + end + + def yytranslate + int_array_to_string(@context.yytranslate) + end + + def yytranslate_inverted + int_array_to_string(@context.yytranslate_inverted) + end + + def yyrline + int_array_to_string(@context.yyrline) + end + + def yytname + string_array_to_string(@context.yytname) + " YY_NULLPTR" + end + + # b4_int_type_for + def int_type_for(ary) + min = ary.min + max = ary.max + + case + when (-127 <= min && min <= 127) && (-127 <= max && max <= 127) + "yytype_int8" + when (0 <= min && min <= 255) && (0 <= max && max <= 255) + "yytype_uint8" + when (-32767 <= min && min <= 32767) && (-32767 <= max && max <= 32767) + "yytype_int16" + when (0 <= min && min <= 65535) && (0 <= max && max <= 65535) + "yytype_uint16" + else + "int" + end + end + + def symbol_actions_for_printer + @grammar.symbols.map do |sym| + next unless sym.printer + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.printer.lineno} "#{@grammar_file_path}" + {#{sym.printer.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + def symbol_actions_for_destructor + @grammar.symbols.map do |sym| + next unless sym.destructor + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.destructor.lineno} "#{@grammar_file_path}" + {#{sym.destructor.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + # b4_user_initial_action + def user_initial_action(comment = "") + return "" unless @grammar.initial_action + + <<-STR + #{comment} +#line #{@grammar.initial_action.line} "#{@grammar_file_path}" + {#{@grammar.initial_action.translated_code}} + STR + end + + def after_shift_function(comment = "") + return "" unless @grammar.after_shift + + <<-STR + #{comment} +#line #{@grammar.after_shift.line} "#{@grammar_file_path}" + {#{@grammar.after_shift.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def before_reduce_function(comment = "") + return "" unless @grammar.before_reduce + + <<-STR + #{comment} +#line #{@grammar.before_reduce.line} "#{@grammar_file_path}" + {#{@grammar.before_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_reduce_function(comment = "") + return "" unless @grammar.after_reduce + + <<-STR + #{comment} +#line #{@grammar.after_reduce.line} "#{@grammar_file_path}" + {#{@grammar.after_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_shift_error_token_function(comment = "") + return "" unless @grammar.after_shift_error_token + + <<-STR + #{comment} +#line #{@grammar.after_shift_error_token.line} "#{@grammar_file_path}" + {#{@grammar.after_shift_error_token.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def after_pop_stack_function(len, comment = "") + return "" unless @grammar.after_pop_stack + + <<-STR + #{comment} +#line #{@grammar.after_pop_stack.line} "#{@grammar_file_path}" + {#{@grammar.after_pop_stack.s_value}(#{len}#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def symbol_actions_for_error_token + @grammar.symbols.map do |sym| + next unless sym.error_token + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.error_token.lineno} "#{@grammar_file_path}" + {#{sym.error_token.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + # b4_user_actions + def user_actions + action = @context.states.rules.map do |rule| + next unless rule.token_code + + code = rule.token_code + spaces = " " * (code.column - 1) + + <<-STR + case #{rule.id + 1}: /* #{rule.as_comment} */ +#line #{code.line} "#{@grammar_file_path}" +#{spaces}{#{rule.translated_code}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + + action + <<-STR + +#line [@oline@] [@ofile@] + STR + end + + def omit_blanks(param) + param.strip + end + + # b4_parse_param + def parse_param + if @grammar.parse_param + omit_blanks(@grammar.parse_param) + else + "" + end + end + + def lex_param + if @grammar.lex_param + omit_blanks(@grammar.lex_param) + else + "" + end + end + + # b4_user_formals + def user_formals + if @grammar.parse_param + ", #{parse_param}" + else + "" + end + end + + # b4_user_args + def user_args + if @grammar.parse_param + ", #{parse_param_name}" + else + "" + end + end + + def extract_param_name(param) + param[/\b([a-zA-Z0-9_]+)(?=\s*\z)/] + end + + def parse_param_name + if @grammar.parse_param + extract_param_name(parse_param) + else + "" + end + end + + def lex_param_name + if @grammar.lex_param + extract_param_name(lex_param) + else + "" + end + end + + # b4_parse_param_use + def parse_param_use(val, loc) + str = <<-STR.dup + YY_USE (#{val}); + YY_USE (#{loc}); + STR + + if @grammar.parse_param + str << " YY_USE (#{parse_param_name});" + end + + str + end + + # b4_yylex_formals + def yylex_formals + ary = ["&yylval"] + ary << "&yylloc" if @grammar.locations + + if @grammar.lex_param + ary << lex_param_name + end + + "(#{ary.join(', ')})" + end + + # b4_table_value_equals + def table_value_equals(table, value, literal, symbol) + if literal < table.min || table.max < literal + "0" + else + "((#{value}) == #{symbol})" + end + end + + # b4_yyerror_args + def yyerror_args + ary = ["&yylloc"] + + if @grammar.parse_param + ary << parse_param_name + end + + "#{ary.join(', ')}" + end + + def template_basename + File.basename(template_file) + end + + def aux + @grammar.aux + end + + def int_array_to_string(ary) + last = ary.count - 1 + + ary.each_with_index.each_slice(10).map do |slice| + " " + slice.map { |e, i| sprintf("%6d%s", e, (i == last) ? "" : ",") }.join + end.join("\n") + end + + def spec_mapped_header_file + @header_file_path + end + + def b4_cpp_guard__b4_spec_mapped_header_file + if @header_file_path + "YY_YY_" + @header_file_path.gsub(/[^a-zA-Z_0-9]+/, "_").upcase + "_INCLUDED" + else + "" + end + end + + # b4_percent_code_get + def percent_code(name) + @grammar.percent_codes.select do |percent_code| + percent_code.name == name + end.map do |percent_code| + percent_code.code + end.join + end + + private + + def eval_template(file, path) + tmp = ERB.render(file, context: @context, output: self) + replace_special_variables(tmp, path) + end + + def template_file + File.join(template_dir, @template_name) + end + + def header_template_file + File.join(template_dir, "bison/yacc.h") + end + + def partial_file(file) + File.join(template_dir, file) + end + + def template_dir + File.expand_path('../../template', __dir__) + end + + def string_array_to_string(ary) + result = "" + tmp = " " + + ary.each do |s| + replaced = s.gsub('\\', '\\\\\\\\').gsub('"', '\\"') + if (tmp + replaced + " \"\",").length > 75 + result = "#{result}#{tmp}\n" + tmp = " \"#{replaced}\"," + else + tmp = "#{tmp} \"#{replaced}\"," + end + end + + result + tmp + end + + def replace_special_variables(str, ofile) + str.each_line.with_index(1).map do |line, i| + line.gsub!("[@oline@]", (i + 1).to_s) + line.gsub!("[@ofile@]", "\"#{ofile}\"") + line + end.join + end + end +end diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb new file mode 100644 index 0000000000..20c3ad347f --- /dev/null +++ b/tool/lrama/lib/lrama/parser.rb @@ -0,0 +1,2275 @@ +# frozen_string_literal: true +# +# DO NOT MODIFY!!!! +# This file is automatically generated by Racc 1.8.1 +# from Racc grammar file "parser.y". +# + +###### racc/parser.rb begin +unless $".find {|p| p.end_with?('/racc/parser.rb')} +$".push "#{__dir__}/racc/parser.rb" +self.class.module_eval(<<'...end racc/parser.rb/module_eval...', 'racc/parser.rb', 1) +#-- +# Copyright (c) 1999-2006 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the same terms of ruby. +# +# As a special exception, when this code is copied by Racc +# into a Racc output file, you may use that output file +# without restriction. +#++ + +unless $".find {|p| p.end_with?('/racc/info.rb')} +$".push "#{__dir__}/racc/info.rb" + +module Racc + VERSION = '1.8.1' + Version = VERSION + Copyright = 'Copyright (c) 1999-2006 Minero Aoki' +end + +end + + +module Racc + class ParseError < StandardError; end +end +unless defined?(::ParseError) + ParseError = Racc::ParseError # :nodoc: +end + +# Racc is an LALR(1) parser generator. +# It is written in Ruby itself, and generates Ruby programs. +# +# == Command-line Reference +# +# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] +# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] +# [-v] [--verbose] +# [-O<var>filename</var>] [--log-file=<var>filename</var>] +# [-g] [--debug] +# [-E] [--embedded] +# [-l] [--no-line-convert] +# [-c] [--line-convert-all] +# [-a] [--no-omit-actions] +# [-C] [--check-only] +# [-S] [--output-status] +# [--version] [--copyright] [--help] <var>grammarfile</var> +# +# [+grammarfile+] +# Racc grammar file. Any extension is permitted. +# [-o+outfile+, --output-file=+outfile+] +# A filename for output. default is <+filename+>.tab.rb +# [-O+filename+, --log-file=+filename+] +# Place logging output in file +filename+. +# Default log file name is <+filename+>.output. +# [-e+rubypath+, --executable=+rubypath+] +# output executable file(mode 755). where +path+ is the Ruby interpreter. +# [-v, --verbose] +# verbose mode. create +filename+.output file, like yacc's y.output file. +# [-g, --debug] +# add debug code to parser class. To display debugging information, +# use this '-g' option and set @yydebug true in parser class. +# [-E, --embedded] +# Output parser which doesn't need runtime files (racc/parser.rb). +# [-F, --frozen] +# Output parser which declares frozen_string_literals: true +# [-C, --check-only] +# Check syntax of racc grammar file and quit. +# [-S, --output-status] +# Print messages time to time while compiling. +# [-l, --no-line-convert] +# turns off line number converting. +# [-c, --line-convert-all] +# Convert line number of actions, inner, header and footer. +# [-a, --no-omit-actions] +# Call all actions, even if an action is empty. +# [--version] +# print Racc version and quit. +# [--copyright] +# Print copyright and quit. +# [--help] +# Print usage and quit. +# +# == Generating Parser Using Racc +# +# To compile Racc grammar file, simply type: +# +# $ racc parse.y +# +# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. +# +# == Writing A Racc Grammar File +# +# If you want your own parser, you have to write a grammar file. +# A grammar file contains the name of your parser class, grammar for the parser, +# user code, and anything else. +# When writing a grammar file, yacc's knowledge is helpful. +# If you have not used yacc before, Racc is not too difficult. +# +# Here's an example Racc grammar file. +# +# class Calcparser +# rule +# target: exp { print val[0] } +# +# exp: exp '+' exp +# | exp '*' exp +# | '(' exp ')' +# | NUMBER +# end +# +# Racc grammar files resemble yacc files. +# But (of course), this is Ruby code. +# yacc's $$ is the 'result', $0, $1... is +# an array called 'val', and $-1, $-2... is an array called '_values'. +# +# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for +# more information on grammar files. +# +# == Parser +# +# Then you must prepare the parse entry method. There are two types of +# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse +# +# Racc::Parser#do_parse is simple. +# +# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). +# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. +# EOF is [false, false]. +# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. +# If you want to change this, see the grammar reference. +# +# Racc::Parser#yyparse is little complicated, but useful. +# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. +# +# For example, <code>yyparse(obj, :scan)</code> causes +# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. +# +# == Debugging +# +# When debugging, "-v" or/and the "-g" option is helpful. +# +# "-v" creates verbose log file (.output). +# "-g" creates a "Verbose Parser". +# Verbose Parser prints the internal status when parsing. +# But it's _not_ automatic. +# You must use -g option and set +@yydebug+ to +true+ in order to get output. +# -g option only creates the verbose parser. +# +# === Racc reported syntax error. +# +# Isn't there too many "end"? +# grammar of racc file is changed in v0.10. +# +# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. +# +# === Racc reported "XXXX conflicts". +# +# Try "racc -v xxxx.y". +# It causes producing racc's internal log file, xxxx.output. +# +# === Generated parsers does not work correctly +# +# Try "racc -g xxxx.y". +# This command let racc generate "debugging parser". +# Then set @yydebug=true in your parser. +# It produces a working log of your parser. +# +# == Re-distributing Racc runtime +# +# A parser, which is created by Racc, requires the Racc runtime module; +# racc/parser.rb. +# +# Ruby 1.8.x comes with Racc runtime module, +# you need NOT distribute Racc runtime files. +# +# If you want to include the Racc runtime module with your parser. +# This can be done by using '-E' option: +# +# $ racc -E -omyparser.rb myparser.y +# +# This command creates myparser.rb which `includes' Racc runtime. +# Only you must do is to distribute your parser file (myparser.rb). +# +# Note: parser.rb is ruby license, but your parser is not. +# Your own parser is completely yours. +module Racc + + unless defined?(Racc_No_Extensions) + Racc_No_Extensions = false # :nodoc: + end + + class Parser + + Racc_Runtime_Version = ::Racc::VERSION + Racc_Runtime_Core_Version_R = ::Racc::VERSION + + begin + if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' + require 'jruby' + require 'racc/cparse-jruby.jar' + com.headius.racc.Cparse.new.load(JRuby.runtime, false) + else + require 'racc/cparse' + end + + unless new.respond_to?(:_racc_do_parse_c, true) + raise LoadError, 'old cparse.so' + end + if Racc_No_Extensions + raise LoadError, 'selecting ruby version of racc runtime core' + end + + Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: + Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: + Racc_Runtime_Type = 'c' # :nodoc: + rescue LoadError + Racc_Main_Parsing_Routine = :_racc_do_parse_rb + Racc_YY_Parse_Method = :_racc_yyparse_rb + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R + Racc_Runtime_Type = 'ruby' + end + + def Parser.racc_runtime_type # :nodoc: + Racc_Runtime_Type + end + + def _racc_setup + @yydebug = false unless self.class::Racc_debug_parser + @yydebug = false unless defined?(@yydebug) + if @yydebug + @racc_debug_out = $stderr unless defined?(@racc_debug_out) + @racc_debug_out ||= $stderr + end + arg = self.class::Racc_arg + arg[13] = true if arg.size < 14 + arg + end + + def _racc_init_sysvars + @racc_state = [0] + @racc_tstack = [] + @racc_vstack = [] + + @racc_t = nil + @racc_val = nil + + @racc_read_next = true + + @racc_user_yyerror = false + @racc_error_status = 0 + end + + # The entry point of the parser. This method is used with #next_token. + # If Racc wants to get token (and its value), calls next_token. + # + # Example: + # def parse + # @q = [[1,1], + # [2,2], + # [3,3], + # [false, '$']] + # do_parse + # end + # + # def next_token + # @q.shift + # end + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def do_parse + #{Racc_Main_Parsing_Routine}(_racc_setup(), false) + end + RUBY + + # The method to fetch next token. + # If you use #do_parse method, you must implement #next_token. + # + # The format of return value is [TOKEN_SYMBOL, VALUE]. + # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT + # for 'IDENT'. ";" (String) for ';'. + # + # The final symbol (End of file) must be false. + def next_token + raise NotImplementedError, "#{self.class}\#next_token is not defined" + end + + def _racc_do_parse_rb(arg, in_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + tok = act = i = nil + + catch(:racc_end_parse) { + while true + if i = action_pointer[@racc_state[-1]] + if @racc_read_next + if @racc_t != 0 # not EOF + tok, @racc_val = next_token() + unless tok # EOF + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + racc_read_token(@racc_t, tok, @racc_val) if @yydebug + @racc_read_next = false + end + end + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + else + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + } + end + + # Another entry point for the parser. + # If you use this method, you must implement RECEIVER#METHOD_ID method. + # + # RECEIVER#METHOD_ID is a method to get next token. + # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def yyparse(recv, mid) + #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) + end + RUBY + + def _racc_yyparse_rb(recv, mid, arg, c_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + + catch(:racc_end_parse) { + until i = action_pointer[@racc_state[-1]] + while act = _racc_evalact(action_default[@racc_state[-1]], arg) + ; + end + end + recv.__send__(mid) do |tok, val| + unless tok + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + @racc_val = val + @racc_read_next = false + + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + + while !(i = action_pointer[@racc_state[-1]]) || + ! @racc_read_next || + @racc_t == 0 # $ + unless i and i += @racc_t and + i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + end + } + end + + ### + ### common + ### + + def _racc_evalact(act, arg) + action_table, action_check, _, action_pointer, + _, _, _, _, + _, _, _, shift_n, + reduce_n, * = arg + nerr = 0 # tmp + + if act > 0 and act < shift_n + # + # shift + # + if @racc_error_status > 0 + @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF + end + @racc_vstack.push @racc_val + @racc_state.push act + @racc_read_next = true + if @yydebug + @racc_tstack.push @racc_t + racc_shift @racc_t, @racc_tstack, @racc_vstack + end + + elsif act < 0 and act > -reduce_n + # + # reduce + # + code = catch(:racc_jump) { + @racc_state.push _racc_do_reduce(arg, act) + false + } + if code + case code + when 1 # yyerror + @racc_user_yyerror = true # user_yyerror + return -reduce_n + when 2 # yyaccept + return shift_n + else + raise '[Racc Bug] unknown jump code' + end + end + + elsif act == shift_n + # + # accept + # + racc_accept if @yydebug + throw :racc_end_parse, @racc_vstack[0] + + elsif act == -reduce_n + # + # error + # + case @racc_error_status + when 0 + unless arg[21] # user_yyerror + nerr += 1 + on_error @racc_t, @racc_val, @racc_vstack + end + when 3 + if @racc_t == 0 # is $ + # We're at EOF, and another error occurred immediately after + # attempting auto-recovery + throw :racc_end_parse, nil + end + @racc_read_next = true + end + @racc_user_yyerror = false + @racc_error_status = 3 + while true + if i = action_pointer[@racc_state[-1]] + i += 1 # error token + if i >= 0 and + (act = action_table[i]) and + action_check[i] == @racc_state[-1] + break + end + end + throw :racc_end_parse, nil if @racc_state.size <= 1 + @racc_state.pop + @racc_vstack.pop + if @yydebug + @racc_tstack.pop + racc_e_pop @racc_state, @racc_tstack, @racc_vstack + end + end + return act + + else + raise "[Racc Bug] unknown action #{act.inspect}" + end + + racc_next_state(@racc_state[-1], @racc_state) if @yydebug + + nil + end + + def _racc_do_reduce(arg, act) + _, _, _, _, + goto_table, goto_check, goto_default, goto_pointer, + nt_base, reduce_table, _, _, + _, use_result, * = arg + + state = @racc_state + vstack = @racc_vstack + tstack = @racc_tstack + + i = act * -3 + len = reduce_table[i] + reduce_to = reduce_table[i+1] + method_id = reduce_table[i+2] + void_array = [] + + tmp_t = tstack[-len, len] if @yydebug + tmp_v = vstack[-len, len] + tstack[-len, len] = void_array if @yydebug + vstack[-len, len] = void_array + state[-len, len] = void_array + + # tstack must be updated AFTER method call + if use_result + vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) + else + vstack.push __send__(method_id, tmp_v, vstack) + end + tstack.push reduce_to + + racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug + + k1 = reduce_to - nt_base + if i = goto_pointer[k1] + i += state[-1] + if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 + return curstate + end + end + goto_default[k1] + end + + # This method is called when a parse error is found. + # + # ERROR_TOKEN_ID is an internal ID of token which caused error. + # You can get string representation of this ID by calling + # #token_to_str. + # + # ERROR_VALUE is a value of error token. + # + # value_stack is a stack of symbol values. + # DO NOT MODIFY this object. + # + # This method raises ParseError by default. + # + # If this method returns, parsers enter "error recovering mode". + def on_error(t, val, vstack) + raise ParseError, sprintf("parse error on value %s (%s)", + val.inspect, token_to_str(t) || '?') + end + + # Enter error recovering mode. + # This method does not call #on_error. + def yyerror + throw :racc_jump, 1 + end + + # Exit parser. + # Return value is +Symbol_Value_Stack[0]+. + def yyaccept + throw :racc_jump, 2 + end + + # Leave error recovering mode. + def yyerrok + @racc_error_status = 0 + end + + # For debugging output + def racc_read_token(t, tok, val) + @racc_debug_out.print 'read ' + @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' + @racc_debug_out.puts val.inspect + @racc_debug_out.puts + end + + def racc_shift(tok, tstack, vstack) + @racc_debug_out.puts "shift #{racc_token2str tok}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_reduce(toks, sim, tstack, vstack) + out = @racc_debug_out + out.print 'reduce ' + if toks.empty? + out.print ' <none>' + else + toks.each {|t| out.print ' ', racc_token2str(t) } + end + out.puts " --> #{racc_token2str(sim)}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_accept + @racc_debug_out.puts 'accept' + @racc_debug_out.puts + end + + def racc_e_pop(state, tstack, vstack) + @racc_debug_out.puts 'error recovering mode: pop token' + racc_print_states state + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_next_state(curstate, state) + @racc_debug_out.puts "goto #{curstate}" + racc_print_states state + @racc_debug_out.puts + end + + def racc_print_stacks(t, v) + out = @racc_debug_out + out.print ' [' + t.each_index do |i| + out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' + end + out.puts ' ]' + end + + def racc_print_states(s) + out = @racc_debug_out + out.print ' [' + s.each {|st| out.print ' ', st } + out.puts ' ]' + end + + def racc_token2str(tok) + self.class::Racc_token_to_s_table[tok] or + raise "[Racc Bug] can't convert token #{tok} to string" + end + + # Convert internal ID of token symbol to the string. + def token_to_str(t) + self.class::Racc_token_to_s_table[t] + end + + end + +end + +...end racc/parser.rb/module_eval... +end +###### racc/parser.rb end +module Lrama + class Parser < Racc::Parser + +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 504) + +include Lrama::Tracer::Duration + +def initialize(text, path, debug = false, locations = false, define = {}) + @path = path + @grammar_file = Lrama::Lexer::GrammarFile.new(path, text) + @yydebug = debug || define.key?('parse.trace') + @rule_counter = Lrama::Grammar::Counter.new(0) + @midrule_action_counter = Lrama::Grammar::Counter.new(1) + @locations = locations + @define = define +end + +def parse + message = "parse '#{File.basename(@path)}'" + report_duration(message) do + @lexer = Lrama::Lexer.new(@grammar_file) + @grammar = Lrama::Grammar.new(@rule_counter, @locations, @define) + @precedence_number = 0 + reset_precs + do_parse + @grammar + end +end + +def next_token + @lexer.next_token +end + +def on_error(error_token_id, error_value, value_stack) + case error_value + when Lrama::Lexer::Token::Int + location = error_value.location + value = "#{error_value.s_value}" + when Lrama::Lexer::Token::Token + location = error_value.location + value = "\"#{error_value.s_value}\"" + when Lrama::Lexer::Token::Base + location = error_value.location + value = "'#{error_value.s_value}'" + else + location = @lexer.location + value = error_value.inspect + end + + error_message = "parse error on value #{value} (#{token_to_str(error_token_id) || '?'})" + + raise_parse_error(error_message, location) +end + +def on_action_error(error_message, error_value) + if error_value.is_a?(Lrama::Lexer::Token::Base) + location = error_value.location + else + location = @lexer.location + end + + raise_parse_error(error_message, location) +end + +private + +def reset_precs + @opening_prec_seen = false + @trailing_prec_seen = false + @code_after_prec = false +end + +def prec_seen? + @opening_prec_seen || @trailing_prec_seen +end + +def begin_c_declaration(end_symbol) + @lexer.status = :c_declaration + @lexer.end_symbol = end_symbol +end + +def end_c_declaration + @lexer.status = :initial + @lexer.end_symbol = nil +end + +def raise_parse_error(error_message, location) + raise ParseError, location.generate_error_message(error_message) +end +...end parser.y/module_eval... +##### State transition tables begin ### + +racc_action_table = [ + 98, 98, 99, 99, 87, 53, 53, 52, 178, 110, + 110, 97, 53, 53, 184, 178, 110, 110, 53, 181, + 184, 162, 110, 6, 163, 181, 181, 53, 53, 52, + 52, 181, 79, 79, 53, 53, 52, 52, 43, 79, + 79, 53, 4, 52, 5, 110, 88, 94, 182, 125, + 126, 163, 100, 100, 180, 193, 194, 195, 137, 185, + 188, 180, 4, 44, 5, 185, 188, 94, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 46, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 47, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 47, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 50, + 57, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 57, + 60, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 53, 53, 52, + 52, 110, 105, 53, 53, 52, 52, 110, 105, 53, + 53, 52, 52, 110, 105, 53, 53, 52, 52, 110, + 105, 53, 53, 52, 52, 110, 110, 53, 53, 52, + 209, 110, 110, 53, 53, 209, 52, 110, 110, 53, + 53, 209, 52, 110, 193, 194, 195, 137, 216, 222, + 229, 217, 217, 217, 53, 53, 52, 52, 193, 194, + 195, 57, 57, 57, 57, 66, 67, 68, 69, 70, + 72, 72, 72, 86, 89, 47, 57, 57, 113, 117, + 117, 79, 123, 124, 131, 47, 133, 137, 139, 143, + 149, 150, 151, 152, 133, 155, 156, 157, 110, 166, + 149, 169, 172, 173, 72, 175, 176, 183, 189, 166, + 196, 137, 200, 202, 137, 166, 211, 166, 137, 72, + 176, 218, 176, 72, 72, 227, 137, 72 ] + +racc_action_check = [ + 51, 97, 51, 97, 41, 75, 165, 75, 165, 75, + 165, 51, 171, 190, 171, 190, 171, 190, 201, 165, + 201, 148, 201, 1, 148, 171, 190, 36, 37, 36, + 37, 201, 36, 37, 38, 39, 38, 39, 5, 38, + 39, 117, 0, 117, 0, 117, 41, 46, 168, 88, + 88, 168, 51, 97, 165, 177, 177, 177, 177, 171, + 171, 190, 2, 6, 2, 201, 201, 90, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 9, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 10, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 11, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 3, 3, 12, + 14, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 15, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 8, 8, 16, + 17, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 18, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 63, 13, 63, + 13, 63, 63, 64, 73, 64, 73, 64, 64, 65, + 78, 65, 78, 65, 65, 106, 79, 106, 79, 106, + 106, 118, 180, 118, 180, 118, 180, 188, 196, 188, + 196, 188, 196, 202, 217, 202, 217, 202, 217, 218, + 113, 218, 113, 218, 186, 186, 186, 186, 208, 213, + 226, 208, 213, 226, 114, 123, 114, 123, 210, 210, + 210, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 40, 42, 47, 55, 60, 71, 74, + 76, 80, 81, 87, 91, 92, 93, 94, 102, 116, + 124, 125, 126, 127, 133, 136, 137, 138, 144, 150, + 151, 153, 156, 158, 162, 163, 164, 170, 174, 176, + 178, 179, 182, 184, 187, 189, 199, 200, 204, 205, + 207, 209, 212, 214, 216, 221, 222, 228 ] + +racc_action_pointer = [ + 32, 23, 52, 93, nil, 31, 63, nil, 123, 68, + 74, 84, 103, 165, 94, 111, 123, 135, 141, nil, + nil, nil, nil, nil, 215, 216, 217, 218, 230, 231, + 232, 233, 234, 232, 233, 234, 24, 25, 31, 32, + 238, -1, 242, nil, nil, nil, 43, 232, nil, nil, + nil, -5, nil, nil, nil, 230, nil, nil, nil, nil, + 231, nil, nil, 164, 170, 176, nil, nil, nil, nil, + nil, 240, nil, 171, 241, 2, 242, nil, 177, 183, + 243, 244, nil, nil, nil, nil, nil, 209, 45, nil, + 63, 245, 242, 243, 202, nil, nil, -4, nil, nil, + nil, nil, 256, nil, nil, nil, 182, nil, nil, nil, + nil, nil, nil, 207, 221, nil, 253, 38, 188, nil, + nil, nil, nil, 222, 255, 215, 218, 252, nil, nil, + nil, nil, nil, 251, nil, nil, 219, 261, 250, nil, + nil, nil, nil, nil, 261, nil, nil, nil, -24, nil, + 219, 265, nil, 269, nil, nil, 216, nil, 256, nil, + nil, nil, 266, 270, 227, 3, nil, nil, 3, nil, + 228, 9, nil, nil, 232, nil, 229, 3, 236, 226, + 189, nil, 236, nil, 239, nil, 162, 229, 194, 235, + 10, nil, nil, nil, nil, nil, 195, nil, nil, 284, + 237, 15, 200, nil, 233, 281, nil, 241, 173, 247, + 176, nil, 243, 174, 285, nil, 286, 201, 206, nil, + nil, 278, 241, nil, nil, nil, 175, nil, 289, nil, + nil ] + +racc_action_default = [ + -1, -136, -1, -3, -10, -136, -136, -2, -3, -136, + -14, -14, -136, -136, -136, -136, -136, -136, -136, -28, + -29, -34, -35, -36, -136, -136, -136, -136, -136, -136, + -136, -136, -136, -54, -54, -54, -136, -136, -136, -136, + -136, -136, -136, -13, 231, -4, -136, -14, -16, -17, + -20, -131, -100, -101, -130, -18, -23, -89, -24, -25, + -136, -27, -37, -136, -136, -136, -41, -42, -43, -44, + -45, -46, -55, -136, -47, -136, -48, -49, -92, -136, + -95, -97, -98, -50, -51, -52, -53, -136, -136, -11, + -5, -7, -14, -136, -72, -15, -21, -131, -132, -133, + -134, -19, -136, -26, -30, -31, -32, -38, -87, -88, + -135, -39, -40, -136, -56, -58, -60, -136, -83, -85, + -93, -94, -96, -136, -136, -136, -136, -136, -6, -8, + -9, -128, -104, -102, -105, -73, -136, -136, -136, -90, + -33, -59, -57, -61, -80, -86, -84, -99, -136, -66, + -70, -136, -12, -136, -103, -109, -136, -22, -136, -62, + -81, -82, -54, -136, -64, -68, -71, -74, -136, -129, + -106, -107, -127, -91, -136, -67, -70, -72, -100, -72, + -136, -124, -136, -109, -100, -110, -72, -72, -136, -70, + -69, -75, -76, -116, -117, -118, -136, -78, -79, -136, + -70, -108, -136, -111, -72, -54, -115, -63, -136, -100, + -119, -125, -65, -136, -54, -114, -54, -136, -136, -120, + -121, -136, -72, -112, -77, -122, -136, -126, -54, -123, + -113 ] + +racc_goto_table = [ + 73, 118, 136, 54, 48, 49, 164, 96, 91, 120, + 121, 93, 187, 148, 107, 111, 112, 119, 134, 171, + 56, 58, 59, 3, 61, 7, 78, 78, 78, 78, + 62, 63, 64, 65, 115, 74, 76, 192, 1, 129, + 168, 95, 187, 118, 118, 207, 204, 201, 77, 83, + 84, 85, 128, 138, 147, 93, 212, 140, 154, 145, + 146, 101, 130, 116, 42, 127, 103, 208, 78, 78, + 219, 9, 51, 213, 141, 142, 45, 71, 159, 144, + 190, 160, 161, 102, 158, 191, 132, 197, 122, 226, + 170, 177, 220, 199, 203, 205, 221, 186, 153, nil, + nil, nil, nil, 116, 116, nil, 198, nil, nil, nil, + nil, nil, 214, 78, 206, nil, 177, nil, nil, nil, + nil, nil, 210, nil, nil, nil, nil, 186, 210, 174, + 228, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 225, 210, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 215, nil, nil, nil, nil, nil, nil, nil, + nil, 223, nil, 224, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 230 ] + +racc_goto_check = [ + 29, 22, 42, 31, 14, 14, 35, 16, 8, 48, + 48, 13, 40, 34, 24, 24, 24, 45, 52, 54, + 18, 18, 18, 6, 17, 6, 31, 31, 31, 31, + 17, 17, 17, 17, 30, 26, 26, 38, 1, 5, + 34, 14, 40, 22, 22, 35, 38, 54, 27, 27, + 27, 27, 8, 16, 48, 13, 35, 24, 52, 45, + 45, 18, 9, 31, 10, 11, 17, 39, 31, 31, + 38, 7, 15, 39, 30, 30, 7, 25, 32, 33, + 36, 43, 44, 46, 47, 42, 14, 42, 50, 39, + 53, 22, 55, 56, 42, 42, 57, 22, 58, nil, + nil, nil, nil, 31, 31, nil, 22, nil, nil, nil, + nil, nil, 42, 31, 22, nil, 22, nil, nil, nil, + nil, nil, 22, nil, nil, nil, nil, 22, 22, 29, + 42, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 22, 22, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 29, nil, nil, nil, nil, nil, nil, nil, + nil, 29, nil, 29, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 29 ] + +racc_goto_pointer = [ + nil, 38, nil, nil, nil, -52, 23, 68, -38, -29, + 60, -24, nil, -35, -6, 59, -44, 6, 6, nil, + nil, nil, -74, nil, -49, 44, 1, 12, nil, -33, + -39, -10, -66, -37, -111, -144, -96, nil, -140, -129, + -159, nil, -92, -63, -62, -58, 26, -55, -69, nil, + 8, nil, -75, -65, -136, -118, -88, -115, -33 ] + +racc_goto_default = [ + nil, nil, 2, 8, 90, nil, nil, nil, nil, nil, + nil, nil, 10, 11, nil, nil, nil, 55, nil, 21, + 22, 23, 104, 106, nil, nil, nil, nil, 114, 75, + nil, 108, nil, nil, nil, nil, 165, 135, nil, nil, + 179, 167, nil, 109, nil, nil, nil, nil, 81, 80, + 82, 92, nil, nil, nil, nil, nil, nil, nil ] + +racc_reduce_table = [ + 0, 0, :racc_error, + 0, 64, :_reduce_1, + 2, 64, :_reduce_2, + 0, 65, :_reduce_3, + 2, 65, :_reduce_4, + 1, 66, :_reduce_5, + 2, 66, :_reduce_6, + 0, 67, :_reduce_none, + 1, 67, :_reduce_none, + 5, 59, :_reduce_none, + 0, 68, :_reduce_10, + 0, 69, :_reduce_11, + 5, 60, :_reduce_12, + 2, 60, :_reduce_13, + 0, 72, :_reduce_14, + 2, 72, :_reduce_15, + 2, 61, :_reduce_none, + 2, 61, :_reduce_none, + 1, 76, :_reduce_18, + 2, 76, :_reduce_19, + 2, 70, :_reduce_20, + 3, 70, :_reduce_21, + 5, 70, :_reduce_22, + 2, 70, :_reduce_none, + 2, 70, :_reduce_24, + 2, 70, :_reduce_25, + 3, 70, :_reduce_26, + 2, 70, :_reduce_27, + 1, 70, :_reduce_28, + 1, 70, :_reduce_29, + 1, 81, :_reduce_30, + 1, 81, :_reduce_31, + 1, 82, :_reduce_32, + 2, 82, :_reduce_33, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 2, 71, :_reduce_37, + 3, 71, :_reduce_38, + 3, 71, :_reduce_39, + 3, 71, :_reduce_40, + 2, 71, :_reduce_41, + 2, 71, :_reduce_42, + 2, 71, :_reduce_43, + 2, 71, :_reduce_44, + 2, 71, :_reduce_45, + 2, 77, :_reduce_none, + 2, 77, :_reduce_47, + 2, 77, :_reduce_48, + 2, 77, :_reduce_49, + 2, 77, :_reduce_50, + 2, 77, :_reduce_51, + 2, 77, :_reduce_52, + 2, 77, :_reduce_53, + 0, 87, :_reduce_none, + 1, 87, :_reduce_none, + 1, 88, :_reduce_56, + 2, 88, :_reduce_57, + 2, 83, :_reduce_58, + 3, 83, :_reduce_59, + 0, 91, :_reduce_none, + 1, 91, :_reduce_none, + 3, 86, :_reduce_62, + 8, 78, :_reduce_63, + 5, 79, :_reduce_64, + 8, 79, :_reduce_65, + 1, 92, :_reduce_66, + 3, 92, :_reduce_67, + 1, 93, :_reduce_68, + 3, 93, :_reduce_69, + 0, 99, :_reduce_none, + 1, 99, :_reduce_none, + 0, 100, :_reduce_none, + 1, 100, :_reduce_none, + 1, 94, :_reduce_74, + 3, 94, :_reduce_75, + 3, 94, :_reduce_76, + 6, 94, :_reduce_77, + 3, 94, :_reduce_78, + 3, 94, :_reduce_79, + 0, 102, :_reduce_none, + 1, 102, :_reduce_none, + 1, 90, :_reduce_82, + 1, 103, :_reduce_83, + 2, 103, :_reduce_84, + 2, 84, :_reduce_85, + 3, 84, :_reduce_86, + 1, 80, :_reduce_none, + 1, 80, :_reduce_none, + 0, 104, :_reduce_89, + 0, 105, :_reduce_90, + 5, 75, :_reduce_91, + 1, 106, :_reduce_92, + 2, 106, :_reduce_93, + 2, 107, :_reduce_94, + 1, 108, :_reduce_95, + 2, 108, :_reduce_96, + 1, 85, :_reduce_97, + 1, 85, :_reduce_98, + 3, 85, :_reduce_99, + 1, 89, :_reduce_none, + 1, 89, :_reduce_none, + 1, 110, :_reduce_102, + 2, 110, :_reduce_103, + 2, 62, :_reduce_none, + 2, 62, :_reduce_none, + 4, 109, :_reduce_106, + 1, 111, :_reduce_107, + 3, 111, :_reduce_108, + 0, 112, :_reduce_109, + 2, 112, :_reduce_110, + 3, 112, :_reduce_111, + 5, 112, :_reduce_112, + 7, 112, :_reduce_113, + 4, 112, :_reduce_114, + 3, 112, :_reduce_115, + 1, 96, :_reduce_116, + 1, 96, :_reduce_117, + 1, 96, :_reduce_118, + 0, 113, :_reduce_none, + 1, 113, :_reduce_none, + 2, 97, :_reduce_121, + 3, 97, :_reduce_122, + 4, 97, :_reduce_123, + 0, 114, :_reduce_124, + 0, 115, :_reduce_125, + 5, 98, :_reduce_126, + 3, 95, :_reduce_127, + 0, 116, :_reduce_128, + 3, 63, :_reduce_129, + 1, 73, :_reduce_none, + 0, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 101, :_reduce_135 ] + +racc_reduce_n = 136 + +racc_shift_n = 231 + +racc_token_table = { + false => 0, + :error => 1, + :C_DECLARATION => 2, + :CHARACTER => 3, + :IDENT_COLON => 4, + :IDENTIFIER => 5, + :INTEGER => 6, + :STRING => 7, + :TAG => 8, + "%%" => 9, + "%{" => 10, + "%}" => 11, + "%require" => 12, + ";" => 13, + "%expect" => 14, + "%define" => 15, + "{" => 16, + "}" => 17, + "%param" => 18, + "%lex-param" => 19, + "%parse-param" => 20, + "%code" => 21, + "%initial-action" => 22, + "%no-stdlib" => 23, + "%locations" => 24, + "%union" => 25, + "%destructor" => 26, + "%printer" => 27, + "%error-token" => 28, + "%after-shift" => 29, + "%before-reduce" => 30, + "%after-reduce" => 31, + "%after-shift-error-token" => 32, + "%after-pop-stack" => 33, + "-temp-group" => 34, + "%token" => 35, + "%type" => 36, + "%nterm" => 37, + "%left" => 38, + "%right" => 39, + "%precedence" => 40, + "%nonassoc" => 41, + "%start" => 42, + "%rule" => 43, + "(" => 44, + ")" => 45, + ":" => 46, + "%inline" => 47, + "," => 48, + "|" => 49, + "%empty" => 50, + "%prec" => 51, + "?" => 52, + "+" => 53, + "*" => 54, + "[" => 55, + "]" => 56, + "{...}" => 57 } + +racc_nt_base = 58 + +racc_use_result_var = true + +Racc_arg = [ + racc_action_table, + racc_action_check, + racc_action_default, + racc_action_pointer, + racc_goto_table, + racc_goto_check, + racc_goto_default, + racc_goto_pointer, + racc_nt_base, + racc_reduce_table, + racc_token_table, + racc_shift_n, + racc_reduce_n, + racc_use_result_var ] +Ractor.make_shareable(Racc_arg) if defined?(Ractor) + +Racc_token_to_s_table = [ + "$end", + "error", + "C_DECLARATION", + "CHARACTER", + "IDENT_COLON", + "IDENTIFIER", + "INTEGER", + "STRING", + "TAG", + "\"%%\"", + "\"%{\"", + "\"%}\"", + "\"%require\"", + "\";\"", + "\"%expect\"", + "\"%define\"", + "\"{\"", + "\"}\"", + "\"%param\"", + "\"%lex-param\"", + "\"%parse-param\"", + "\"%code\"", + "\"%initial-action\"", + "\"%no-stdlib\"", + "\"%locations\"", + "\"%union\"", + "\"%destructor\"", + "\"%printer\"", + "\"%error-token\"", + "\"%after-shift\"", + "\"%before-reduce\"", + "\"%after-reduce\"", + "\"%after-shift-error-token\"", + "\"%after-pop-stack\"", + "\"-temp-group\"", + "\"%token\"", + "\"%type\"", + "\"%nterm\"", + "\"%left\"", + "\"%right\"", + "\"%precedence\"", + "\"%nonassoc\"", + "\"%start\"", + "\"%rule\"", + "\"(\"", + "\")\"", + "\":\"", + "\"%inline\"", + "\",\"", + "\"|\"", + "\"%empty\"", + "\"%prec\"", + "\"?\"", + "\"+\"", + "\"*\"", + "\"[\"", + "\"]\"", + "\"{...}\"", + "$start", + "input", + "prologue_declaration", + "bison_declaration", + "rules_or_grammar_declaration", + "epilogue_declaration", + "\"-many@prologue_declaration\"", + "\"-many@bison_declaration\"", + "\"-many1@rules_or_grammar_declaration\"", + "\"-option@epilogue_declaration\"", + "@1", + "@2", + "parser_option", + "grammar_declaration", + "\"-many@;\"", + "variable", + "value", + "param", + "\"-many1@param\"", + "symbol_declaration", + "rule_declaration", + "inline_declaration", + "symbol", + "\"-group@symbol|TAG\"", + "\"-many1@-group@symbol|TAG\"", + "token_declarations", + "symbol_declarations", + "token_declarations_for_precedence", + "token_declaration", + "\"-option@TAG\"", + "\"-many1@token_declaration\"", + "id", + "alias", + "\"-option@INTEGER\"", + "rule_args", + "rule_rhs_list", + "rule_rhs", + "named_ref", + "parameterized_suffix", + "parameterized_args", + "action", + "\"-option@%empty\"", + "\"-option@named_ref\"", + "string_as_id", + "\"-option@string_as_id\"", + "\"-many1@symbol\"", + "@3", + "@4", + "\"-many1@id\"", + "\"-group@TAG-\\\"-many1@id\\\"\"", + "\"-many1@-group@TAG-\\\"-many1@id\\\"\"", + "rules", + "\"-many1@;\"", + "rhs_list", + "rhs", + "\"-option@parameterized_suffix\"", + "@5", + "@6", + "@7" ] +Ractor.make_shareable(Racc_token_to_s_table) if defined?(Ractor) + +Racc_debug_parser = true + +##### State transition tables end ##### + +# reduce 0 omitted + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_1(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_2(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_3(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_4(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_5(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_6(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 7 omitted + +# reduce 8 omitted + +# reduce 9 omitted + +module_eval(<<'.,.,', 'parser.y', 13) + def _reduce_10(val, _values, result) + begin_c_declaration("%}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 17) + def _reduce_11(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 21) + def _reduce_12(val, _values, result) + @grammar.prologue_first_lineno = val[0].first_line + @grammar.prologue = val[2].s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 26) + def _reduce_13(val, _values, result) + @grammar.required = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_14(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_15(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 16 omitted + +# reduce 17 omitted + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_18(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_19(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 36) + def _reduce_20(val, _values, result) + @grammar.expect = val[1].s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 40) + def _reduce_21(val, _values, result) + @grammar.define[val[1].s_value] = val[2]&.s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 44) + def _reduce_22(val, _values, result) + @grammar.define[val[1].s_value] = val[3]&.s_value + + result + end +.,., + +# reduce 23 omitted + +module_eval(<<'.,.,', 'parser.y', 49) + def _reduce_24(val, _values, result) + val[1].each {|token| + @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 55) + def _reduce_25(val, _values, result) + val[1].each {|token| + @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 61) + def _reduce_26(val, _values, result) + @grammar.add_percent_code(id: val[1], code: val[2]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 65) + def _reduce_27(val, _values, result) + @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 69) + def _reduce_28(val, _values, result) + @grammar.no_stdlib = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 73) + def _reduce_29(val, _values, result) + @grammar.locations = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_30(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_31(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_32(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_33(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 34 omitted + +# reduce 35 omitted + +# reduce 36 omitted + +module_eval(<<'.,.,', 'parser.y', 82) + def _reduce_37(val, _values, result) + @grammar.set_union( + Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), + val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 89) + def _reduce_38(val, _values, result) + @grammar.add_destructor( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 97) + def _reduce_39(val, _values, result) + @grammar.add_printer( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 105) + def _reduce_40(val, _values, result) + @grammar.add_error_token( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 113) + def _reduce_41(val, _values, result) + @grammar.after_shift = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 117) + def _reduce_42(val, _values, result) + @grammar.before_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 121) + def _reduce_43(val, _values, result) + @grammar.after_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 125) + def _reduce_44(val, _values, result) + @grammar.after_shift_error_token = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 129) + def _reduce_45(val, _values, result) + @grammar.after_pop_stack = val[1] + + result + end +.,., + +# reduce 46 omitted + +module_eval(<<'.,.,', 'parser.y', 136) + def _reduce_47(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + @grammar.add_type(id: id, tag: hash[:tag]) + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 144) + def _reduce_48(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + if @grammar.find_term_by_s_value(id.s_value) + on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) + else + @grammar.add_type(id: id, tag: hash[:tag]) + end + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 156) + def _reduce_49(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_left(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 166) + def _reduce_50(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_right(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 176) + def _reduce_51(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_precedence(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 186) + def _reduce_52(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_nonassoc(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 196) + def _reduce_53(val, _values, result) + @grammar.set_start_nterm(val[1]) + + result + end +.,., + +# reduce 54 omitted + +# reduce 55 omitted + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_56(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_57(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 202) + def _reduce_58(val, _values, result) + val[1].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[0], replace: true) + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 208) + def _reduce_59(val, _values, result) + val[2].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[1], replace: true) + } + + result + end +.,., + +# reduce 60 omitted + +# reduce 61 omitted + +module_eval(<<'.,.,', 'parser.y', 213) + def _reduce_62(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 218) + def _reduce_63(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 225) + def _reduce_64(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, [], val[4], is_inline: true) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 230) + def _reduce_65(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 235) + def _reduce_66(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 236) + def _reduce_67(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 241) + def _reduce_68(val, _values, result) + builder = val[0] + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 246) + def _reduce_69(val, _values, result) + builder = val[2] + result = val[0].append(builder) + + result + end +.,., + +# reduce 70 omitted + +# reduce 71 omitted + +# reduce 72 omitted + +# reduce 73 omitted + +module_eval(<<'.,.,', 'parser.y', 253) + def _reduce_74(val, _values, result) + reset_precs + result = Grammar::Parameterized::Rhs.new + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 258) + def _reduce_75(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.symbols << token + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 267) + def _reduce_76(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 274) + def _reduce_77(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 281) + def _reduce_78(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 289) + def _reduce_79(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +# reduce 80 omitted + +# reduce 81 omitted + +module_eval(<<'.,.,', 'parser.y', 301) + def _reduce_82(val, _values, result) + result = val[0].s_value if val[0] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_83(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_84(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 306) + def _reduce_85(val, _values, result) + result = if val[0] + [{tag: val[0], tokens: val[1]}] + else + [{tag: nil, tokens: val[1]}] + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 312) + def _reduce_86(val, _values, result) + result = val[0].append({tag: val[1], tokens: val[2]}) + result + end +.,., + +# reduce 87 omitted + +# reduce 88 omitted + +module_eval(<<'.,.,', 'parser.y', 321) + def _reduce_89(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 325) + def _reduce_90(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 329) + def _reduce_91(val, _values, result) + result = val[2] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_92(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_93(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_94(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_95(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_96(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 333) + def _reduce_97(val, _values, result) + result = [{tag: nil, tokens: val[0]}] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 334) + def _reduce_98(val, _values, result) + result = val[0].map {|tag, ids| {tag: tag, tokens: ids} } + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 335) + def _reduce_99(val, _values, result) + result = [{tag: nil, tokens: val[0]}, {tag: val[1], tokens: val[2]}] + result + end +.,., + +# reduce 100 omitted + +# reduce 101 omitted + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_102(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_103(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 104 omitted + +# reduce 105 omitted + +module_eval(<<'.,.,', 'parser.y', 348) + def _reduce_106(val, _values, result) + lhs = val[0] + lhs.alias_name = val[1] + val[3].each do |builder| + builder.lhs = lhs + builder.complete_input + @grammar.add_rule_builder(builder) + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 360) + def _reduce_107(val, _values, result) + if val[0].rhs.count > 1 + empties = val[0].rhs.select { |sym| sym.is_a?(Lrama::Lexer::Token::Empty) } + empties.each do |empty| + on_action_error("%empty on non-empty rule", empty) + end + end + builder = val[0] + if !builder.line + builder.line = @lexer.line - 1 + end + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 374) + def _reduce_108(val, _values, result) + builder = val[2] + if !builder.line + builder.line = @lexer.line - 1 + end + result = val[0].append(builder) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 384) + def _reduce_109(val, _values, result) + reset_precs + result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 389) + def _reduce_110(val, _values, result) + builder = val[0] + builder.add_rhs(Lrama::Lexer::Token::Empty.new(location: @lexer.location)) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 395) + def _reduce_111(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.add_rhs(token) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 404) + def _reduce_112(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 413) + def _reduce_113(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 422) + def _reduce_114(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + user_code.tag = val[3] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 431) + def _reduce_115(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 444) + def _reduce_116(val, _values, result) + result = "option" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 445) + def _reduce_117(val, _values, result) + result = "nonempty_list" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 446) + def _reduce_118(val, _values, result) + result = "list" + result + end +.,., + +# reduce 119 omitted + +# reduce 120 omitted + +module_eval(<<'.,.,', 'parser.y', 451) + def _reduce_121(val, _values, result) + result = if val[1] + [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] + else + [val[0]] + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 457) + def _reduce_122(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 458) + def _reduce_123(val, _values, result) + result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 463) + def _reduce_124(val, _values, result) + if prec_seen? + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 471) + def _reduce_125(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 475) + def _reduce_126(val, _values, result) + result = val[2] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 478) + def _reduce_127(val, _values, result) + result = val[1].s_value + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 483) + def _reduce_128(val, _values, result) + begin_c_declaration('\Z') + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 487) + def _reduce_129(val, _values, result) + end_c_declaration + @grammar.epilogue_first_lineno = val[0].first_line + 1 + @grammar.epilogue = val[2].s_value + + result + end +.,., + +# reduce 130 omitted + +# reduce 131 omitted + +# reduce 132 omitted + +# reduce 133 omitted + +# reduce 134 omitted + +module_eval(<<'.,.,', 'parser.y', 499) + def _reduce_135(val, _values, result) + result = Lrama::Lexer::Token::Ident.new(s_value: val[0].s_value) + result + end +.,., + +def _reduce_none(val, _values, result) + val[0] +end + + end # class Parser +end # module Lrama diff --git a/tool/lrama/lib/lrama/reporter.rb b/tool/lrama/lib/lrama/reporter.rb new file mode 100644 index 0000000000..ed25cc7f8f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'reporter/conflicts' +require_relative 'reporter/grammar' +require_relative 'reporter/precedences' +require_relative 'reporter/profile' +require_relative 'reporter/rules' +require_relative 'reporter/states' +require_relative 'reporter/terms' + +module Lrama + class Reporter + include Lrama::Tracer::Duration + + # @rbs (**bool options) -> void + def initialize(**options) + @options = options + @rules = Rules.new(**options) + @terms = Terms.new(**options) + @conflicts = Conflicts.new + @precedences = Precedences.new + @grammar = Grammar.new(**options) + @states = States.new(**options) + end + + # @rbs (File io, Lrama::States states) -> void + def report(io, states) + report_duration(:report) do + report_duration(:report_rules) { @rules.report(io, states) } + report_duration(:report_terms) { @terms.report(io, states) } + report_duration(:report_conflicts) { @conflicts.report(io, states) } + report_duration(:report_precedences) { @precedences.report(io, states) } + report_duration(:report_grammar) { @grammar.report(io, states) } + report_duration(:report_states) { @states.report(io, states, ielr: states.ielr_defined?) } + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/conflicts.rb b/tool/lrama/lib/lrama/reporter/conflicts.rb new file mode 100644 index 0000000000..f4d8c604c9 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/conflicts.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Conflicts + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_conflicts(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_conflicts(io, states) + has_conflict = false + + states.states.each do |state| + messages = format_conflict_messages(state.conflicts) + + unless messages.empty? + has_conflict = true + io << "State #{state.id} conflicts: #{messages.join(', ')}\n" + end + end + + io << "\n\n" if has_conflict + end + + # @rbs (Array[(Lrama::State::ShiftReduceConflict | Lrama::State::ReduceReduceConflict)] conflicts) -> Array[String] + def format_conflict_messages(conflicts) + conflict_types = { + shift_reduce: "shift/reduce", + reduce_reduce: "reduce/reduce" + } + + conflict_types.keys.map do |type| + type_conflicts = conflicts.select { |c| c.type == type } + "#{type_conflicts.count} #{conflict_types[type]}" unless type_conflicts.empty? + end.compact + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/grammar.rb b/tool/lrama/lib/lrama/reporter/grammar.rb new file mode 100644 index 0000000000..dc3f3f6bfd --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/grammar.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Grammar + # @rbs (?grammar: bool, **bool _) -> void + def initialize(grammar: false, **_) + @grammar = grammar + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @grammar + + io << "Grammar\n" + last_lhs = nil + + states.rules.each do |rule| + if rule.empty_rule? + r = "ε" + else + r = rule.rhs.map(&:display_name).join(" ") + end + + if rule.lhs == last_lhs + io << sprintf("%5d %s| %s", rule.id, " " * rule.lhs.display_name.length, r) << "\n" + else + io << "\n" + io << sprintf("%5d %s: %s", rule.id, rule.lhs.display_name, r) << "\n" + end + + last_lhs = rule.lhs + end + io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/precedences.rb b/tool/lrama/lib/lrama/reporter/precedences.rb new file mode 100644 index 0000000000..73c0888700 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/precedences.rb @@ -0,0 +1,54 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Precedences + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_precedences(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_precedences(io, states) + used_precedences = states.precedences.select(&:used_by?) + + return if used_precedences.empty? + + io << "Precedences\n\n" + + used_precedences.each do |precedence| + io << " precedence on #{precedence.symbol.display_name} is used to resolve conflict on\n" + + if precedence.used_by_lalr? + io << " LALR\n" + + precedence.used_by_lalr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + + if precedence.used_by_ielr? + io << " IELR\n" + + precedence.used_by_ielr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile.rb b/tool/lrama/lib/lrama/reporter/profile.rb new file mode 100644 index 0000000000..b569b94d4f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'profile/call_stack' +require_relative 'profile/memory' diff --git a/tool/lrama/lib/lrama/reporter/profile/call_stack.rb b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb new file mode 100644 index 0000000000..8a4d44b61c --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module CallStack + # See "Call-stack Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_stackprof + ex = nil #: Exception? + path = 'tmp/stackprof-cpu-myapp.dump' + + StackProf.run(mode: :cpu, raw: true, out: path) do + yield + rescue Exception => e + ex = e + end + + STDERR.puts("Call-stack Profiling result is generated on #{path}") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_stackprof + require "stackprof" + true + rescue LoadError + warn "stackprof is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile/memory.rb b/tool/lrama/lib/lrama/reporter/profile/memory.rb new file mode 100644 index 0000000000..a019581fdf --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/memory.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module Memory + # See "Memory Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_memory_profiler + ex = nil #: Exception? + + report = MemoryProfiler.report do # steep:ignore UnknownConstant + yield + rescue Exception => e + ex = e + end + + report.pretty_print(to_file: "tmp/memory_profiler.txt") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_memory_profiler + require "memory_profiler" + true + rescue LoadError + warn "memory_profiler is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/rules.rb b/tool/lrama/lib/lrama/reporter/rules.rb new file mode 100644 index 0000000000..3e8bf19a0a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/rules.rb @@ -0,0 +1,43 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Rules + # @rbs (?rules: bool, **bool _) -> void + def initialize(rules: false, **_) + @rules = rules + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @rules + + used_rules = states.rules.flat_map(&:rhs) + + unless used_rules.empty? + io << "Rule Usage Frequency\n\n" + frequency_counts = used_rules.each_with_object(Hash.new(0)) { |rule, counts| counts[rule] += 1 } + + frequency_counts + .select { |rule,| !rule.midrule? } + .sort_by { |rule, count| [-count, rule.name] } + .each_with_index { |(rule, count), i| io << sprintf("%5d %s (%d times)", i, rule.name, count) << "\n" } + io << "\n\n" + end + + unused_rules = states.rules.map(&:lhs).select do |rule| + !used_rules.include?(rule) && rule.token_id != 0 + end + + unless unused_rules.empty? + io << "#{unused_rules.count} Unused Rules\n\n" + unused_rules.each_with_index do |rule, index| + io << sprintf("%5d %s", index, rule.display_name) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/states.rb b/tool/lrama/lib/lrama/reporter/states.rb new file mode 100644 index 0000000000..d152d0511a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/states.rb @@ -0,0 +1,387 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class States + # @rbs (?itemsets: bool, ?lookaheads: bool, ?solved: bool, ?counterexamples: bool, ?verbose: bool, **bool _) -> void + def initialize(itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false, **_) + @itemsets = itemsets + @lookaheads = lookaheads + @solved = solved + @counterexamples = counterexamples + @verbose = verbose + end + + # @rbs (IO io, Lrama::States states, ielr: bool) -> void + def report(io, states, ielr: false) + cex = Counterexamples.new(states) if @counterexamples + + states.compute_la_sources_for_conflicted_states + report_split_states(io, states.states) if ielr + + states.states.each do |state| + report_state_header(io, state) + report_items(io, state) + report_conflicts(io, state) + report_shifts(io, state) + report_nonassoc_errors(io, state) + report_reduces(io, state) + report_nterm_transitions(io, state) + report_conflict_resolutions(io, state) if @solved + report_counterexamples(io, state, cex) if @counterexamples && state.has_conflicts? # @type var cex: Lrama::Counterexamples + report_verbose_info(io, state, states) if @verbose + # End of Report State + io << "\n" + end + end + + private + + # @rbs (IO io, Array[Lrama::State] states) -> void + def report_split_states(io, states) + ss = states.select(&:split_state?) + + return if ss.empty? + + io << "Split States\n\n" + + ss.each do |state| + io << " State #{state.id} is split from state #{state.lalr_isocore.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_state_header(io, state) + io << "State #{state.id}\n\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_items(io, state) + last_lhs = nil + list = @itemsets ? state.items : state.kernels + + list.sort_by {|i| [i.rule_id, i.position] }.each do |item| + r = item.empty_rule? ? "ε •" : item.rhs.map(&:display_name).insert(item.position, "•").join(" ") + + l = if item.lhs == last_lhs + " " * item.lhs.id.s_value.length + "|" + else + item.lhs.id.s_value + ":" + end + + la = "" + if @lookaheads && item.end_of_rule? + reduce = state.find_reduce_by_item!(item) + look_ahead = reduce.selected_look_ahead + unless look_ahead.empty? + la = " [#{look_ahead.compact.map(&:display_name).join(", ")}]" + end + end + + last_lhs = item.lhs + io << sprintf("%5i %s %s%s", item.rule_id, l, r, la) << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflicts(io, state) + return if state.conflicts.empty? + + state.conflicts.each do |conflict| + syms = conflict.symbols.map { |sym| sym.display_name } + io << " Conflict on #{syms.join(", ")}. " + + case conflict.type + when :shift_reduce + # @type var conflict: Lrama::State::ShiftReduceConflict + io << "shift/reduce(#{conflict.reduce.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + when :reduce_reduce + # @type var conflict: Lrama::State::ReduceReduceConflict + io << "reduce(#{conflict.reduce1.item.rule.lhs.display_name})/reduce(#{conflict.reduce2.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce1.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + + conflict.reduce2.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + else + raise "Unknown conflict type #{conflict.type}" + end + + io << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_shifts(io, state) + shifts = state.term_transitions.reject(&:not_selected) + + return if shifts.empty? + + next_syms = shifts.map(&:next_sym) + max_len = next_syms.map(&:display_name).map(&:length).max + shifts.each do |shift| + io << " #{shift.next_sym.display_name.ljust(max_len)} shift, and go to state #{shift.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nonassoc_errors(io, state) + error_symbols = state.resolved_conflicts.select { |resolved| resolved.which == :error }.map { |error| error.symbol.display_name } + + return if error_symbols.empty? + + max_len = error_symbols.map(&:length).max + error_symbols.each do |name| + io << " #{name.ljust(max_len)} error (nonassociative)\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_reduces(io, state) + reduce_pairs = [] #: Array[[Lrama::Grammar::Symbol, Lrama::State::Action::Reduce]] + + state.non_default_reduces.each do |reduce| + reduce.look_ahead&.each do |term| + reduce_pairs << [term, reduce] + end + end + + return if reduce_pairs.empty? && !state.default_reduction_rule + + max_len = [ + reduce_pairs.map(&:first).map(&:display_name).map(&:length).max || 0, + state.default_reduction_rule ? "$default".length : 0 + ].max + + reduce_pairs.sort_by { |term, _| term.number }.each do |term, reduce| + rule = reduce.item.rule + io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" + end + + if (r = state.default_reduction_rule) + s = "$default".ljust(max_len) + + if r.initial_rule? + io << " #{s} accept\n" + else + io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nterm_transitions(io, state) + return if state.nterm_transitions.empty? + + goto_transitions = state.nterm_transitions.sort_by do |goto| + goto.next_sym.number + end + + max_len = goto_transitions.map(&:next_sym).map do |nterm| + nterm.id.s_value.length + end.max + goto_transitions.each do |goto| + io << " #{goto.next_sym.id.s_value.ljust(max_len)} go to state #{goto.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflict_resolutions(io, state) + return if state.resolved_conflicts.empty? + + state.resolved_conflicts.each do |resolved| + io << " #{resolved.report_message}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::Counterexamples cex) -> void + def report_counterexamples(io, state, cex) + examples = cex.compute(state) + + examples.each do |example| + is_shift_reduce = example.type == :shift_reduce + label0 = is_shift_reduce ? "shift/reduce" : "reduce/reduce" + label1 = is_shift_reduce ? "Shift derivation" : "First Reduce derivation" + label2 = is_shift_reduce ? "Reduce derivation" : "Second Reduce derivation" + + io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" + io << " #{label1}\n" + + example.derivations1.render_strings_for_report.each do |str| + io << " #{str}\n" + end + + io << " #{label2}\n" + + example.derivations2.render_strings_for_report.each do |str| + io << " #{str}\n" + end + end + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_verbose_info(io, state, states) + report_direct_read_sets(io, state, states) + report_reads_relation(io, state, states) + report_read_sets(io, state, states) + report_includes_relation(io, state, states) + report_lookback_relation(io, state, states) + report_follow_sets(io, state, states) + report_look_ahead_sets(io, state, states) + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_direct_read_sets(io, state, states) + io << " [Direct Read sets]\n" + direct_read_sets = states.direct_read_sets + + state.nterm_transitions.each do |goto| + terms = direct_read_sets[goto] + next unless terms && !terms.empty? + + str = terms.map { |sym| sym.id.s_value }.join(", ") + io << " read #{goto.next_sym.id.s_value} shift #{str}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_reads_relation(io, state, states) + io << " [Reads Relation]\n" + + state.nterm_transitions.each do |goto| + goto2 = states.reads_relation[goto] + next unless goto2 + + goto2.each do |goto2| + io << " (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_read_sets(io, state, states) + io << " [Read sets]\n" + read_sets = states.read_sets + + state.nterm_transitions.each do |goto| + terms = read_sets[goto] + next unless terms && !terms.empty? + + terms.each do |sym| + io << " #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_includes_relation(io, state, states) + io << " [Includes Relation]\n" + + state.nterm_transitions.each do |goto| + gotos = states.includes_relation[goto] + next unless gotos + + gotos.each do |goto2| + io << " (State #{state.id}, #{goto.next_sym.id.s_value}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_lookback_relation(io, state, states) + io << " [Lookback Relation]\n" + + states.rules.each do |rule| + gotos = states.lookback_relation.dig(state.id, rule.id) + next unless gotos + + gotos.each do |goto2| + io << " (Rule: #{rule.display_name}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_follow_sets(io, state, states) + io << " [Follow sets]\n" + follow_sets = states.follow_sets + + state.nterm_transitions.each do |goto| + terms = follow_sets[goto] + next unless terms + + terms.each do |sym| + io << " #{goto.next_sym.id.s_value} -> #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_look_ahead_sets(io, state, states) + io << " [Look-Ahead Sets]\n" + look_ahead_rules = [] #: Array[[Lrama::Grammar::Rule, Array[Lrama::Grammar::Symbol]]] + + states.rules.each do |rule| + syms = states.la.dig(state.id, rule.id) + next unless syms + + look_ahead_rules << [rule, syms] + end + + return if look_ahead_rules.empty? + + max_len = look_ahead_rules.flat_map { |_, syms| syms.map { |s| s.id.s_value.length } }.max + + look_ahead_rules.each do |rule, syms| + syms.each do |sym| + io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/terms.rb b/tool/lrama/lib/lrama/reporter/terms.rb new file mode 100644 index 0000000000..f72d8b1a1a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/terms.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Terms + # @rbs (?terms: bool, **bool _) -> void + def initialize(terms: false, **_) + @terms = terms + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @terms + + look_aheads = states.states.each do |state| + state.reduces.flat_map do |reduce| + reduce.look_ahead unless reduce.look_ahead.nil? + end + end + + next_terms = states.states.flat_map do |state| + state.term_transitions.map {|shift| shift.next_sym } + end + + unused_symbols = states.terms.reject do |term| + (look_aheads + next_terms).include?(term) + end + + io << states.terms.count << " Terms\n\n" + + io << states.nterms.count << " Non-Terminals\n\n" + + unless unused_symbols.empty? + io << "#{unused_symbols.count} Unused Terms\n\n" + unused_symbols.each_with_index do |term, index| + io << sprintf("%5d %s", index, term.id.s_value) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb new file mode 100644 index 0000000000..50912e094e --- /dev/null +++ b/tool/lrama/lib/lrama/state.rb @@ -0,0 +1,534 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "state/action" +require_relative "state/inadequacy_annotation" +require_relative "state/item" +require_relative "state/reduce_reduce_conflict" +require_relative "state/resolved_conflict" +require_relative "state/shift_reduce_conflict" + +module Lrama + class State + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type conflict = State::ShiftReduceConflict | State::ReduceReduceConflict + # type transition = Action::Shift | Action::Goto + # type lookahead_set = Hash[Item, Array[Grammar::Symbol]] + # + # @id: Integer + # @accessing_symbol: Grammar::Symbol + # @kernels: Array[Item] + # @items: Array[Item] + # @items_to_state: Hash[Array[Item], State] + # @conflicts: Array[conflict] + # @resolved_conflicts: Array[ResolvedConflict] + # @default_reduction_rule: Grammar::Rule? + # @closure: Array[Item] + # @nterm_transitions: Array[Action::Goto] + # @term_transitions: Array[Action::Shift] + # @transitions: Array[transition] + # @internal_dependencies: Hash[Action::Goto, Array[Action::Goto]] + # @successor_dependencies: Hash[Action::Goto, Array[Action::Goto]] + + attr_reader :id #: Integer + attr_reader :accessing_symbol #: Grammar::Symbol + attr_reader :kernels #: Array[Item] + attr_reader :conflicts #: Array[conflict] + attr_reader :resolved_conflicts #: Array[ResolvedConflict] + attr_reader :default_reduction_rule #: Grammar::Rule? + attr_reader :closure #: Array[Item] + attr_reader :items #: Array[Item] + attr_reader :annotation_list #: Array[InadequacyAnnotation] + attr_reader :predecessors #: Array[State] + attr_reader :items_to_state #: Hash[Array[Item], State] + attr_reader :lane_items #: Hash[State, Array[[Item, Item]]] + + attr_accessor :_transitions #: Array[[Grammar::Symbol, Array[Item]]] + attr_accessor :reduces #: Array[Action::Reduce] + attr_accessor :ielr_isocores #: Array[State] + attr_accessor :lalr_isocore #: State + attr_accessor :lookaheads_recomputed #: bool + attr_accessor :follow_kernel_items #: Hash[Action::Goto, Hash[Item, bool]] + attr_accessor :always_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor :goto_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + + # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void + def initialize(id, accessing_symbol, kernels) + @id = id + @accessing_symbol = accessing_symbol + @kernels = kernels.freeze + @items = @kernels + # Manage relationships between items to state + # to resolve next state + @items_to_state = {} + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + @predecessors = [] + @lalr_isocore = self + @ielr_isocores = [self] + @internal_dependencies = {} + @successor_dependencies = {} + @annotation_list = [] + @lookaheads_recomputed = false + @follow_kernel_items = {} + @always_follows = {} + @goto_follows = {} + @lhs_contributions = {} + @lane_items = {} + end + + # @rbs (State other) -> bool + def ==(other) + self.id == other.id + end + + # @rbs (Array[Item] closure) -> void + def closure=(closure) + @closure = closure + @items = @kernels + @closure + end + + # @rbs () -> Array[Action::Reduce] + def non_default_reduces + reduces.reject do |reduce| + reduce.rule == @default_reduction_rule + end + end + + # @rbs () -> void + def compute_transitions_and_reduces + _transitions = {} + @_lane_items ||= {} + reduces = [] + items.each do |item| + # TODO: Consider what should be pushed + if item.end_of_rule? + reduces << Action::Reduce.new(item) + else + key = item.next_sym + _transitions[key] ||= [] + @_lane_items[key] ||= [] + next_item = item.new_by_next_position + _transitions[key] << next_item + @_lane_items[key] << [item, next_item] + end + end + + # It seems Bison 3.8.2 iterates transitions order by symbol number + transitions = _transitions.sort_by do |next_sym, to_items| + next_sym.number + end + + self._transitions = transitions.freeze + self.reduces = reduces.freeze + end + + # @rbs (Grammar::Symbol next_sym, State next_state) -> void + def set_lane_items(next_sym, next_state) + @lane_items[next_state] = @_lane_items[next_sym] + end + + # @rbs (Array[Item] items, State next_state) -> void + def set_items_to_state(items, next_state) + @items_to_state[items] = next_state + end + + # @rbs (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void + def set_look_ahead(rule, look_ahead) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead = look_ahead + end + + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void + def set_look_ahead_sources(rule, sources) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead_sources = sources + end + + # @rbs () -> Array[Action::Goto] + def nterm_transitions # steep:ignore + @nterm_transitions ||= transitions.select {|transition| transition.is_a?(Action::Goto) } + end + + # @rbs () -> Array[Action::Shift] + def term_transitions # steep:ignore + @term_transitions ||= transitions.select {|transition| transition.is_a?(Action::Shift) } + end + + # @rbs () -> Array[transition] + def transitions + @transitions ||= _transitions.map do |next_sym, to_items| + if next_sym.term? + Action::Shift.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + else + Action::Goto.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + end + end + end + + # @rbs (transition transition, State next_state) -> void + def update_transition(transition, next_state) + set_items_to_state(transition.to_items, next_state) + next_state.append_predecessor(self) + update_transitions_caches(transition) + end + + # @rbs () -> void + def update_transitions_caches(transition) + new_transition = + if transition.next_sym.term? + Action::Shift.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + else + Action::Goto.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + end + + @transitions.delete(transition) + @transitions << new_transition + @nterm_transitions = nil + @term_transitions = nil + + @follow_kernel_items[new_transition] = @follow_kernel_items.delete(transition) + @always_follows[new_transition] = @always_follows.delete(transition) + end + + # @rbs () -> Array[Action::Shift] + def selected_term_transitions + term_transitions.reject do |shift| + shift.not_selected + end + end + + # Move to next state by sym + # + # @rbs (Grammar::Symbol sym) -> State + def transition(sym) + result = nil + + if sym.term? + result = term_transitions.find {|shift| shift.next_sym == sym }.to_state + else + result = nterm_transitions.find {|goto| goto.next_sym == sym }.to_state + end + + raise "Can not transit by #{sym} #{self}" if result.nil? + + result + end + + # @rbs (Item item) -> Action::Reduce + def find_reduce_by_item!(item) + reduces.find do |r| + r.item == item + end || (raise "reduce is not found. #{item}") + end + + # @rbs (Grammar::Rule default_reduction_rule) -> void + def default_reduction_rule=(default_reduction_rule) + @default_reduction_rule = default_reduction_rule + + reduces.each do |r| + if r.rule == default_reduction_rule + r.default_reduction = true + end + end + end + + # @rbs () -> bool + def has_conflicts? + !@conflicts.empty? + end + + # @rbs () -> Array[conflict] + def sr_conflicts + @conflicts.select do |conflict| + conflict.type == :shift_reduce + end + end + + # @rbs () -> Array[conflict] + def rr_conflicts + @conflicts.select do |conflict| + conflict.type == :reduce_reduce + end + end + + # Clear information related to conflicts. + # IELR computation re-calculates conflicts and default reduction of states + # after LALR computation. + # Call this method before IELR computation to avoid duplicated conflicts information + # is stored. + # + # @rbs () -> void + def clear_conflicts + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + + term_transitions.each(&:clear_conflicts) + reduces.each(&:clear_conflicts) + end + + # @rbs () -> bool + def split_state? + @lalr_isocore != self + end + + # Definition 3.40 (propagate_lookaheads) + # + # @rbs (State next_state) -> lookahead_set + def propagate_lookaheads(next_state) + next_state.kernels.map {|next_kernel| + lookahead_sets = + if next_kernel.position > 1 + kernel = kernels.find {|k| k.predecessor_item_of?(next_kernel) } + item_lookahead_set[kernel] + else + goto_follow_set(next_kernel.lhs) + end + + [next_kernel, lookahead_sets & next_state.lookahead_set_filters[next_kernel]] + }.to_h + end + + # Definition 3.43 (is_compatible) + # + # @rbs (lookahead_set filtered_lookahead) -> bool + def is_compatible?(filtered_lookahead) + !lookaheads_recomputed || + @lalr_isocore.annotation_list.all? {|annotation| + a = annotation.dominant_contribution(item_lookahead_set) + b = annotation.dominant_contribution(filtered_lookahead) + a.nil? || b.nil? || a == b + } + end + + # Definition 3.38 (lookahead_set_filters) + # + # @rbs () -> lookahead_set + def lookahead_set_filters + @lookahead_set_filters ||= kernels.map {|kernel| + [kernel, @lalr_isocore.annotation_list.select {|annotation| annotation.contributed?(kernel) }.map(&:token)] + }.to_h + end + + # Definition 3.27 (inadequacy_lists) + # + # @rbs () -> Hash[Grammar::Symbol, Array[Action::Shift | Action::Reduce]] + def inadequacy_list + return @inadequacy_list if @inadequacy_list + + inadequacy_list = {} + + term_transitions.each do |shift| + inadequacy_list[shift.next_sym] ||= [] + inadequacy_list[shift.next_sym] << shift.dup + end + reduces.each do |reduce| + next if reduce.look_ahead.nil? + + reduce.look_ahead.each do |token| + inadequacy_list[token] ||= [] + inadequacy_list[token] << reduce.dup + end + end + + @inadequacy_list = inadequacy_list.select {|token, actions| actions.size > 1 } + end + + # Definition 3.30 (annotate_manifestation) + # + # @rbs () -> void + def annotate_manifestation + inadequacy_list.each {|token, actions| + contribution_matrix = actions.map {|action| + if action.is_a?(Action::Shift) + [action, nil] + else + [action, action.rule.empty_rule? ? lhs_contributions(action.rule.lhs, token) : kernels.map {|k| [k, k.rule == action.item.rule && k.end_of_rule?] }.to_h] + end + }.to_h + @annotation_list << InadequacyAnnotation.new(self, token, actions, contribution_matrix) + } + end + + # Definition 3.32 (annotate_predecessor) + # + # @rbs (State predecessor) -> void + def annotate_predecessor(predecessor) + propagating_list = annotation_list.map {|annotation| + contribution_matrix = annotation.contribution_matrix.map {|action, contributions| + if contributions.nil? + [action, nil] + elsif first_kernels.any? {|kernel| contributions[kernel] && predecessor.lhs_contributions(kernel.lhs, annotation.token).empty? } + [action, nil] + else + cs = predecessor.lane_items[self].map {|pred_kernel, kernel| + c = contributions[kernel] && ( + (kernel.position > 1 && predecessor.item_lookahead_set[pred_kernel].include?(annotation.token)) || + (kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, annotation.token)[pred_kernel]) + ) + [pred_kernel, c] + }.to_h + [action, cs] + end + }.to_h + + # Observation 3.33 (Simple Split-Stable Dominance) + # + # If all of contributions in the contribution_matrix are + # always contribution or never contribution, we can stop annotate propagations + # to the predecessor state. + next nil if contribution_matrix.all? {|_, contributions| contributions.nil? || contributions.all? {|_, contributed| !contributed } } + + InadequacyAnnotation.new(annotation.state, annotation.token, annotation.actions, contribution_matrix) + }.compact + predecessor.append_annotation_list(propagating_list) + end + + # @rbs () -> Array[Item] + def first_kernels + @first_kernels ||= kernels.select {|kernel| kernel.position == 1 } + end + + # @rbs (Array[InadequacyAnnotation] propagating_list) -> void + def append_annotation_list(propagating_list) + annotation_list.each do |annotation| + merging_list = propagating_list.select {|a| a.state == annotation.state && a.token == annotation.token && a.actions == annotation.actions } + annotation.merge_matrix(merging_list.map(&:contribution_matrix)) + propagating_list -= merging_list + end + + @annotation_list += propagating_list + end + + # Definition 3.31 (compute_lhs_contributions) + # + # @rbs (Grammar::Symbol sym, Grammar::Symbol token) -> (nil | Hash[Item, bool]) + def lhs_contributions(sym, token) + return @lhs_contributions[sym][token] unless @lhs_contributions.dig(sym, token).nil? + + transition = nterm_transitions.find {|goto| goto.next_sym == sym } + @lhs_contributions[sym] ||= {} + @lhs_contributions[sym][token] = + if always_follows[transition].include?(token) + {} + else + kernels.map {|kernel| [kernel, follow_kernel_items[transition][kernel] && item_lookahead_set[kernel].include?(token)] }.to_h + end + end + + # Definition 3.26 (item_lookahead_sets) + # + # @rbs () -> lookahead_set + def item_lookahead_set + return @item_lookahead_set if @item_lookahead_set + + @item_lookahead_set = kernels.map {|k| [k, []] }.to_h + @item_lookahead_set = kernels.map {|kernel| + value = + if kernel.lhs.accept_symbol? + [] + elsif kernel.position > 1 + prev_items = predecessors_with_item(kernel) + prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms } + elsif kernel.position == 1 + prev_state = @predecessors.find {|p| p.transitions.any? {|transition| transition.next_sym == kernel.lhs } } + goto = prev_state.nterm_transitions.find {|goto| goto.next_sym == kernel.lhs } + prev_state.goto_follows[goto] + end + [kernel, value] + }.to_h + end + + # @rbs (lookahead_set k) -> void + def item_lookahead_set=(k) + @item_lookahead_set = k + end + + # @rbs (Item item) -> Array[[State, Item]] + def predecessors_with_item(item) + result = [] + @predecessors.each do |pre| + pre.items.each do |i| + result << [pre, i] if i.predecessor_item_of?(item) + end + end + result + end + + # @rbs (State prev_state) -> void + def append_predecessor(prev_state) + @predecessors << prev_state + @predecessors.uniq! + end + + # Definition 3.39 (compute_goto_follow_set) + # + # @rbs (Grammar::Symbol nterm_token) -> Array[Grammar::Symbol] + def goto_follow_set(nterm_token) + return [] if nterm_token.accept_symbol? + goto = @lalr_isocore.nterm_transitions.find {|g| g.next_sym == nterm_token } + + @kernels + .select {|kernel| @lalr_isocore.follow_kernel_items[goto][kernel] } + .map {|kernel| item_lookahead_set[kernel] } + .reduce(@lalr_isocore.always_follows[goto]) {|result, terms| result |= terms } + end + + # Definition 3.8 (Goto Follows Internal Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def internal_dependencies(goto) + return @internal_dependencies[goto] if @internal_dependencies[goto] + + syms = @items.select {|i| + i.next_sym == goto.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0 + }.map(&:lhs).uniq + @internal_dependencies[goto] = nterm_transitions.select {|goto2| syms.include?(goto2.next_sym) } + end + + # Definition 3.5 (Goto Follows Successor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def successor_dependencies(goto) + return @successor_dependencies[goto] if @successor_dependencies[goto] + + @successor_dependencies[goto] = goto.to_state.nterm_transitions.select {|next_goto| next_goto.next_sym.nullable } + end + + # Definition 3.9 (Goto Follows Predecessor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def predecessor_dependencies(goto) + state_items = [] + @kernels.select {|kernel| + kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) + }.each do |item| + queue = predecessors_with_item(item) + until queue.empty? + st, i = queue.pop + if i.position == 0 + state_items << [st, i] + else + st.predecessors_with_item(i).each {|v| queue << v } + end + end + end + + state_items.map {|state, item| + state.nterm_transitions.find {|goto2| goto2.next_sym == item.lhs } + } + end + end +end diff --git a/tool/lrama/lib/lrama/state/action.rb b/tool/lrama/lib/lrama/state/action.rb new file mode 100644 index 0000000000..791685fc23 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "action/goto" +require_relative "action/reduce" +require_relative "action/shift" diff --git a/tool/lrama/lib/lrama/state/action/goto.rb b/tool/lrama/lib/lrama/state/action/goto.rb new file mode 100644 index 0000000000..4c2c82afdc --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/goto.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Goto + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/reduce.rb b/tool/lrama/lib/lrama/state/action/reduce.rb new file mode 100644 index 0000000000..9678ab0a98 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/reduce.rb @@ -0,0 +1,71 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Reduce + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @item: Item + # @look_ahead: Array[Grammar::Symbol]? + # @look_ahead_sources: Hash[Grammar::Symbol, Array[Action::Goto]]? + # @not_selected_symbols: Array[Grammar::Symbol] + + attr_reader :item #: Item + attr_reader :look_ahead #: Array[Grammar::Symbol]? + attr_reader :look_ahead_sources #: Hash[Grammar::Symbol, Array[Action::Goto]]? + attr_reader :not_selected_symbols #: Array[Grammar::Symbol] + + # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html + attr_accessor :default_reduction #: bool + + # @rbs (Item item) -> void + def initialize(item) + @item = item + @look_ahead = nil + @look_ahead_sources = nil + @not_selected_symbols = [] + end + + # @rbs () -> Grammar::Rule + def rule + @item.rule + end + + # @rbs (Array[Grammar::Symbol] look_ahead) -> Array[Grammar::Symbol] + def look_ahead=(look_ahead) + @look_ahead = look_ahead.freeze + end + + # @rbs (Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> Hash[Grammar::Symbol, Array[Action::Goto]] + def look_ahead_sources=(sources) + @look_ahead_sources = sources.freeze + end + + # @rbs (Grammar::Symbol sym) -> Array[Grammar::Symbol] + def add_not_selected_symbol(sym) + @not_selected_symbols << sym + end + + # @rbs () -> (::Array[Grammar::Symbol?]) + def selected_look_ahead + if look_ahead + look_ahead - @not_selected_symbols + else + [] + end + end + + # @rbs () -> void + def clear_conflicts + @not_selected_symbols = [] + @default_reduction = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/shift.rb b/tool/lrama/lib/lrama/state/action/shift.rb new file mode 100644 index 0000000000..52d9f8c4f0 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/shift.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Shift + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + attr_accessor :not_selected #: bool + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + + # @rbs () -> void + def clear_conflicts + @not_selected = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/inadequacy_annotation.rb b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb new file mode 100644 index 0000000000..3654fa4607 --- /dev/null +++ b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb @@ -0,0 +1,140 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class InadequacyAnnotation + # @rbs! + # type action = Action::Shift | Action::Reduce + + attr_accessor :state #: State + attr_accessor :token #: Grammar::Symbol + attr_accessor :actions #: Array[action] + attr_accessor :contribution_matrix #: Hash[action, Hash[Item, bool]] + + # @rbs (State state, Grammar::Symbol token, Array[action] actions, Hash[action, Hash[Item, bool]] contribution_matrix) -> void + def initialize(state, token, actions, contribution_matrix) + @state = state + @token = token + @actions = actions + @contribution_matrix = contribution_matrix + end + + # @rbs (Item item) -> bool + def contributed?(item) + @contribution_matrix.any? {|action, contributions| !contributions.nil? && contributions[item] } + end + + # @rbs (Array[Hash[action, Hash[Item, bool]]] another_matrixes) -> void + def merge_matrix(another_matrixes) + another_matrixes.each do |another_matrix| + @contribution_matrix.merge!(another_matrix) {|action, contributions, another_contributions| + next contributions if another_contributions.nil? + next another_contributions if contributions.nil? + + contributions.merge!(another_contributions) {|_, contributed, another_contributed| contributed || another_contributed } + } + end + end + + # Definition 3.42 (dominant_contribution) + # + # @rbs (State::lookahead_set lookaheads) -> Array[action]? + def dominant_contribution(lookaheads) + actions = @actions.select {|action| + contribution_matrix[action].nil? || contribution_matrix[action].any? {|item, contributed| contributed && lookaheads[item].include?(@token) } + } + return nil if actions.empty? + + resolve_conflict(actions) + end + + # @rbs (Array[action] actions) -> Array[action] + def resolve_conflict(actions) + # @type var shifts: Array[Action::Shift] + # @type var reduces: Array[Action::Reduce] + shifts = actions.select {|action| action.is_a?(Action::Shift)} + reduces = actions.select {|action| action.is_a?(Action::Reduce) } + + shifts.each do |shift| + reduces.each do |reduce| + sym = shift.next_sym + + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + actions.delete(shift) + next + when shift_prec > reduce_prec + # Shift is selected + actions.delete(reduce) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence&.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + next + when :right + # Shift is selected + actions.delete(reduce) + next + when :left + # Reduce is selected + actions.delete(shift) + next + when :nonassoc + # Can not resolve + # + # nonassoc creates "run-time" error, precedence creates "compile-time" error. + # Then omit both the shift and reduce. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + actions.delete(shift) + actions.delete(reduce) + else + raise "Unknown precedence type. #{sym}" + end + end + end + + actions + end + + # @rbs () -> String + def to_s + "State: #{@state.id}, Token: #{@token.id.s_value}, Actions: #{actions_to_s}, Contributions: #{contribution_matrix_to_s}" + end + + private + + # @rbs () -> String + def actions_to_s + '[' + @actions.map {|action| + if action.is_a?(Action::Shift) || action.is_a?(Action::Goto) + action.class.name + elsif action.is_a?(Action::Reduce) + "#{action.class.name}: (#{action.item})" + end + }.join(', ') + ']' + end + + # @rbs () -> String + def contribution_matrix_to_s + '[' + @contribution_matrix.map {|action, contributions| + "#{(action.is_a?(Action::Shift) || action.is_a?(Action::Goto)) ? action.class.name : "#{action.class.name}: (#{action.item})"}: " + contributions&.transform_keys(&:to_s).to_s + }.join(', ') + ']' + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/item.rb b/tool/lrama/lib/lrama/state/item.rb new file mode 100644 index 0000000000..3ecdd70b76 --- /dev/null +++ b/tool/lrama/lib/lrama/state/item.rb @@ -0,0 +1,120 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# TODO: Validate position is not over rule rhs + +require "forwardable" + +module Lrama + class State + class Item < Struct.new(:rule, :position, keyword_init: true) + # @rbs! + # include Grammar::Rule::_DelegatedMethods + # + # attr_accessor rule: Grammar::Rule + # attr_accessor position: Integer + # + # def initialize: (?rule: Grammar::Rule, ?position: Integer) -> void + + extend Forwardable + + def_delegators "rule", :lhs, :rhs + + # Optimization for States#setup_state + # + # @rbs () -> Integer + def hash + [rule_id, position].hash + end + + # @rbs () -> Integer + def rule_id + rule.id + end + + # @rbs () -> bool + def empty_rule? + rule.empty_rule? + end + + # @rbs () -> Integer + def number_of_rest_symbols + @number_of_rest_symbols ||= rhs.count - position + end + + # @rbs () -> Grammar::Symbol + def next_sym + rhs[position] + end + + # @rbs () -> Grammar::Symbol + def next_next_sym + @next_next_sym ||= rhs[position + 1] + end + + # @rbs () -> Grammar::Symbol + def previous_sym + rhs[position - 1] + end + + # @rbs () -> bool + def end_of_rule? + rhs.count == position + end + + # @rbs () -> bool + def beginning_of_rule? + position == 0 + end + + # @rbs () -> bool + def start_item? + rule.initial_rule? && beginning_of_rule? + end + + # @rbs () -> State::Item + def new_by_next_position + Item.new(rule: rule, position: position + 1) + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_before_dot # steep:ignore + rhs[0...position] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_dot # steep:ignore + rhs[position..-1] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_transition # steep:ignore + rhs[position+1..-1] + end + + # @rbs () -> ::String + def to_s + "#{lhs.id.s_value}: #{display_name}" + end + + # @rbs () -> ::String + def display_name + r = rhs.map(&:display_name).insert(position, "•").join(" ") + "#{r} (rule #{rule_id})" + end + + # Right after position + # + # @rbs () -> ::String + def display_rest + r = symbols_after_dot.map(&:display_name).join(" ") + ". #{r} (rule #{rule_id})" + end + + # @rbs (State::Item other_item) -> bool + def predecessor_item_of?(other_item) + rule == other_item.rule && position == other_item.position - 1 + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb new file mode 100644 index 0000000000..55ecad40bd --- /dev/null +++ b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class ReduceReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :reduce1 #: State::Action::Reduce + attr_reader :reduce2 #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], reduce1: State::Action::Reduce, reduce2: State::Action::Reduce) -> void + def initialize(symbols:, reduce1:, reduce2:) + @symbols = symbols + @reduce1 = reduce1 + @reduce2 = reduce2 + end + + # @rbs () -> :reduce_reduce + def type + :reduce_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/resolved_conflict.rb b/tool/lrama/lib/lrama/state/resolved_conflict.rb new file mode 100644 index 0000000000..014533c233 --- /dev/null +++ b/tool/lrama/lib/lrama/state/resolved_conflict.rb @@ -0,0 +1,65 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + # * state: A state on which the conflct is resolved + # * symbol: A symbol under discussion + # * reduce: A reduce under discussion + # * which: For which a conflict is resolved. :shift, :reduce or :error (for nonassociative) + # * resolved_by_precedence: If the conflict is resolved by precedence definition or not + class ResolvedConflict + # @rbs! + # type which_enum = :reduce | :shift | :error + + attr_reader :state #: State + attr_reader :symbol #: Grammar::Symbol + attr_reader :reduce #: State::Action::Reduce + attr_reader :which #: which_enum + attr_reader :resolved_by_precedence #: bool + + # @rbs (state: State, symbol: Grammar::Symbol, reduce: State::Action::Reduce, which: which_enum, resolved_by_precedence: bool) -> void + def initialize(state:, symbol:, reduce:, which:, resolved_by_precedence:) + @state = state + @symbol = symbol + @reduce = reduce + @which = which + @resolved_by_precedence = resolved_by_precedence + end + + # @rbs () -> (::String | bot) + def report_message + "Conflict between rule #{reduce.rule.id} and token #{symbol.display_name} #{how_resolved}." + end + + # @rbs () -> (::String | bot) + def report_precedences_message + "Conflict between reduce by \"#{reduce.rule.display_name}\" and shift #{symbol.display_name} #{how_resolved}." + end + + private + + # @rbs () -> (::String | bot) + def how_resolved + s = symbol.display_name + r = reduce.rule.precedence_sym&.display_name + case + when which == :shift && resolved_by_precedence + msg = "resolved as #{which} (%right #{s})" + when which == :shift + msg = "resolved as #{which} (#{r} < #{s})" + when which == :reduce && resolved_by_precedence + msg = "resolved as #{which} (%left #{s})" + when which == :reduce + msg = "resolved as #{which} (#{s} < #{r})" + when which == :error + msg = "resolved as an #{which} (%nonassoc #{s})" + else + raise "Unknown direction. #{self}" + end + + msg + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb new file mode 100644 index 0000000000..548f2de614 --- /dev/null +++ b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class ShiftReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :shift #: State::Action::Shift + attr_reader :reduce #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], shift: State::Action::Shift, reduce: State::Action::Reduce) -> void + def initialize(symbols:, shift:, reduce:) + @symbols = symbols + @shift = shift + @reduce = reduce + end + + # @rbs () -> :shift_reduce + def type + :shift_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb new file mode 100644 index 0000000000..ddce627df4 --- /dev/null +++ b/tool/lrama/lib/lrama/states.rb @@ -0,0 +1,867 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "tracer/duration" +require_relative "state/item" + +module Lrama + # States is passed to a template file + # + # "Efficient Computation of LALR(1) Look-Ahead Sets" + # https://dl.acm.org/doi/pdf/10.1145/69622.357187 + class States + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type state_id = Integer + # type rule_id = Integer + # + # include Grammar::_DelegatedMethods + # + # @grammar: Grammar + # @tracer: Tracer + # @states: Array[State] + # @direct_read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @reads_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @includes_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @lookback_relation: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + # @follow_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @la: Hash[state_id, Hash[rule_id, Bitmap::bitmap]] + + extend Forwardable + include Lrama::Tracer::Duration + + def_delegators "@grammar", :symbols, :terms, :nterms, :rules, :precedences, + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined? + + attr_reader :states #: Array[State] + attr_reader :reads_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :includes_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :lookback_relation #: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + + # @rbs (Grammar grammar, Tracer tracer) -> void + def initialize(grammar, tracer) + @grammar = grammar + @tracer = tracer + + @states = [] + + # `DR(p, A) = {t ∈ T | p -(A)-> r -(t)-> }` + # where p is state, A is nterm, t is term. + # + # `@direct_read_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @direct_read_sets = {} + + # Reads relation on nonterminal transitions (pair of state and nterm) + # `(p, A) reads (r, C) iff p -(A)-> r -(C)-> and C =>* ε` + # where p, r are state, A, C are nterm. + # + # `@reads_relation` is a hash whose + # key is goto, + # value is array of goto. + @reads_relation = {} + + # `Read(p, A) =s DR(p, A) ∪ ∪{Read(r, C) | (p, A) reads (r, C)}` + # + # `@read_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @read_sets = {} + + # `(p, A) includes (p', B) iff B -> βAγ, γ =>* ε, p' -(β)-> p` + # where p, p' are state, A, B are nterm, β, γ is sequence of symbol. + # + # `@includes_relation` is a hash whose + # key is goto, + # value is array of goto. + @includes_relation = {} + + # `(q, A -> ω) lookback (p, A) iff p -(ω)-> q` + # where p, q are state, A -> ω is rule, A is nterm, ω is sequence of symbol. + # + # `@lookback_relation` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is array of goto. + @lookback_relation = {} + + # `Follow(p, A) =s Read(p, A) ∪ ∪{Follow(p', B) | (p, A) includes (p', B)}` + # + # `@follow_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @follow_sets = {} + + # `LA(q, A -> ω) = ∪{Follow(p, A) | (q, A -> ω) lookback (p, A)` + # + # `@la` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is bitmap of term. + @la = {} + end + + # @rbs () -> void + def compute + report_duration(:compute_lr0_states) { compute_lr0_states } + + # Look Ahead Sets + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + + # Conflicts + report_duration(:compute_conflicts) { compute_conflicts(:lalr) } + + report_duration(:compute_default_reduction) { compute_default_reduction } + end + + # @rbs () -> void + def compute_ielr + # Preparation + report_duration(:clear_conflicts) { clear_conflicts } + # Phase 1 + report_duration(:compute_predecessors) { compute_predecessors } + report_duration(:compute_follow_kernel_items) { compute_follow_kernel_items } + report_duration(:compute_always_follows) { compute_always_follows } + report_duration(:compute_goto_follows) { compute_goto_follows } + # Phase 2 + report_duration(:compute_inadequacy_annotations) { compute_inadequacy_annotations } + # Phase 3 + report_duration(:split_states) { split_states } + # Phase 4 + report_duration(:clear_look_ahead_sets) { clear_look_ahead_sets } + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + # Phase 5 + report_duration(:compute_conflicts) { compute_conflicts(:ielr) } + report_duration(:compute_default_reduction) { compute_default_reduction } + end + + # @rbs () -> Integer + def states_count + @states.count + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def direct_read_sets + @_direct_read_sets ||= @direct_read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def read_sets + @_read_sets ||= @read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def follow_sets + @_follow_sets ||= @follow_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[state_id, Hash[rule_id, Array[Grammar::Symbol]]] + def la + @_la ||= @la.transform_values do |second_hash| + second_hash.transform_values do |v| + bitmap_to_terms(v) + end + end + end + + # @rbs () -> Integer + def sr_conflicts_count + @sr_conflicts_count ||= @states.flat_map(&:sr_conflicts).count + end + + # @rbs () -> Integer + def rr_conflicts_count + @rr_conflicts_count ||= @states.flat_map(&:rr_conflicts).count + end + + # @rbs (Logger logger) -> void + def validate!(logger) + validate_conflicts_within_threshold!(logger) + end + + def compute_la_sources_for_conflicted_states + reflexive = {} + @states.each do |state| + state.nterm_transitions.each do |goto| + reflexive[goto] = [goto] + end + end + + # compute_read_sets + read_sets = Digraph.new(nterm_transitions, @reads_relation, reflexive).compute + # compute_follow_sets + follow_sets = Digraph.new(nterm_transitions, @includes_relation, read_sets).compute + + @states.select(&:has_conflicts?).each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + sources = {} + + ary.each do |goto| + source = follow_sets[goto] + + next unless source + + source.each do |goto2| + tokens = direct_read_sets[goto2] + tokens.each do |token| + sources[token] ||= [] + sources[token] |= [goto2] + end + end + end + + state.set_look_ahead_sources(rule, sources) + end + end + end + + private + + # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] + def create_state(accessing_symbol, kernels, states_created) + # A item can appear in some states, + # so need to use `kernels` (not `kernels.first`) as a key. + # + # For example... + # + # %% + # program: '+' strings_1 + # | '-' strings_2 + # ; + # + # strings_1: string_1 + # ; + # + # strings_2: string_1 + # | string_2 + # ; + # + # string_1: string + # ; + # + # string_2: string '+' + # ; + # + # string: tSTRING + # ; + # %% + # + # For these grammar, there are 2 states + # + # State A + # string_1: string • + # + # State B + # string_1: string • + # string_2: string • '+' + # + return [states_created[kernels], false] if states_created[kernels] + + state = State.new(@states.count, accessing_symbol, kernels) + @states << state + states_created[kernels] = state + + return [state, true] + end + + # @rbs (State state) -> void + def setup_state(state) + # closure + closure = [] + queued = {} + items = state.kernels.dup + + items.each do |item| + queued[item.rule_id] = true if item.position == 0 + end + + while (item = items.shift) do + if (sym = item.next_sym) && sym.nterm? + @grammar.find_rules_by_symbol!(sym).each do |rule| + next if queued[rule.id] + i = State::Item.new(rule: rule, position: 0) + closure << i + items << i + queued[i.rule_id] = true + end + end + end + + state.closure = closure.sort_by {|i| i.rule.id } + + # Trace + @tracer.trace_closure(state) + + # shift & reduce + state.compute_transitions_and_reduces + end + + # @rbs (Array[State] states, State state) -> void + def enqueue_state(states, state) + # Trace + @tracer.trace_state_list_append(@states.count, state) + + states << state + end + + # @rbs () -> void + def compute_lr0_states + # State queue + states = [] + states_created = {} + + state, _ = create_state(symbols.first, [State::Item.new(rule: @grammar.rules.first, position: 0)], states_created) + enqueue_state(states, state) + + while (state = states.shift) do + # Trace + @tracer.trace_state(state) + + setup_state(state) + + # `State#transitions` can not be used here + # because `items_to_state` of the `state` is not set yet. + state._transitions.each do |next_sym, to_items| + new_state, created = create_state(next_sym, to_items, states_created) + state.set_items_to_state(to_items, new_state) + state.set_lane_items(next_sym, new_state) + enqueue_state(states, new_state) if created + end + end + end + + # @rbs () -> Array[State::Action::Goto] + def nterm_transitions + a = [] + + @states.each do |state| + state.nterm_transitions.each do |goto| + a << goto + end + end + + a + end + + # @rbs () -> void + def compute_look_ahead_sets + report_duration(:compute_direct_read_sets) { compute_direct_read_sets } + report_duration(:compute_reads_relation) { compute_reads_relation } + report_duration(:compute_read_sets) { compute_read_sets } + report_duration(:compute_includes_relation) { compute_includes_relation } + report_duration(:compute_lookback_relation) { compute_lookback_relation } + report_duration(:compute_follow_sets) { compute_follow_sets } + report_duration(:compute_la) { compute_la } + end + + # @rbs () -> void + def compute_direct_read_sets + @states.each do |state| + state.nterm_transitions.each do |goto| + ary = goto.to_state.term_transitions.map do |shift| + shift.next_sym.number + end + + @direct_read_sets[goto] = Bitmap.from_array(ary) + end + end + end + + # @rbs () -> void + def compute_reads_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + goto.to_state.nterm_transitions.each do |goto2| + nterm2 = goto2.next_sym + if nterm2.nullable + @reads_relation[goto] ||= [] + @reads_relation[goto] << goto2 + end + end + end + end + end + + # @rbs () -> void + def compute_read_sets + @read_sets = Digraph.new(nterm_transitions, @reads_relation, @direct_read_sets).compute + end + + # Execute transition of state by symbols + # then return final state. + # + # @rbs (State state, Array[Grammar::Symbol] symbols) -> State + def transition(state, symbols) + symbols.each do |sym| + state = state.transition(sym) + end + + state + end + + # @rbs () -> void + def compute_includes_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + nterm = goto.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + i = rule.rhs.count - 1 + + while (i > -1) do + sym = rule.rhs[i] + + break if sym.term? + state2 = transition(state, rule.rhs[0...i]) + # p' = state, B = nterm, p = state2, A = sym + key = state2.nterm_transitions.find do |goto2| + goto2.next_sym.token_id == sym.token_id + end || (raise "Goto by #{sym.name} on state #{state2.id} is not found") + # TODO: need to omit if state == state2 ? + @includes_relation[key] ||= [] + @includes_relation[key] << goto + break unless sym.nullable + i -= 1 + end + end + end + end + end + + # @rbs () -> void + def compute_lookback_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + nterm = goto.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + state2 = transition(state, rule.rhs) + # p = state, A = nterm, q = state2, A -> ω = rule + @lookback_relation[state2.id] ||= {} + @lookback_relation[state2.id][rule.id] ||= [] + @lookback_relation[state2.id][rule.id] << goto + end + end + end + end + + # @rbs () -> void + def compute_follow_sets + @follow_sets = Digraph.new(nterm_transitions, @includes_relation, @read_sets).compute + end + + # @rbs () -> void + def compute_la + @states.each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + ary.each do |goto| + # q = state, A -> ω = rule, p = state2, A = nterm + follows = @follow_sets[goto] + + next if follows == 0 + + @la[state.id] ||= {} + @la[state.id][rule.id] ||= 0 + look_ahead = @la[state.id][rule.id] | follows + @la[state.id][rule.id] |= look_ahead + + # No risk of conflict when + # * the state only has single reduce + # * the state only has nterm_transitions (GOTO) + next if state.reduces.count == 1 && state.term_transitions.count == 0 + + state.set_look_ahead(rule, bitmap_to_terms(look_ahead)) + end + end + end + end + + # @rbs (Bitmap::bitmap bit) -> Array[Grammar::Symbol] + def bitmap_to_terms(bit) + ary = Bitmap.to_array(bit) + ary.map do |i| + @grammar.find_symbol_by_number!(i) + end + end + + # @rbs () -> void + def compute_conflicts(lr_type) + compute_shift_reduce_conflicts(lr_type) + compute_reduce_reduce_conflicts + end + + # @rbs () -> void + def compute_shift_reduce_conflicts(lr_type) + states.each do |state| + state.term_transitions.each do |shift| + state.reduces.each do |reduce| + sym = shift.next_sym + + next unless reduce.look_ahead + next unless reduce.look_ahead.include?(sym) + + # Shift/Reduce conflict + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when shift_prec > reduce_prec + # Shift is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence.type + when :precedence + # Can not resolve the conflict + # + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + when :right + # Shift is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when :left + # Reduce is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when :nonassoc + # The conflict is resolved + # + # %nonassoc creates "run-time" error by removing both shift and reduce from + # the state. This makes the state to get syntax error if the conflicted token appears. + # On the other hand, %precedence creates "compile-time" error by keeping both + # shift and reduce on the state. This makes the state to be conflicted on the token. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :error, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + else + raise "Unknown precedence type. #{sym}" + end + end + end + end + end + + # @rbs (Grammar::Precedence shift_prec, Grammar::Precedence reduce_prec, State::ResolvedConflict resolved_conflict) -> void + def mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + case lr_type + when :lalr + shift_prec.mark_used_by_lalr(resolved_conflict) + reduce_prec.mark_used_by_lalr(resolved_conflict) + when :ielr + shift_prec.mark_used_by_ielr(resolved_conflict) + reduce_prec.mark_used_by_ielr(resolved_conflict) + end + end + + # @rbs () -> void + def compute_reduce_reduce_conflicts + states.each do |state| + state.reduces.combination(2) do |reduce1, reduce2| + next if reduce1.look_ahead.nil? || reduce2.look_ahead.nil? + + intersection = reduce1.look_ahead & reduce2.look_ahead + + unless intersection.empty? + state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) + end + end + end + end + + # @rbs () -> void + def compute_default_reduction + states.each do |state| + next if state.reduces.empty? + # Do not set, if conflict exist + next unless state.conflicts.empty? + # Do not set, if shift with `error` exists. + next if state.term_transitions.map {|shift| shift.next_sym }.include?(@grammar.error_symbol) + + state.default_reduction_rule = state.reduces.map do |r| + [r.rule, r.rule.id, (r.look_ahead || []).count] + end.min_by do |rule, rule_id, count| + [-count, rule_id] + end.first + end + end + + # @rbs () -> void + def clear_conflicts + states.each(&:clear_conflicts) + end + + # Definition 3.15 (Predecessors) + # + # @rbs () -> void + def compute_predecessors + @states.each do |state| + state.transitions.each do |transition| + transition.to_state.append_predecessor(state) + end + end + end + + # Definition 3.16 (follow_kernel_items) + # + # @rbs () -> void + def compute_follow_kernel_items + set = nterm_transitions + relation = compute_goto_internal_relation + base_function = compute_goto_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, follow_kernel_items| + state = goto.from_state + state.follow_kernel_items[goto] = state.kernels.map {|kernel| + [kernel, Bitmap.to_bool_array(follow_kernel_items, state.kernels.count)] + }.to_h + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_goto_bitmaps + nterm_transitions.map {|goto| + bools = goto.from_state.kernels.map.with_index {|kernel, i| i if kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) }.compact + [goto, Bitmap.from_array(bools)] + }.to_h + end + + # Definition 3.20 (always_follows, one closure) + # + # @rbs () -> void + def compute_always_follows + set = nterm_transitions + relation = compute_goto_successor_or_internal_relation + base_function = compute_transition_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, always_follows_bitmap| + goto.from_state.always_follows[goto] = bitmap_to_terms(always_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_successor_or_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.successor_dependencies(goto) + state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_transition_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.to_state.term_transitions.map {|shift| shift.next_sym.number })] + }.to_h + end + + # Definition 3.24 (goto_follows, via always_follows) + # + # @rbs () -> void + def compute_goto_follows + set = nterm_transitions + relation = compute_goto_internal_or_predecessor_dependencies + base_function = compute_always_follows_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, goto_follows_bitmap| + goto.from_state.goto_follows[goto] = bitmap_to_terms(goto_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_or_predecessor_dependencies + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + state.predecessor_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_always_follows_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.from_state.always_follows[goto].map(&:number))] + }.to_h + end + + # @rbs () -> void + def split_states + @states.each do |state| + state.transitions.each do |transition| + compute_state(state, transition, transition.to_state) + end + end + end + + # @rbs () -> void + def compute_inadequacy_annotations + @states.each do |state| + state.annotate_manifestation + end + + queue = @states.reject {|state| state.annotation_list.empty? } + + while (curr = queue.shift) do + curr.predecessors.each do |pred| + cache = pred.annotation_list.dup + curr.annotate_predecessor(pred) + queue << pred if cache != pred.annotation_list && !queue.include?(pred) + end + end + end + + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> void + def merge_lookaheads(state, filtered_lookaheads) + return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? } + + state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 } + state.transitions.each do |transition| + next if transition.to_state.lookaheads_recomputed + compute_state(state, transition, transition.to_state) + end + end + + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void + def compute_state(state, transition, next_state) + propagating_lookaheads = state.propagate_lookaheads(next_state) + s = next_state.ielr_isocores.find {|st| st.is_compatible?(propagating_lookaheads) } + + if s.nil? + s = next_state.lalr_isocore + new_state = State.new(@states.count, s.accessing_symbol, s.kernels) + new_state.closure = s.closure + new_state.compute_transitions_and_reduces + s.transitions.each do |transition| + new_state.set_items_to_state(transition.to_items, transition.to_state) + end + @states << new_state + new_state.lalr_isocore = s + s.ielr_isocores << new_state + s.ielr_isocores.each do |st| + st.ielr_isocores = s.ielr_isocores + end + new_state.lookaheads_recomputed = true + new_state.item_lookahead_set = propagating_lookaheads + state.update_transition(transition, new_state) + elsif(!s.lookaheads_recomputed) + s.lookaheads_recomputed = true + s.item_lookahead_set = propagating_lookaheads + else + merge_lookaheads(s, propagating_lookaheads) + state.update_transition(transition, s) if state.items_to_state[transition.to_items].id != s.id + end + end + + # @rbs (Logger logger) -> void + def validate_conflicts_within_threshold!(logger) + exit false unless conflicts_within_threshold?(logger) + end + + # @rbs (Logger logger) -> bool + def conflicts_within_threshold?(logger) + return true unless @grammar.expect + + [sr_conflicts_within_threshold?(logger), rr_conflicts_within_threshold?(logger)].all? + end + + # @rbs (Logger logger) -> bool + def sr_conflicts_within_threshold?(logger) + return true if @grammar.expect == sr_conflicts_count + + logger.error("shift/reduce conflicts: #{sr_conflicts_count} found, #{@grammar.expect} expected") + false + end + + # @rbs (Logger logger) -> bool + def rr_conflicts_within_threshold?(logger, expected: 0) + return true if expected == rr_conflicts_count + + logger.error("reduce/reduce conflicts: #{rr_conflicts_count} found, #{expected} expected") + false + end + + # @rbs () -> void + def clear_look_ahead_sets + @direct_read_sets.clear + @reads_relation.clear + @read_sets.clear + @includes_relation.clear + @lookback_relation.clear + @follow_sets.clear + @la.clear + + @_direct_read_sets = nil + @_read_sets = nil + @_follow_sets = nil + @_la = nil + end + end +end diff --git a/tool/lrama/lib/lrama/tracer.rb b/tool/lrama/lib/lrama/tracer.rb new file mode 100644 index 0000000000..fda699a665 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer.rb @@ -0,0 +1,51 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "tracer/actions" +require_relative "tracer/closure" +require_relative "tracer/duration" +require_relative "tracer/only_explicit_rules" +require_relative "tracer/rules" +require_relative "tracer/state" + +module Lrama + class Tracer + # @rbs (IO io, **bool options) -> void + def initialize(io, **options) + @io = io + @options = options + @only_explicit_rules = OnlyExplicitRules.new(io, **options) + @rules = Rules.new(io, **options) + @actions = Actions.new(io, **options) + @closure = Closure.new(io, **options) + @state = State.new(io, **options) + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + @only_explicit_rules.trace(grammar) + @rules.trace(grammar) + @actions.trace(grammar) + end + + # @rbs (Lrama::State state) -> void + def trace_closure(state) + @closure.trace(state) + end + + # @rbs (Lrama::State state) -> void + def trace_state(state) + @state.trace(state) + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_state_list_append(state_count, state) + @state.trace_list_append(state_count, state) + end + + # @rbs () -> void + def enable_duration + Duration.enable if @options[:time] + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/actions.rb b/tool/lrama/lib/lrama/tracer/actions.rb new file mode 100644 index 0000000000..7b9c9b9f53 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/actions.rb @@ -0,0 +1,22 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Actions + # @rbs (IO io, ?actions: bool, **bool options) -> void + def initialize(io, actions: false, **options) + @io = io + @actions = actions + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @actions + + @io << "Grammar rules with actions:" << "\n" + grammar.rules.each { |rule| @io << rule.with_actions << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/closure.rb b/tool/lrama/lib/lrama/tracer/closure.rb new file mode 100644 index 0000000000..5b2f0b27e6 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/closure.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Closure + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @closure = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @closure + + @io << "Closure: input" << "\n" + state.kernels.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + @io << "Closure: output" << "\n" + state.items.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/duration.rb b/tool/lrama/lib/lrama/tracer/duration.rb new file mode 100644 index 0000000000..91c49625b2 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/duration.rb @@ -0,0 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + module Duration + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @_report_duration_enabled: bool + + # @rbs () -> void + def self.enable + @_report_duration_enabled = true + end + + # @rbs () -> bool + def self.enabled? + !!@_report_duration_enabled + end + + # @rbs [T] (_ToS message) { -> T } -> T + def report_duration(message) + time1 = Time.now.to_f + result = yield + time2 = Time.now.to_f + + if Duration.enabled? + STDERR.puts sprintf("%s %10.5f s", message, time2 - time1) + end + + return result + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb new file mode 100644 index 0000000000..4f64e7d2f4 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class OnlyExplicitRules + # @rbs (IO io, ?only_explicit: bool, **bool) -> void + def initialize(io, only_explicit: false, **_) + @io = io + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each do |rule| + @io << rule.display_name_without_action << "\n" if rule.lhs.first_set.any? + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/rules.rb b/tool/lrama/lib/lrama/tracer/rules.rb new file mode 100644 index 0000000000..d6e85b8432 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Rules + # @rbs (IO io, ?rules: bool, ?only_explicit: bool, **bool) -> void + def initialize(io, rules: false, only_explicit: false, **_) + @io = io + @rules = rules + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return if !@rules || @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each { |rule| @io << rule.display_name << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/state.rb b/tool/lrama/lib/lrama/tracer/state.rb new file mode 100644 index 0000000000..21c0047f8e --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/state.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class State + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @state = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @state + + # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but + # I think it is not correct... + previous = state.kernels.first.previous_sym + @io << "Processing state #{state.id} (reached by #{previous.display_name})" << "\n" + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_list_append(state_count, state) + return unless @state + + previous = state.kernels.first.previous_sym + @io << sprintf("state_list_append (state = %d, symbol = %d (%s))", + state_count, previous.number, previous.display_name) << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb new file mode 100644 index 0000000000..d649b74939 --- /dev/null +++ b/tool/lrama/lib/lrama/version.rb @@ -0,0 +1,6 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + VERSION = "0.7.1".freeze #: String +end diff --git a/tool/lrama/lib/lrama/warnings.rb b/tool/lrama/lib/lrama/warnings.rb new file mode 100644 index 0000000000..52f09144ef --- /dev/null +++ b/tool/lrama/lib/lrama/warnings.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'warnings/conflicts' +require_relative 'warnings/implicit_empty' +require_relative 'warnings/name_conflicts' +require_relative 'warnings/redefined_rules' +require_relative 'warnings/required' +require_relative 'warnings/useless_precedence' + +module Lrama + class Warnings + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @conflicts = Conflicts.new(logger, warnings) + @implicit_empty = ImplicitEmpty.new(logger, warnings) + @name_conflicts = NameConflicts.new(logger, warnings) + @redefined_rules = RedefinedRules.new(logger, warnings) + @required = Required.new(logger, warnings) + @useless_precedence = UselessPrecedence.new(logger, warnings) + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + @conflicts.warn(states) + @implicit_empty.warn(grammar) + @name_conflicts.warn(grammar) + @redefined_rules.warn(grammar) + @required.warn(grammar) + @useless_precedence.warn(grammar, states) + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/conflicts.rb b/tool/lrama/lib/lrama/warnings/conflicts.rb new file mode 100644 index 0000000000..6ba0de6f9c --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/conflicts.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Conflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::States states) -> void + def warn(states) + return unless @warnings + + if states.sr_conflicts_count != 0 + @logger.warn("shift/reduce conflicts: #{states.sr_conflicts_count} found") + end + + if states.rr_conflicts_count != 0 + @logger.warn("reduce/reduce conflicts: #{states.rr_conflicts_count} found") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/implicit_empty.rb b/tool/lrama/lib/lrama/warnings/implicit_empty.rb new file mode 100644 index 0000000000..ba81adca01 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/implicit_empty.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Empty rules are easily overlooked and ambiguous + # - Empty alternatives like `rule: | "token";` can be missed during code reading + # - Difficult to distinguish between intentional empty rules vs. omissions + # - Explicit marking with %empty directive comment improves clarity + class ImplicitEmpty + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.rule_builders.each do |builder| + if builder.rhs.empty? + @logger.warn("warning: empty rule without %empty") + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/name_conflicts.rb b/tool/lrama/lib/lrama/warnings/name_conflicts.rb new file mode 100644 index 0000000000..c0754ab551 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/name_conflicts.rb @@ -0,0 +1,63 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Parameterized rule names conflicting with symbol names + # - When a %rule name is identical to a terminal or non-terminal symbol name, + # it reduces grammar readability and may cause unintended behavior + # - Detecting these conflicts helps improve grammar definition quality + class NameConflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return if grammar.parameterized_rules.empty? + + symbol_names = collect_symbol_names(grammar) + check_conflicts(grammar.parameterized_rules, symbol_names) + end + + private + + # @rbs (Lrama::Grammar grammar) -> Set[String] + def collect_symbol_names(grammar) + symbol_names = Set.new + + collect_term_names(grammar.terms, symbol_names) + collect_nterm_names(grammar.nterms, symbol_names) + + symbol_names + end + + # @rbs (Array[untyped] terms, Set[String] symbol_names) -> void + def collect_term_names(terms, symbol_names) + terms.each do |term| + symbol_names.add(term.id.s_value) + symbol_names.add(term.alias_name) if term.alias_name + end + end + + # @rbs (Array[untyped] nterms, Set[String] symbol_names) -> void + def collect_nterm_names(nterms, symbol_names) + nterms.each do |nterm| + symbol_names.add(nterm.id.s_value) + end + end + + # @rbs (Array[untyped] parameterized_rules, Set[String] symbol_names) -> void + def check_conflicts(parameterized_rules, symbol_names) + parameterized_rules.each do |param_rule| + next unless symbol_names.include?(param_rule.name) + + @logger.warn("warning: parameterized rule name \"#{param_rule.name}\" conflicts with symbol name") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/redefined_rules.rb b/tool/lrama/lib/lrama/warnings/redefined_rules.rb new file mode 100644 index 0000000000..8ac2f1f103 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/redefined_rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class RedefinedRules + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.parameterized_resolver.redefined_rules.each do |rule| + @logger.warn("parameterized rule redefined: #{rule}") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/required.rb b/tool/lrama/lib/lrama/warnings/required.rb new file mode 100644 index 0000000000..4ab1ed787e --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/required.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Required + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings = false, **_) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + if grammar.required + @logger.warn("currently, %require is simply valid as a grammar but does nothing") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/useless_precedence.rb b/tool/lrama/lib/lrama/warnings/useless_precedence.rb new file mode 100644 index 0000000000..2913d6d7e5 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/useless_precedence.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class UselessPrecedence + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + return unless @warnings + + grammar.precedences.each do |precedence| + unless precedence.used_by? + @logger.warn("Precedence #{precedence.s_value} (line: #{precedence.lineno}) is defined but not used in any rule.") + end + end + end + end + end +end diff --git a/tool/lrama/template/bison/_yacc.h b/tool/lrama/template/bison/_yacc.h new file mode 100644 index 0000000000..3e270c9171 --- /dev/null +++ b/tool/lrama/template/bison/_yacc.h @@ -0,0 +1,79 @@ +<%# b4_shared_declarations -%> + <%-# b4_cpp_guard_open([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#ifndef <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> +# define <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> + <%- end -%> + <%-# b4_declare_yydebug & b4_YYDEBUG_define -%> +/* Debug traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif +#if YYDEBUG && !defined(yydebug) +extern int yydebug; +#endif +<%= output.percent_code("requires") %> + + <%-# b4_token_enums_defines -%> +/* Token kinds. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + enum yytokentype + { +<%= output.token_enums -%> + }; + typedef enum yytokentype yytoken_kind_t; +#endif + + <%-# b4_declare_yylstype -%> + <%-# b4_value_type_define -%> +/* Value type. */ +<% if output.grammar.union %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +union YYSTYPE +{ +#line <%= output.grammar.union.lineno %> "<%= output.grammar_file_path %>" +<%= output.grammar.union.braces_less_code %> +#line [@oline@] [@ofile@] + +}; +typedef union YYSTYPE YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% else %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef int YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% end %> + + <%-# b4_location_type_define -%> +/* Location type. */ +#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED +typedef struct YYLTYPE YYLTYPE; +struct YYLTYPE +{ + int first_line; + int first_column; + int last_line; + int last_column; +}; +# define YYLTYPE_IS_DECLARED 1 +# define YYLTYPE_IS_TRIVIAL 1 +#endif + + + + + <%-# b4_declare_yyerror_and_yylex. Not supported -%> + <%-# b4_declare_yyparse -%> +int yyparse (<%= output.parse_param %>); + + +<%= output.percent_code("provides") %> + <%-# b4_cpp_guard_close([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#endif /* !<%= output.b4_cpp_guard__b4_spec_mapped_header_file %> */ + <%- end -%> diff --git a/tool/lrama/template/bison/yacc.c b/tool/lrama/template/bison/yacc.c new file mode 100644 index 0000000000..6edd59a0d5 --- /dev/null +++ b/tool/lrama/template/bison/yacc.c @@ -0,0 +1,2068 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison implementation for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* C LALR(1) parser skeleton written by Richard Stallman, by + simplifying the original so-called "semantic" parser. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + +/* All symbols defined below should begin with yy or YY, to avoid + infringing on user name space. This should be done even for local + variables, as they might otherwise be expanded by user macros. + There are some unavoidable exceptions within include files to + define necessary library symbols; they are noted "INFRINGES ON + USER NAME SPACE" below. */ + +<%# b4_identification -%> +/* Identify Bison output, and Bison version. */ +#define YYBISON 30802 + +/* Bison version string. */ +#define YYBISON_VERSION "3.8.2" + +/* Skeleton name. */ +#define YYSKELETON_NAME "<%= output.template_basename %>" + +/* Pure parsers. */ +#define YYPURE 1 + +/* Push parsers. */ +#define YYPUSH 0 + +/* Pull parsers. */ +#define YYPULL 1 + + +<%# b4_user_pre_prologue -%> +<%- if output.aux.prologue -%> +/* First part of user prologue. */ +#line <%= output.aux.prologue_first_lineno %> "<%= output.grammar_file_path %>" +<%= output.aux.prologue %> +#line [@oline@] [@ofile@] +<%- end -%> + +<%# b4_cast_define -%> +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast<Type> (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast<Type> (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif +<%# b4_null_define -%> +# ifndef YY_NULLPTR +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif +# else +# define YY_NULLPTR ((void*)0) +# endif +# endif + +<%# b4_header_include_if -%> +<%- if output.include_header -%> +#include "<%= output.include_header %>" +<%- else -%> +/* Use api.header.include to #include this header + instead of duplicating it here. */ +<%= output.render_partial("bison/_yacc.h") %> +<%- end -%> +<%# b4_declare_symbol_enum -%> +/* Symbol kind. */ +enum yysymbol_kind_t +{ +<%= output.symbol_enum -%> +}; +typedef enum yysymbol_kind_t yysymbol_kind_t; + + + + +<%# b4_user_post_prologue -%> +<%# b4_c99_int_type_define -%> +#ifdef short +# undef short +#endif + +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + <limits.h> and (if available) <stdint.h> are included + so that the code can choose integer types of a good width. */ + +#ifndef __PTRDIFF_MAX__ +# include <limits.h> /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stdint.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif + +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ + +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; +#endif + +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; +#else +typedef short yytype_int16; +#endif + +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + <https://h20195.www2.hpe.com/V2/getpdf.aspx/4AA4-7673ENW.pdf>. */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 +#endif + +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; +#else +typedef short yytype_uint8; +#endif + +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; +#else +typedef int yytype_uint16; +#endif + +<%# b4_sizes_types_define -%> +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif +#endif + +#ifndef YYSIZE_T +# ifdef __SIZE_TYPE__ +# define YYSIZE_T __SIZE_TYPE__ +# elif defined size_t +# define YYSIZE_T size_t +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# define YYSIZE_T size_t +# else +# define YYSIZE_T unsigned +# endif +#endif + +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef <%= output.int_type_for([output.yynstates - 1]) %> yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include <libintl.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_(Msgid) dgettext ("bison-runtime", Msgid) +# endif +# endif +# ifndef YY_ +# define YY_(Msgid) Msgid +# endif +#endif + + +<%# b4_attribute_define -%> +#ifndef YY_ATTRIBUTE_PURE +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif +#endif + +#ifndef YY_ATTRIBUTE_UNUSED +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) +# else +# define YY_ATTRIBUTE_UNUSED +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YY_USE(E) ((void) (E)) +#else +# define YY_USE(E) /* empty */ +#endif + +/* Suppress an incorrect diagnostic about yylval being uninitialized. */ +#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__ +# if __GNUC__ * 100 + __GNUC_MINOR__ < 407 +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") +# else +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ + _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") +# endif +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ + _Pragma ("GCC diagnostic pop") +#else +# define YY_INITIAL_VALUE(Value) Value +#endif +#ifndef YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_END +#endif +#ifndef YY_INITIAL_VALUE +# define YY_INITIAL_VALUE(Value) /* Nothing. */ +#endif + +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + + +#define YY_ASSERT(E) ((void) (0 && (E))) + +#if 1 + +/* The parser invokes alloca or malloc; define the necessary symbols. */ + +# ifdef YYSTACK_USE_ALLOCA +# if YYSTACK_USE_ALLOCA +# ifdef __GNUC__ +# define YYSTACK_ALLOC __builtin_alloca +# elif defined __BUILTIN_VA_ARG_INCR +# include <alloca.h> /* INFRINGES ON USER NAME SPACE */ +# elif defined _AIX +# define YYSTACK_ALLOC __alloca +# elif defined _MSC_VER +# include <malloc.h> /* INFRINGES ON USER NAME SPACE */ +# define alloca _alloca +# else +# define YYSTACK_ALLOC alloca +# if ! defined _ALLOCA_H && ! defined EXIT_SUCCESS +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ + /* Use EXIT_SUCCESS as a witness for stdlib.h. */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# endif +# endif +# endif + +# ifdef YYSTACK_ALLOC + /* Pacify GCC's 'empty if-body' warning. */ +# define YYSTACK_FREE(Ptr) do { /* empty */; } while (0) +# ifndef YYSTACK_ALLOC_MAXIMUM + /* The OS might guarantee only one guard page at the bottom of the stack, + and a page size can be as small as 4096 bytes. So we cannot safely + invoke alloca (N) if N exceeds 4096. Use a slightly smaller number + to allow for a few compiler-allocated temporary stack slots. */ +# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +# endif +# else +# define YYSTACK_ALLOC YYMALLOC +# define YYSTACK_FREE YYFREE +# ifndef YYSTACK_ALLOC_MAXIMUM +# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +# endif +# if (defined __cplusplus && ! defined EXIT_SUCCESS \ + && ! ((defined YYMALLOC || defined malloc) \ + && (defined YYFREE || defined free))) +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# ifndef YYMALLOC +# define YYMALLOC malloc +# if ! defined malloc && ! defined EXIT_SUCCESS +void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# ifndef YYFREE +# define YYFREE free +# if ! defined free && ! defined EXIT_SUCCESS +void free (void *); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# endif +#endif /* 1 */ + +#if (! defined yyoverflow \ + && (! defined __cplusplus \ + || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL \ + && defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) + +/* A type that is properly aligned for any stack member. */ +union yyalloc +{ + yy_state_t yyss_alloc; + YYSTYPE yyvs_alloc; + YYLTYPE yyls_alloc; +}; + +/* The size of the maximum gap between one aligned stack and the next. */ +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) + +/* The size of an array large to enough to hold all stacks, each with + N elements. */ +# define YYSTACK_BYTES(N) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE) \ + + YYSIZEOF (YYLTYPE)) \ + + 2 * YYSTACK_GAP_MAXIMUM) + +# define YYCOPY_NEEDED 1 + +/* Relocate STACK from its old location to the new one. The + local variables YYSIZE and YYSTACKSIZE give the old and new number of + elements in the stack, and YYPTR gives the new location of the + stack. Advance YYPTR to a properly aligned location for the next + stack. */ +# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYPTRDIFF_T yynewbytes; \ + YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ + } \ + while (0) + +#endif + +#if defined YYCOPY_NEEDED && YYCOPY_NEEDED +/* Copy COUNT objects from SRC to DST. The source and destination do + not overlap. */ +# ifndef YYCOPY +# if defined __GNUC__ && 1 < __GNUC__ +# define YYCOPY(Dst, Src, Count) \ + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) +# else +# define YYCOPY(Dst, Src, Count) \ + do \ + { \ + YYPTRDIFF_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (Dst)[yyi] = (Src)[yyi]; \ + } \ + while (0) +# endif +# endif +#endif /* !YYCOPY_NEEDED */ + +/* YYFINAL -- State number of the termination state. */ +#define YYFINAL <%= output.yyfinal %> +/* YYLAST -- Last index in YYTABLE. */ +#define YYLAST <%= output.yylast %> + +/* YYNTOKENS -- Number of terminals. */ +#define YYNTOKENS <%= output.yyntokens %> +/* YYNNTS -- Number of nonterminals. */ +#define YYNNTS <%= output.yynnts %> +/* YYNRULES -- Number of rules. */ +#define YYNRULES <%= output.yynrules %> +/* YYNSTATES -- Number of states. */ +#define YYNSTATES <%= output.yynstates %> + +/* YYMAXUTOK -- Last valid token kind. */ +#define YYMAXUTOK <%= output.yymaxutok %> + + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) + +/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM + as returned by yylex. */ +static const <%= output.int_type_for(output.context.yytranslate) %> yytranslate[] = +{ +<%= output.yytranslate %> +}; + +<%- if output.error_recovery -%> +/* YYTRANSLATE_INVERTED[SYMBOL-NUM] -- Token number corresponding to SYMBOL-NUM */ +static const <%= output.int_type_for(output.context.yytranslate_inverted) %> yytranslate_inverted[] = +{ +<%= output.yytranslate_inverted %> +}; +<%- end -%> +#if YYDEBUG +/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ +static const <%= output.int_type_for(output.context.yyrline) %> yyrline[] = +{ +<%= output.yyrline %> +}; +#endif + +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if 1 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + +/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. + First, the terminals, then, starting at YYNTOKENS, nonterminals. */ +static const char *const yytname[] = +{ +<%= output.yytname %> +}; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} +#endif + +#define YYPACT_NINF (<%= output.yypact_ninf %>) + +#define yypact_value_is_default(Yyn) \ + <%= output.table_value_equals(output.context.yypact, "Yyn", output.yypact_ninf, "YYPACT_NINF") %> + +#define YYTABLE_NINF (<%= output.yytable_ninf %>) + +#define yytable_value_is_error(Yyn) \ + <%= output.table_value_equals(output.context.yytable, "Yyn", output.yytable_ninf, "YYTABLE_NINF") %> + +<%# b4_parser_tables_define -%> +/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + STATE-NUM. */ +static const <%= output.int_type_for(output.context.yypact) %> yypact[] = +{ +<%= output.int_array_to_string(output.context.yypact) %> +}; + +/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. + Performed when YYTABLE does not specify something else to do. Zero + means the default is an error. */ +static const <%= output.int_type_for(output.context.yydefact) %> yydefact[] = +{ +<%= output.int_array_to_string(output.context.yydefact) %> +}; + +/* YYPGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yypgoto) %> yypgoto[] = +{ +<%= output.int_array_to_string(output.context.yypgoto) %> +}; + +/* YYDEFGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yydefgoto) %> yydefgoto[] = +{ +<%= output.int_array_to_string(output.context.yydefgoto) %> +}; + +/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If + positive, shift that token. If negative, reduce the rule whose + number is the opposite. If YYTABLE_NINF, syntax error. */ +static const <%= output.int_type_for(output.context.yytable) %> yytable[] = +{ +<%= output.int_array_to_string(output.context.yytable) %> +}; + +static const <%= output.int_type_for(output.context.yycheck) %> yycheck[] = +{ +<%= output.int_array_to_string(output.context.yycheck) %> +}; + +/* YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of + state STATE-NUM. */ +static const <%= output.int_type_for(output.context.yystos) %> yystos[] = +{ +<%= output.int_array_to_string(output.context.yystos) %> +}; + +/* YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr1) %> yyr1[] = +{ +<%= output.int_array_to_string(output.context.yyr1) %> +}; + +/* YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr2) %> yyr2[] = +{ +<%= output.int_array_to_string(output.context.yyr2) %> +}; + + +enum { YYENOMEM = -2 }; + +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab +#define YYNOMEM goto yyexhaustedlab + + +#define YYRECOVERING() (!!yyerrstatus) + +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (<%= output.yyerror_args %>, YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF + +<%# b4_yylloc_default_define -%> +/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. + If N is 0, then set CURRENT to the empty location which ends + the previous symbol: RHS[0] (always defined). */ + +#ifndef YYLLOC_DEFAULT +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = \ + YYRHSLOC (Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC (Rhs, 0).last_column; \ + } \ + while (0) +#endif + +#define YYRHSLOC(Rhs, K) ((Rhs)[K]) + + +/* Enable debugging if requested. */ +#if YYDEBUG + +# ifndef YYFPRINTF +# include <stdio.h> /* INFRINGES ON USER NAME SPACE */ +# define YYFPRINTF fprintf +# endif + +# define YYDPRINTF(Args) \ +do { \ + if (yydebug) \ + YYFPRINTF Args; \ +} while (0) + + +<%# b4_yylocation_print_define -%> +/* YYLOCATION_PRINT -- Print the location on the stream. + This macro was not mandated originally: define only if we know + we won't break user code: when these are the locations we know. */ + +# ifndef YYLOCATION_PRINT + +# if defined YY_LOCATION_PRINT + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) YY_LOCATION_PRINT(File, *(Loc)<%= output.user_args %>) + +# elif defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + +/* Print *YYLOCP on YYO. Private, do not rely on its existence. */ + +YY_ATTRIBUTE_UNUSED +static int +yy_location_print_ (FILE *yyo, YYLTYPE const * const yylocp) +{ + int res = 0; + int end_col = 0 != yylocp->last_column ? yylocp->last_column - 1 : 0; + if (0 <= yylocp->first_line) + { + res += YYFPRINTF (yyo, "%d", yylocp->first_line); + if (0 <= yylocp->first_column) + res += YYFPRINTF (yyo, ".%d", yylocp->first_column); + } + if (0 <= yylocp->last_line) + { + if (yylocp->first_line < yylocp->last_line) + { + res += YYFPRINTF (yyo, "-%d", yylocp->last_line); + if (0 <= end_col) + res += YYFPRINTF (yyo, ".%d", end_col); + } + else if (0 <= end_col && yylocp->first_column < end_col) + res += YYFPRINTF (yyo, "-%d", end_col); + } + return res; +} + +# define YYLOCATION_PRINT yy_location_print_ + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT(File, Loc<%= output.user_args %>) YYLOCATION_PRINT(File, &(Loc)<%= output.user_args %>) + +# else + +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) ((void) 0) + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT YYLOCATION_PRINT + +# endif +# endif /* !defined YYLOCATION_PRINT */ + + +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) \ +do { \ + if (yydebug) \ + { \ + YYFPRINTF (stderr, "%s ", Title); \ + yy_symbol_print (stderr, \ + Kind, Value, Location<%= output.user_args %>); \ + YYFPRINTF (stderr, "\n"); \ + } \ +} while (0) + + +<%# b4_yy_symbol_print_define -%> +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ + +static void +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + FILE *yyoutput = yyo; +<%= output.parse_param_use("yyoutput", "yylocationp") %> + if (!yyvaluep) + return; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +<%# b4_symbol_actions(printer) -%> +switch (yykind) + { +<%= output.symbol_actions_for_printer -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ + +static void +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); + + YYLOCATION_PRINT (yyo, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ": "); + yy_symbol_value_print (yyo, yykind, yyvaluep, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ")"); +} + +/*------------------------------------------------------------------. +| yy_stack_print -- Print the state stack from its BOTTOM up to its | +| TOP (included). | +`------------------------------------------------------------------*/ + +static void +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop<%= output.user_formals %>) +{ + YYFPRINTF (stderr, "Stack now"); + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF (stderr, " %d", yybot); + } + YYFPRINTF (stderr, "\n"); +} + +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_stack_print ((Bottom), (Top)<%= output.user_args %>); \ +} while (0) + + +/*------------------------------------------------. +| Report that the YYRULE is going to be reduced. | +`------------------------------------------------*/ + +static void +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, YYLTYPE *yylsp, + int yyrule<%= output.user_formals %>) +{ + int yylno = yyrline[yyrule]; + int yynrhs = yyr2[yyrule]; + int yyi; + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", + yyrule - 1, yylno); + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF (stderr, " $%d = ", yyi + 1); + yy_symbol_print (stderr, + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)], + &(yylsp[(yyi + 1) - (yynrhs)])<%= output.user_args %>); + YYFPRINTF (stderr, "\n"); + } +} + +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_reduce_print (yyssp, yyvsp, yylsp, Rule<%= output.user_args %>); \ +} while (0) + +/* Nonzero means print parse trace. It is left uninitialized so that + multiple parsers can coexist. */ +#ifndef yydebug +int yydebug; +#endif +#else /* !YYDEBUG */ +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) +#endif /* !YYDEBUG */ + + +/* YYINITDEPTH -- initial size of the parser's stacks. */ +#ifndef YYINITDEPTH +# define YYINITDEPTH 200 +#endif + +/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only + if the built-in stack extension method is used). + + Do not make this value too large; the results are undefined if + YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) + evaluated with infinite-precision integer arithmetic. */ + +#ifndef YYMAXDEPTH +# define YYMAXDEPTH 10000 +#endif + + +/* Context of a parse error. */ +typedef struct +{ + yy_state_t *yyssp; + yysymbol_kind_t yytoken; + YYLTYPE *yylloc; +} yypcontext_t; + +/* Put in YYARG at most YYARGN of the expected tokens given the + current YYCTX, and return the number of tokens stored in YYARG. If + YYARG is null, return the number of expected tokens (guaranteed to + be less than YYNTOKENS). Return YYENOMEM on memory exhaustion. + Return 0 if there are more than YYARGN expected tokens, yet fill + YYARG up to YYARGN. */ +static int +yypcontext_expected_tokens (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + int yyn = yypact[+*yyctx->yyssp]; + if (!yypact_value_is_default (yyn)) + { + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. In other words, skip the first -YYN actions for + this state because they are default actions. */ + int yyxbegin = yyn < 0 ? -yyn : 0; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYSYMBOL_YYerror + && !yytable_value_is_error (yytable[yyx + yyn])) + { + if (!yyarg) + ++yycount; + else if (yycount == yyargn) + return 0; + else + yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx); + } + } + if (yyarg && yycount == 0 && 0 < yyargn) + yyarg[0] = YYSYMBOL_YYEMPTY; + return yycount; +} + + + + +#ifndef yystrlen +# if defined __GLIBC__ && defined _STRING_H +# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S))) +# else +/* Return the length of YYSTR. */ +static YYPTRDIFF_T +yystrlen (const char *yystr) +{ + YYPTRDIFF_T yylen; + for (yylen = 0; yystr[yylen]; yylen++) + continue; + return yylen; +} +# endif +#endif + +#ifndef yystpcpy +# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +# define yystpcpy stpcpy +# else +/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in + YYDEST. */ +static char * +yystpcpy (char *yydest, const char *yysrc) +{ + char *yyd = yydest; + const char *yys = yysrc; + + while ((*yyd++ = *yys++) != '\0') + continue; + + return yyd - 1; +} +# endif +#endif + +#ifndef yytnamerr +/* Copy to YYRES the contents of YYSTR after stripping away unnecessary + quotes and backslashes, so that it's suitable for yyerror. The + heuristic is that double-quoting is unnecessary unless the string + contains an apostrophe, a comma, or backslash (other than + backslash-backslash). YYSTR is taken from yytname. If YYRES is + null, do not copy; instead, return the length of what the result + would have been. */ +static YYPTRDIFF_T +yytnamerr (char *yyres, const char *yystr) +{ + if (*yystr == '"') + { + YYPTRDIFF_T yyn = 0; + char const *yyp = yystr; + for (;;) + switch (*++yyp) + { + case '\'': + case ',': + goto do_not_strip_quotes; + + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; + else + goto append; + + append: + default: + if (yyres) + yyres[yyn] = *yyp; + yyn++; + break; + + case '"': + if (yyres) + yyres[yyn] = '\0'; + return yyn; + } + do_not_strip_quotes: ; + } + + if (yyres) + return yystpcpy (yyres, yystr) - yyres; + else + return yystrlen (yystr); +} +#endif + + +static int +yy_syntax_error_arguments (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + /* There are many possibilities here to consider: + - If this state is a consistent state with a default action, then + the only way this function was invoked is if the default action + is an error action. In that case, don't check for expected + tokens because there are none. + - The only way there can be no lookahead present (in yychar) is if + this state is a consistent state with a default action. Thus, + detecting the absence of a lookahead is sufficient to determine + that there is no unexpected or expected token to report. In that + case, just report a simple "syntax error". + - Don't assume there isn't a lookahead just because this state is a + consistent state with a default action. There might have been a + previous inconsistent state, consistent state with a non-default + action, or user semantic action that manipulated yychar. + - Of course, the expected token list depends on states to have + correct lookahead information, and it depends on the parser not + to perform extra reductions after fetching a lookahead from the + scanner and before detecting a syntax error. Thus, state merging + (from LALR or IELR) and default reductions corrupt the expected + token list. However, the list is correct for canonical LR with + one exception: it will still contain any token that will not be + accepted due to an error action in a later state. + */ + if (yyctx->yytoken != YYSYMBOL_YYEMPTY) + { + int yyn; + if (yyarg) + yyarg[yycount] = yyctx->yytoken; + ++yycount; + yyn = yypcontext_expected_tokens (yyctx, + yyarg ? yyarg + 1 : yyarg, yyargn - 1); + if (yyn == YYENOMEM) + return YYENOMEM; + else + yycount += yyn; + } + return yycount; +} + +/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message + about the unexpected token YYTOKEN for the state stack whose top is + YYSSP. + + Return 0 if *YYMSG was successfully written. Return -1 if *YYMSG is + not large enough to hold the message. In that case, also set + *YYMSG_ALLOC to the required number of bytes. Return YYENOMEM if the + required number of bytes is too large to store. */ +static int +yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg, + const yypcontext_t *yyctx<%= output.user_formals %>) +{ + enum { YYARGS_MAX = 5 }; + /* Internationalized format string. */ + const char *yyformat = YY_NULLPTR; + /* Arguments of yyformat: reported tokens (one for the "unexpected", + one per "expected"). */ + yysymbol_kind_t yyarg[YYARGS_MAX]; + /* Cumulated lengths of YYARG. */ + YYPTRDIFF_T yysize = 0; + + /* Actual size of YYARG. */ + int yycount = yy_syntax_error_arguments (yyctx, yyarg, YYARGS_MAX); + if (yycount == YYENOMEM) + return YYENOMEM; + + switch (yycount) + { +#define YYCASE_(N, S) \ + case N: \ + yyformat = S; \ + break + default: /* Avoid compiler warnings. */ + YYCASE_(0, YY_("syntax error")); + YYCASE_(1, YY_("syntax error, unexpected %s")); + YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); + YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s")); + YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); + YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); +#undef YYCASE_ + } + + /* Compute error message size. Don't count the "%s"s, but reserve + room for the terminator. */ + yysize = yystrlen (yyformat) - 2 * yycount + 1; + { + int yyi; + for (yyi = 0; yyi < yycount; ++yyi) + { + YYPTRDIFF_T yysize1 + = yysize + yytnamerr (YY_NULLPTR, yytname[yyarg[yyi]]); + if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM) + yysize = yysize1; + else + return YYENOMEM; + } + } + + if (*yymsg_alloc < yysize) + { + *yymsg_alloc = 2 * yysize; + if (! (yysize <= *yymsg_alloc + && *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM)) + *yymsg_alloc = YYSTACK_ALLOC_MAXIMUM; + return -1; + } + + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + { + char *yyp = *yymsg; + int yyi = 0; + while ((*yyp = *yyformat) != '\0') + if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount) + { + yyp += yytnamerr (yyp, yytname[yyarg[yyi++]]); + yyformat += 2; + } + else + { + ++yyp; + ++yyformat; + } + } + return 0; +} + +<%# b4_yydestruct_define %> +/*-----------------------------------------------. +| Release the memory associated to this symbol. | +`-----------------------------------------------*/ + +static void +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep, YYLTYPE *yylocationp<%= output.user_formals %>) +{ +<%= output.parse_param_use("yyvaluep", "yylocationp") %> + if (!yymsg) + yymsg = "Deleting"; + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp<%= output.user_args %>); + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + switch (yykind) + { +<%= output.symbol_actions_for_destructor -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + + +<%- if output.error_recovery -%> +#ifndef YYMAXREPAIR +# define YYMAXREPAIR(<%= output.parse_param_name %>) (3) +#endif + +#ifndef YYERROR_RECOVERY_ENABLED +# define YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>) (1) +#endif + +enum yy_repair_type { + inserting, + deleting, + shifting, +}; + +struct yy_repair { + enum yy_repair_type type; + yysymbol_kind_t term; +}; +typedef struct yy_repair yy_repair; + +struct yy_repairs { + /* For debug */ + int id; + /* For breadth-first traversing */ + struct yy_repairs *next; + YYPTRDIFF_T stack_length; + /* Bottom of states */ + yy_state_t *states; + /* Top of states */ + yy_state_t *state; + /* repair length */ + int repair_length; + /* */ + struct yy_repairs *prev_repair; + struct yy_repair repair; +}; +typedef struct yy_repairs yy_repairs; + +struct yy_term { + yysymbol_kind_t kind; + YYSTYPE value; + YYLTYPE location; +}; +typedef struct yy_term yy_term; + +struct yy_repair_terms { + int id; + int length; + yy_term terms[]; +}; +typedef struct yy_repair_terms yy_repair_terms; + +static void +yy_error_token_initialize (yysymbol_kind_t yykind, YYSTYPE * const yyvaluep, YYLTYPE * const yylocationp<%= output.user_formals %>) +{ + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +switch (yykind) + { +<%= output.symbol_actions_for_error_token -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + +static yy_repair_terms * +yy_create_repair_terms(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + yy_repair_terms *rep_terms; + int count = 0; + + while (r->prev_repair) + { + count++; + r = r->prev_repair; + } + + rep_terms = (yy_repair_terms *) YYMALLOC (sizeof (yy_repair_terms) + sizeof (yy_term) * count); + rep_terms->id = reps->id; + rep_terms->length = count; + + r = reps; + while (r->prev_repair) + { + rep_terms->terms[count-1].kind = r->repair.term; + count--; + r = r->prev_repair; + } + + return rep_terms; +} + +static void +yy_print_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + + YYDPRINTF ((stderr, + "id: %d, repair_length: %d, repair_state: %d, prev_repair_id: %d\n", + reps->id, reps->repair_length, *reps->state, reps->prev_repair->id)); + + while (r->prev_repair) + { + YYDPRINTF ((stderr, "%s ", yysymbol_name (r->repair.term))); + r = r->prev_repair; + } + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_print_repair_terms(yy_repair_terms *rep_terms<%= output.user_formals %>) +{ + for (int i = 0; i < rep_terms->length; i++) + YYDPRINTF ((stderr, "%s ", yysymbol_name (rep_terms->terms[i].kind))); + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_free_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + while (reps) + { + yy_repairs *r = reps; + reps = reps->next; + YYFREE (r->states); + YYFREE (r); + } +} + +static int +yy_process_repairs(yy_repairs *reps, yysymbol_kind_t token) +{ + int yyn; + int yystate = *reps->state; + int yylen = 0; + yysymbol_kind_t yytoken = token; + + goto yyrecover_backup; + +yyrecover_newstate: + // TODO: check reps->stack_length + reps->state += 1; + *reps->state = (yy_state_t) yystate; + + +yyrecover_backup: + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yyrecover_default; + + /* "Reading a token" */ + if (yytoken == YYSYMBOL_YYEMPTY) + return 1; + + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yyrecover_default; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyrecover_errlab; + yyn = -yyn; + goto yyrecover_reduce; + } + + /* shift */ + yystate = yyn; + yytoken = YYSYMBOL_YYEMPTY; + goto yyrecover_newstate; + + +yyrecover_default: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyrecover_errlab; + goto yyrecover_reduce; + + +yyrecover_reduce: + yylen = yyr2[yyn]; + /* YYPOPSTACK */ + reps->state -= yylen; + yylen = 0; + + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *reps->state; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *reps->state + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yyrecover_newstate; + +yyrecover_errlab: + return 0; +} + +static yy_repair_terms * +yyrecover(yy_state_t *yyss, yy_state_t *yyssp, int yychar<%= output.user_formals %>) +{ + yysymbol_kind_t yytoken = YYTRANSLATE (yychar); + yy_repair_terms *rep_terms = YY_NULLPTR; + int count = 0; + + yy_repairs *head = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + yy_repairs *current = head; + yy_repairs *tail = head; + YYPTRDIFF_T stack_length = yyssp - yyss + 1; + + head->id = count; + head->next = 0; + head->stack_length = stack_length; + head->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + head->state = head->states + (yyssp - yyss); + YYCOPY (head->states, yyss, stack_length); + head->repair_length = 0; + head->prev_repair = 0; + + stack_length = (stack_length * 2 > 100) ? (stack_length * 2) : 100; + count++; + + while (current) + { + int yystate = *current->state; + int yyn = yypact[yystate]; + /* See also: yypcontext_expected_tokens */ + if (!yypact_value_is_default (yyn)) + { + int yyxbegin = yyn < 0 ? -yyn : 0; + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + { + if (yyx != YYSYMBOL_YYerror) + { + if (current->repair_length + 1 > YYMAXREPAIR(<%= output.parse_param_name %>)) + continue; + + yy_repairs *reps = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + reps->id = count; + reps->next = 0; + reps->stack_length = stack_length; + reps->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + reps->state = reps->states + (current->state - current->states); + YYCOPY (reps->states, current->states, current->state - current->states + 1); + reps->repair_length = current->repair_length + 1; + reps->prev_repair = current; + reps->repair.type = inserting; + reps->repair.term = (yysymbol_kind_t) yyx; + + /* Process PDA assuming next token is yyx */ + if (! yy_process_repairs (reps, (yysymbol_kind_t)yyx)) + { + YYFREE (reps); + continue; + } + + tail->next = reps; + tail = reps; + count++; + + if (yyx == yytoken) + { + rep_terms = yy_create_repair_terms (current<%= output.user_args %>); + YYDPRINTF ((stderr, "repair_terms found. id: %d, length: %d\n", rep_terms->id, rep_terms->length)); + yy_print_repairs (current<%= output.user_args %>); + yy_print_repair_terms (rep_terms<%= output.user_args %>); + + goto done; + } + + YYDPRINTF ((stderr, + "New repairs is enqueued. count: %d, yystate: %d, yyx: %d\n", + count, yystate, yyx)); + yy_print_repairs (reps<%= output.user_args %>); + } + } + } + + current = current->next; + } + +done: + + yy_free_repairs(head<%= output.user_args %>); + + if (!rep_terms) + { + YYDPRINTF ((stderr, "repair_terms not found\n")); + } + + return rep_terms; +} +<%- end -%> + + + +/*----------. +| yyparse. | +`----------*/ + +int +yyparse (<%= output.parse_param %>) +{ +<%# b4_declare_scanner_communication_variables -%> +/* Lookahead token kind. */ +int yychar; + + +/* The semantic value of the lookahead symbol. */ +/* Default value used for initialization, for pacifying older GCCs + or non-GCC compilers. */ +#ifdef __cplusplus +static const YYSTYPE yyval_default = {}; +(void) yyval_default; +#else +YY_INITIAL_VALUE (static const YYSTYPE yyval_default;) +#endif +YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); + +/* Location data for the lookahead symbol. */ +static const YYLTYPE yyloc_default +# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + = { 1, 1, 1, 1 } +# endif +; +YYLTYPE yylloc = yyloc_default; + +<%# b4_declare_parser_state_variables -%> + /* Number of syntax errors so far. */ + int yynerrs = 0; + YY_USE (yynerrs); /* Silence compiler warning. */ + + yy_state_fast_t yystate = 0; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus = 0; + + /* Refer to the stacks through separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ + + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; + + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; + + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; + + /* The location stack: array, bottom, top. */ + YYLTYPE yylsa[YYINITDEPTH]; + YYLTYPE *yyls = yylsa; + YYLTYPE *yylsp = yyls; + + int yyn; + /* The return value of yyparse. */ + int yyresult; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; + YYLTYPE yyloc; + + /* The locations where the error started and ended. */ + YYLTYPE yyerror_range[3]; +<%- if output.error_recovery -%> + yy_repair_terms *rep_terms = 0; + yy_term term_backup; + int rep_terms_index; + int yychar_backup; +<%- end -%> + + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char *yymsg = yymsgbuf; + YYPTRDIFF_T yymsg_alloc = sizeof yymsgbuf; + +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N), yylsp -= (N)) + + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; + + YYDPRINTF ((stderr, "Starting parse\n")); + + yychar = YYEMPTY; /* Cause a token to be read. */ + + +<%# b4_user_initial_action -%> +<%= output.user_initial_action("/* User initialization code. */") %> +#line [@oline@] [@ofile@] + + yylsp[0] = yylloc; + goto yysetstate; + + +/*------------------------------------------------------------. +| yynewstate -- push a new state, which is found in yystate. | +`------------------------------------------------------------*/ +yynewstate: + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; + + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + + if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + YYNOMEM; +#else + { + /* Get the current used size of the three stacks, in elements. */ + YYPTRDIFF_T yysize = yyssp - yyss + 1; + +# if defined yyoverflow + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + yy_state_t *yyss1 = yyss; + YYSTYPE *yyvs1 = yyvs; + YYLTYPE *yyls1 = yyls; + + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow (YY_("memory exhausted"), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), + &yyls1, yysize * YYSIZEOF (*yylsp), + &yystacksize); + yyss = yyss1; + yyvs = yyvs1; + yyls = yyls1; + } +# else /* defined YYSTACK_RELOCATE */ + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + YYNOMEM; + yystacksize *= 2; + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yy_state_t *yyss1 = yyss; + union yyalloc *yyptr = + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); + if (! yyptr) + YYNOMEM; + YYSTACK_RELOCATE (yyss_alloc, yyss); + YYSTACK_RELOCATE (yyvs_alloc, yyvs); + YYSTACK_RELOCATE (yyls_alloc, yyls); +# undef YYSTACK_RELOCATE + if (yyss1 != yyssa) + YYSTACK_FREE (yyss1); + } +# endif + + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; + yylsp = yyls + yysize - 1; + + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END + + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ + + + if (yystate == YYFINAL) + YYACCEPT; + + goto yybackup; + + +/*-----------. +| yybackup. | +`-----------*/ +yybackup: + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ + + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yydefault; + + /* Not known => get a lookahead token if don't already have one. */ + +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + if (yychar == YYEMPTY && rep_terms) + { + + if (rep_terms_index < rep_terms->length) + { + YYDPRINTF ((stderr, "An error recovery token is used\n")); + yy_term term = rep_terms->terms[rep_terms_index]; + yytoken = term.kind; + yylval = term.value; + yylloc = term.location; + yychar = yytranslate_inverted[yytoken]; + YY_SYMBOL_PRINT ("Next error recovery token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + rep_terms_index++; + } + else + { + YYDPRINTF ((stderr, "Error recovery is completed\n")); + yytoken = term_backup.kind; + yylval = term_backup.value; + yylloc = term_backup.location; + yychar = yychar_backup; + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + + YYFREE (rep_terms); + rep_terms = 0; + yychar_backup = 0; + } + } + } +<%- end -%> + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token\n")); + yychar = yylex <%= output.yylex_formals %>; + } + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + yychar = <%= output.eof_symbol.id.s_value %>; + yytoken = <%= output.eof_symbol.enum_name %>; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else if (yychar == <%= output.error_symbol.id.s_value %>) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = <%= output.undef_symbol.id.s_value %>; + yytoken = <%= output.error_symbol.enum_name %>; + yyerror_range[1] = yylloc; + goto yyerrlab1; + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyerrlab; + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc<%= output.user_args %>); + yystate = yyn; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + *++yylsp = yylloc; +<%= output.after_shift_function("/* %after-shift code. */") %> + + /* Discard the shifted token. */ + yychar = YYEMPTY; + goto yynewstate; + + +/*-----------------------------------------------------------. +| yydefault -- do the default action for the current state. | +`-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + +/*-----------------------------. +| yyreduce -- do a reduction. | +`-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + '$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1-yylen]; +<%= output.before_reduce_function("/* %before-reduce function. */") %> + + /* Default location. */ + YYLLOC_DEFAULT (yyloc, (yylsp - yylen), yylen); + yyerror_range[1] = yyloc; + YY_REDUCE_PRINT (yyn<%= output.user_args %>); + switch (yyn) + { +<%= output.user_actions -%> + + default: break; + } + /* User semantic actions sometimes alter yychar, and that requires + that yytoken be updated with the new translation. We take the + approach of translating immediately before every use of yytoken. + One alternative is translating here after every semantic action, + but that translation would be missed if the semantic action invokes + YYABORT, YYACCEPT, or YYERROR immediately after altering yychar or + if it invokes YYBACKUP. In the case of YYABORT or YYACCEPT, an + incorrect destructor might then be invoked immediately. In the + case of YYERROR or YYBACKUP, subsequent parser actions might lead + to an incorrect destructor call or verbose syntax error message + before the lookahead is translated. */ + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc<%= output.user_args %>); + + YYPOPSTACK (yylen); +<%= output.after_reduce_function("/* %after-reduce function. */") %> + yylen = 0; + + *++yyvsp = yyval; + *++yylsp = yyloc; + + /* Now 'shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yynewstate; + + +/*--------------------------------------. +| yyerrlab -- here on detecting error. | +`--------------------------------------*/ +yyerrlab: + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; + { + yypcontext_t yyctx + = {yyssp, yytoken, &yylloc}; + char const *yymsgp = YY_("syntax error"); + int yysyntax_error_status; + yysyntax_error_status = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + if (yysyntax_error_status == 0) + yymsgp = yymsg; + else if (yysyntax_error_status == -1) + { + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + yymsg = YY_CAST (char *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, yymsg_alloc))); + if (yymsg) + { + yysyntax_error_status + = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + yymsgp = yymsg; + } + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + yysyntax_error_status = YYENOMEM; + } + } + yyerror (<%= output.yyerror_args %>, yymsgp); + if (yysyntax_error_status == YYENOMEM) + YYNOMEM; + } + } + + yyerror_range[1] = yylloc; + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + /* Return failure if at end of input. */ + if (yychar == <%= output.eof_symbol.id.s_value %>) + YYABORT; + } + else + { + yydestruct ("Error: discarding", + yytoken, &yylval, &yylloc<%= output.user_args %>); + yychar = YYEMPTY; + } + } + + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; + + +/*---------------------------------------------------. +| yyerrorlab -- error raised explicitly by YYERROR. | +`---------------------------------------------------*/ +yyerrorlab: + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; + ++yynerrs; + + /* Do not reclaim the symbols of the rule whose action triggered + this YYERROR. */ + YYPOPSTACK (yylen); +<%= output.after_pop_stack_function("yylen", "/* %after-pop-stack function. */") %> + yylen = 0; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + yystate = *yyssp; + goto yyerrlab1; + + +/*-------------------------------------------------------------. +| yyerrlab1 -- common code for both syntax error and YYERROR. | +`-------------------------------------------------------------*/ +yyerrlab1: +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + rep_terms = yyrecover (yyss, yyssp, yychar<%= output.user_args %>); + if (rep_terms) + { + for (int i = 0; i < rep_terms->length; i++) + { + yy_term *term = &rep_terms->terms[i]; + yy_error_token_initialize (term->kind, &term->value, &term->location<%= output.user_args %>); + } + + yychar_backup = yychar; + /* Can be packed into (the tail of) rep_terms? */ + term_backup.kind = yytoken; + term_backup.value = yylval; + term_backup.location = yylloc; + rep_terms_index = 0; + yychar = YYEMPTY; + + goto yybackup; + } + } +<%- end -%> + yyerrstatus = 3; /* Each real token shifted decrements this. */ + + /* Pop stack until we find a state that shifts the error token. */ + for (;;) + { + yyn = yypact[yystate]; + if (!yypact_value_is_default (yyn)) + { + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) + { + yyn = yytable[yyn]; + if (0 < yyn) + break; + } + } + + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + + yyerror_range[1] = *yylsp; + yydestruct ("Error: popping", + YY_ACCESSING_SYMBOL (yystate), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); +<%= output.after_pop_stack_function(1, "/* %after-pop-stack function. */") %> + yystate = *yyssp; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + } + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + + yyerror_range[2] = yylloc; + ++yylsp; + YYLLOC_DEFAULT (*yylsp, yyerror_range, 2); + + /* Shift the error token. */ + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp<%= output.user_args %>); +<%= output.after_shift_error_token_function("/* %after-shift-error-token code. */") %> + + yystate = yyn; + goto yynewstate; + + +/*-------------------------------------. +| yyacceptlab -- YYACCEPT comes here. | +`-------------------------------------*/ +yyacceptlab: + yyresult = 0; + goto yyreturnlab; + + +/*-----------------------------------. +| yyabortlab -- YYABORT comes here. | +`-----------------------------------*/ +yyabortlab: + yyresult = 1; + goto yyreturnlab; + + +/*-----------------------------------------------------------. +| yyexhaustedlab -- YYNOMEM (memory exhaustion) comes here. | +`-----------------------------------------------------------*/ +yyexhaustedlab: + yyerror (<%= output.yyerror_args %>, YY_("memory exhausted")); + yyresult = 2; + goto yyreturnlab; + + +/*----------------------------------------------------------. +| yyreturnlab -- parsing is finished, clean up and return. | +`----------------------------------------------------------*/ +yyreturnlab: + if (yychar != YYEMPTY) + { + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = YYTRANSLATE (yychar); + yydestruct ("Cleanup: discarding lookahead", + yytoken, &yylval, &yylloc<%= output.user_args %>); + } + /* Do not reclaim the symbols of the rule whose action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK (yylen); + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + while (yyssp != yyss) + { + yydestruct ("Cleanup: popping", + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); + } +#ifndef yyoverflow + if (yyss != yyssa) + YYSTACK_FREE (yyss); +#endif + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + return yyresult; +} + +<%# b4_percent_code_get([[epilogue]]) -%> +<%- if output.aux.epilogue -%> +#line <%= output.aux.epilogue_first_lineno - 1 %> "<%= output.grammar_file_path %>" +<%= output.aux.epilogue -%> +<%- end -%> + diff --git a/tool/lrama/template/bison/yacc.h b/tool/lrama/template/bison/yacc.h new file mode 100644 index 0000000000..848dbf5961 --- /dev/null +++ b/tool/lrama/template/bison/yacc.h @@ -0,0 +1,40 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison interface for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ +<%= output.render_partial("bison/_yacc.h") %> diff --git a/tool/lrama/template/diagram/diagram.html b/tool/lrama/template/diagram/diagram.html new file mode 100644 index 0000000000..3e87e6e519 --- /dev/null +++ b/tool/lrama/template/diagram/diagram.html @@ -0,0 +1,102 @@ +<!DOCTYPE html> +<html> +<head> + <title>Lrama syntax diagrams</title> + + <style> + <%= output.default_style %> + .diagram-header { + display: inline-block; + font-weight: bold; + font-size: 18px; + margin-bottom: -8px; + text-align: center; + } + + svg { + width: 100%; + } + + svg.railroad-diagram g.non-terminal text { + cursor: pointer; + } + + h2.hover-header { + background-color: #90ee90; + } + + svg.railroad-diagram g.non-terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + + svg.railroad-diagram g.terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + </style> +</head> + +<body align="center"> + <%= output.diagrams %> + <script> + document.addEventListener("DOMContentLoaded", () => { + function addHoverEffect(selector, hoverClass, relatedSelector, relatedHoverClass, getTextElements) { + document.querySelectorAll(selector).forEach(element => { + element.addEventListener("mouseenter", () => { + element.classList.add(hoverClass); + getTextElements(element).forEach(textEl => { + if (!relatedSelector) return; + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.add(relatedHoverClass); + }); + }); + }); + + element.addEventListener("mouseleave", () => { + element.classList.remove(hoverClass); + if (!relatedSelector) return; + getTextElements(element).forEach(textEl => { + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.remove(relatedHoverClass); + }); + }); + }); + }); + } + + function getElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => el.textContent.trim() === text.trim()); + } + + function getParentElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => + [...el.querySelectorAll("text")].some(textEl => textEl.textContent.trim() === text.trim()) + ); + } + + function scrollToMatchingHeader() { + document.querySelectorAll("g.non-terminal").forEach(element => { + element.addEventListener("click", () => { + const textElements = [...element.querySelectorAll("text")]; + for (const textEl of textElements) { + const targetHeader = getElementsByText("h2", textEl.textContent)[0]; + if (targetHeader) { + targetHeader.scrollIntoView({ behavior: "smooth", block: "start" }); + break; + } + } + }); + }); + } + + addHoverEffect("h2", "hover-header", "g.non-terminal", "hover-g", element => [element]); + addHoverEffect("g.non-terminal", "hover-g", "h2", "hover-header", + element => [...element.querySelectorAll("text")] + ); + addHoverEffect("g.terminal", "hover-g", "", "", element => [element]); + scrollToMatchingHeader(); + }); + </script> +</body> +</html> diff --git a/tool/m4/_colorize_result_prepare.m4 b/tool/m4/_colorize_result_prepare.m4 index cc2bbaa703..8439acf3ed 100644 --- a/tool/m4/_colorize_result_prepare.m4 +++ b/tool/m4/_colorize_result_prepare.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([_COLORIZE_RESULT_PREPARE], [ msg_checking= msg_result_yes= msg_result_no= msg_result_other= msg_reset= AS_CASE(["x${CONFIGURE_TTY}"], @@ -9,6 +9,7 @@ AC_DEFUN([_COLORIZE_RESULT_PREPARE], [ [configure_tty=0])]) AS_IF([test $configure_tty -eq 1], [ msg_begin="`tput smso 2>/dev/null`" + AS_IF([test -z "$msg_begin"], [msg_begin="`tput so 2>/dev/null`"]) AS_CASE(["$msg_begin"], ['@<:@'*m], [msg_begin="`echo "$msg_begin" | sed ['s/[0-9]*m$//']`" msg_checking="${msg_begin}33m" diff --git a/tool/m4/ac_msg_result.m4 b/tool/m4/ac_msg_result.m4 index bcc7a63d7d..12a3617c0d 100644 --- a/tool/m4/ac_msg_result.m4 +++ b/tool/m4/ac_msg_result.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([AC_MSG_RESULT], [dnl { _AS_ECHO_LOG([result: $1]) COLORIZE_RESULT([$1]); dnl diff --git a/tool/m4/colorize_result.m4 b/tool/m4/colorize_result.m4 index 19ff6d49e8..83912040e5 100644 --- a/tool/m4/colorize_result.m4 +++ b/tool/m4/colorize_result.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([COLORIZE_RESULT], [AC_REQUIRE([_COLORIZE_RESULT_PREPARE])dnl AS_LITERAL_IF([$1], [m4_case([$1], diff --git a/tool/m4/ruby_append_option.m4 b/tool/m4/ruby_append_option.m4 index 07b6537cb7..8cd2741ae8 100644 --- a/tool/m4/ruby_append_option.m4 +++ b/tool/m4/ruby_append_option.m4 @@ -1,5 +1,9 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_APPEND_OPTION], [# RUBY_APPEND_OPTION($1) AS_CASE([" [$]{$1-} "], [*" $2 "*], [], [' '], [ $1="$2"], [ $1="[$]$1 $2"])])dnl +AC_DEFUN([RUBY_PREPEND_OPTION], + [# RUBY_PREPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_append_options.m4 b/tool/m4/ruby_append_options.m4 index 164a4ab654..14213111ca 100644 --- a/tool/m4/ruby_append_options.m4 +++ b/tool/m4/ruby_append_options.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_APPEND_OPTIONS], [# RUBY_APPEND_OPTIONS($1) for rb_opt in $2; do diff --git a/tool/m4/ruby_check_builtin_func.m4 b/tool/m4/ruby_check_builtin_func.m4 index 3fe5caf429..40abc78ef8 100644 --- a/tool/m4/ruby_check_builtin_func.m4 +++ b/tool/m4/ruby_check_builtin_func.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_CHECK_BUILTIN_FUNC], [dnl AC_CACHE_CHECK([for $1], AS_TR_SH(rb_cv_builtin_$1), [AC_LINK_IFELSE( diff --git a/tool/m4/ruby_check_builtin_overflow.m4 b/tool/m4/ruby_check_builtin_overflow.m4 new file mode 100644 index 0000000000..8568d2c6d9 --- /dev/null +++ b/tool/m4/ruby_check_builtin_overflow.m4 @@ -0,0 +1,28 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_OVERFLOW], [dnl +{ # $0($1) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow, [int x;__builtin_[$1]_overflow(0,0,&x)]) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow_p, [__builtin_[$1]_overflow_p(0,0,(int)0)]) + + AS_IF([test "$rb_cv_builtin___builtin_[$1]_overflow" != no], [ + AC_CACHE_CHECK(for __builtin_[$1]_overflow with long long arguments, rb_cv_use___builtin_[$1]_overflow_long_long, [ + AC_LINK_IFELSE([AC_LANG_SOURCE([[ +@%:@pragma clang optimize off + +int +main(void) +{ + long long x = 0, y; + __builtin_$1_overflow(x, x, &y); + + return 0; +} +]])], + rb_cv_use___builtin_[$1]_overflow_long_long=yes, + rb_cv_use___builtin_[$1]_overflow_long_long=no)]) + ]) + AS_IF([test "$rb_cv_use___builtin_[$1]_overflow_long_long" = yes], [ + AC_DEFINE(USE___BUILTIN_[]AS_TR_CPP($1)_OVERFLOW_LONG_LONG, 1) + ]) +} +])dnl diff --git a/tool/m4/ruby_check_builtin_setjmp.m4 b/tool/m4/ruby_check_builtin_setjmp.m4 index a4289e2e9d..1e5d9b3028 100644 --- a/tool/m4/ruby_check_builtin_setjmp.m4 +++ b/tool/m4/ruby_check_builtin_setjmp.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_CHECK_BUILTIN_SETJMP], [ AS_IF([test x"${ac_cv_func___builtin_setjmp}" = xyes], [ unset ac_cv_func___builtin_setjmp @@ -8,19 +8,19 @@ AC_CACHE_CHECK(for __builtin_setjmp, ac_cv_func___builtin_setjmp, ac_cv_func___builtin_setjmp=no for cast in "" "(void **)"; do RUBY_WERROR_FLAG( - [AC_TRY_LINK([@%:@include <setjmp.h> + [AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include <setjmp.h> @%:@include <stdio.h> jmp_buf jb; @%:@ifdef NORETURN NORETURN(void t(void)); @%:@endif void t(void) {__builtin_longjmp($cast jb, 1);} - int jump(void) {(void)(__builtin_setjmp($cast jb) ? 1 : 0); return 0;}], - [ + int jump(void) {(void)(__builtin_setjmp($cast jb) ? 1 : 0); return 0;}]], + [[ void (*volatile f)(void) = t; if (!jump()) printf("%d\n", f != 0); - ], - [ac_cv_func___builtin_setjmp="yes with cast ($cast)"]) + ]])], + [ac_cv_func___builtin_setjmp="yes${cast:+ with cast ($cast)}"]) ]) test "$ac_cv_func___builtin_setjmp" = no || break done]) diff --git a/tool/m4/ruby_check_header.m4 b/tool/m4/ruby_check_header.m4 new file mode 100644 index 0000000000..6fec9d16c5 --- /dev/null +++ b/tool/m4/ruby_check_header.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_HEADER], + [# RUBY_CHECK_HEADER($@) + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS m4_if([$5], [], [$INCFLAGS], [$5])" + AC_CHECK_HEADERS([$1], [$2], [$3], [$4]) + CPPFLAGS="$save_CPPFLAGS" + unset save_CPPFLAGS]) diff --git a/tool/m4/ruby_check_printf_prefix.m4 b/tool/m4/ruby_check_printf_prefix.m4 index 9007c18c0a..15bb4aee87 100644 --- a/tool/m4/ruby_check_printf_prefix.m4 +++ b/tool/m4/ruby_check_printf_prefix.m4 @@ -1,11 +1,10 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_CHECK_PRINTF_PREFIX], [ AC_CACHE_CHECK([for printf prefix for $1], [rb_cv_pri_prefix_]AS_TR_SH($1),[ [rb_cv_pri_prefix_]AS_TR_SH($1)=[NONE] RUBY_WERROR_FLAG(RUBY_APPEND_OPTIONS(CFLAGS, $rb_cv_wsuppress_flags) for pri in $2; do - AC_TRY_COMPILE( - [@%:@include <stdio.h> + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <stdio.h> @%:@include <stddef.h> @%:@ifdef __GNUC__ @%:@if defined __MINGW_PRINTF_FORMAT @@ -18,9 +17,9 @@ AC_CACHE_CHECK([for printf prefix for $1], [rb_cv_pri_prefix_]AS_TR_SH($1),[ @%:@else @%:@define PRINTF_ARGS(decl, string_index, first_to_check) decl @%:@endif - PRINTF_ARGS(void test_sprintf(const char*, ...), 1, 2);], - [printf("%]${pri}[d", (]$1[)42); - test_sprintf("%]${pri}[d", (]$1[)42);], + PRINTF_ARGS(void test_sprintf(const char*, ...), 1, 2);]], + [[printf("%]${pri}[d", (]$1[)42); + test_sprintf("%]${pri}[d", (]$1[)42);]])], [rb_cv_pri_prefix_]AS_TR_SH($1)[=[$pri]; break]) done)]) AS_IF([test "[$rb_cv_pri_prefix_]AS_TR_SH($1)" != NONE], [ diff --git a/tool/m4/ruby_check_setjmp.m4 b/tool/m4/ruby_check_setjmp.m4 index 59f38581b8..6020b766b8 100644 --- a/tool/m4/ruby_check_setjmp.m4 +++ b/tool/m4/ruby_check_setjmp.m4 @@ -1,15 +1,15 @@ -# -*- Autoconf -*- -# used for AC_ARG_WITH(setjmp-type) +dnl -*- Autoconf -*- +dnl used for AC_ARG_WITH(setjmp-type) AC_DEFUN([RUBY_CHECK_SETJMP], [ AC_CACHE_CHECK([for ]$1[ as a macro or function], ac_cv_func_$1, - [AC_TRY_COMPILE([ + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ @%:@include <setjmp.h> ]AC_INCLUDES_DEFAULT([$3])[ @%:@define JMPARGS_1 env @%:@define JMPARGS_2 env,1 @%:@define JMPARGS JMPARGS_]m4_ifval($2,2,1)[ -], - m4_ifval($2,$2,jmp_buf)[ env; $1(JMPARGS);], +]], + [m4_ifval($2,$2,jmp_buf)[ env; $1(JMPARGS);]])], ac_cv_func_$1=yes, ac_cv_func_$1=no)] ) diff --git a/tool/m4/ruby_check_signedness.m4 b/tool/m4/ruby_check_signedness.m4 index 5376efa208..f9fbb3c088 100644 --- a/tool/m4/ruby_check_signedness.m4 +++ b/tool/m4/ruby_check_signedness.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- dnl RUBY_CHECK_SIGNEDNESS [typename] [if-signed] [if-unsigned] [included] AC_DEFUN([RUBY_CHECK_SIGNEDNESS], [dnl AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([AC_INCLUDES_DEFAULT([$4])], [($1)-1 > 0])], diff --git a/tool/m4/ruby_check_sizeof.m4 b/tool/m4/ruby_check_sizeof.m4 index 38d7918f2e..975ac6c9be 100644 --- a/tool/m4/ruby_check_sizeof.m4 +++ b/tool/m4/ruby_check_sizeof.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- dnl RUBY_CHECK_SIZEOF [typename], [maybe same size types], [macros], [include] AC_DEFUN([RUBY_CHECK_SIZEOF], [dnl diff --git a/tool/m4/ruby_check_sysconf.m4 b/tool/m4/ruby_check_sysconf.m4 index f6b247a16f..f554786e77 100644 --- a/tool/m4/ruby_check_sysconf.m4 +++ b/tool/m4/ruby_check_sysconf.m4 @@ -1,9 +1,9 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_CHECK_SYSCONF], [dnl AC_CACHE_CHECK([whether _SC_$1 is supported], rb_cv_have_sc_[]m4_tolower($1), - [AC_TRY_COMPILE([#include <unistd.h> - ], - [_SC_$1 >= 0], + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> + ]], + [[_SC_$1 >= 0]])], rb_cv_have_sc_[]m4_tolower($1)=yes, rb_cv_have_sc_[]m4_tolower($1)=no) ]) diff --git a/tool/m4/ruby_cppoutfile.m4 b/tool/m4/ruby_cppoutfile.m4 index 7c81c4f354..976cbb1c43 100644 --- a/tool/m4/ruby_cppoutfile.m4 +++ b/tool/m4/ruby_cppoutfile.m4 @@ -1,11 +1,11 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_CPPOUTFILE], [AC_CACHE_CHECK(whether ${CPP} accepts -o, rb_cv_cppoutfile, [save_CPPFLAGS="$CPPFLAGS" CPPFLAGS='-o conftest-1.i' rb_cv_cppoutfile=no -AC_TRY_CPP([test-for-cppout], - [grep test-for-cppout conftest-1.i > /dev/null && rb_cv_cppoutfile=yes]) +AC_PREPROC_IFELSE([AC_LANG_SOURCE([[test-for-cppout]])], + [grep test-for-cppout conftest-1.i > /dev/null && rb_cv_cppoutfile=yes]) CPPFLAGS="$save_CPPFLAGS" rm -f conftest*]) AS_IF([test "$rb_cv_cppoutfile" = yes], [ diff --git a/tool/m4/ruby_decl_attribute.m4 b/tool/m4/ruby_decl_attribute.m4 index 3187b9be60..a8a73dc870 100644 --- a/tool/m4/ruby_decl_attribute.m4 +++ b/tool/m4/ruby_decl_attribute.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- dnl RUBY_DECL_ATTRIBUTE(attrib, macroname, cachevar, condition, type, code) AC_DEFUN([RUBY_DECL_ATTRIBUTE], [dnl m4_ifval([$2], dnl @@ -21,7 +21,7 @@ for mac in \ "__declspec(attrib_code) x" \ x; do m4_ifval([$4],mac="$mac"${rbcv_cond+" /* only if $rbcv_cond */"}) - AC_TRY_COMPILE( + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([ m4_ifval([$4],${rbcv_cond+[@%:@if ]$rbcv_cond}) [@%:@define ]attrib[](attrib_params)[ $mac] m4_ifval([$4],${rbcv_cond+[@%:@else]} @@ -30,7 +30,7 @@ ${rbcv_cond+[@%:@endif]}) $6 @%:@define mesg ("") @%:@define san "address" - attrib[](attrib_params)[;], [], + attrib[](attrib_params)[;]], [[]])], [rbcv="$mac"; break]) done ])]) diff --git a/tool/m4/ruby_default_arch.m4 b/tool/m4/ruby_default_arch.m4 index a53bb5fc41..2f25ba81ee 100644 --- a/tool/m4/ruby_default_arch.m4 +++ b/tool/m4/ruby_default_arch.m4 @@ -1,11 +1,21 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_DEFAULT_ARCH], [ +# Set ARCH_FLAG for different width but family CPU AC_MSG_CHECKING([arch option]) -AS_CASE([$1], - [*64], [ARCH_FLAG=-m64], - [[i[3-6]86]], [ARCH_FLAG=-m32], - [AC_MSG_ERROR(unknown target architecture: $target_archs)] - ) -AC_MSG_RESULT([$ARCH_FLAG]) -]) -dnl +AS_CASE([$1:"$host_cpu"], + [arm64:arm*], [ARCH_FLAG=-m64], + [arm*:arm*], [ARCH_FLAG=-m32], + [x86_64:[i[3-6]86]], [ARCH_FLAG=-m64], + [x64:x86_64], [], + [[i[3-6]86]:x86_64], [ARCH_FLAG=-m32], + [ppc64:ppc*], [ARCH_FLAG=-m64], + [ppc*:ppc64], [ARCH_FLAG=-m32], + [ + ARCH_FLAG= + for flag in "-arch "$1 -march=$1; do + _RUBY_TRY_CFLAGS([$]flag, [ARCH_FLAG="[$]flag"]) + test x"$ARCH_FLAG" = x || break + done] +) +AC_MSG_RESULT([${ARCH_FLAG:-'(none)'}]) +])dnl diff --git a/tool/m4/ruby_define_if.m4 b/tool/m4/ruby_define_if.m4 index b18b104aad..aba55783a2 100644 --- a/tool/m4/ruby_define_if.m4 +++ b/tool/m4/ruby_define_if.m4 @@ -1,12 +1,6 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_DEFINE_IF], [dnl - m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])cat <<EOH >> confdefs.h -@%:@if $1 -EOH -])dnl + m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])printf "@%:@if %s\n" "$1" >>confdefs.h]) AC_DEFINE_UNQUOTED($2, $3)dnl - m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])cat <<EOH >> confdefs.h -@%:@endif /* $1 */ -EOH -])dnl + m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])printf "@%:@endif /* %s */\n" "$1" >>confdefs.h]) ])dnl diff --git a/tool/m4/ruby_defint.m4 b/tool/m4/ruby_defint.m4 index e5b46a788f..7f262a73fc 100644 --- a/tool/m4/ruby_defint.m4 +++ b/tool/m4/ruby_defint.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- dnl RUBY_DEFINT TYPENAME, SIZE, [UNSIGNED], [INCLUDES = DEFAULT-INCLUDES] AC_DEFUN([RUBY_DEFINT], [dnl AS_VAR_PUSHDEF([cond], [rb_defint_cond])dnl @@ -17,7 +17,8 @@ typedef $1 t; int s = sizeof(t) == 42;])], ["${ac_cv_sizeof___int128@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int128"], [ rb_cv_type_$1=no])])]) AS_IF([test "${rb_cv_type_$1}" != no], [ - type="${rb_cv_type_$1@%:@@%:@unsigned }" + type="${rb_cv_type_$1@%:@@%:@*signed }" + AS_IF([test "$type" = "long long"], [type=long_long]) AS_IF([test "$type" != yes && eval 'test -n "${ac_cv_sizeof_'$type'+set}"'], [ eval cond='"${ac_cv_sizeof_'$type'}"' AS_CASE([$cond], [*:*], [ diff --git a/tool/m4/ruby_dtrace_available.m4 b/tool/m4/ruby_dtrace_available.m4 index 79586d152c..e03b7762fc 100644 --- a/tool/m4/ruby_dtrace_available.m4 +++ b/tool/m4/ruby_dtrace_available.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_DTRACE_AVAILABLE], [AC_CACHE_CHECK(whether dtrace USDT is available, rb_cv_dtrace_available, [ @@ -7,7 +7,7 @@ AC_DEFUN([RUBY_DTRACE_AVAILABLE], AS_FOR(opt, rb_dtrace_opt, ["-xnolibs" ""], [dnl AS_IF([$DTRACE opt -h -o conftest_provider.h -s conftest_provider.d >/dev/null 2>/dev/null], [], [continue]) - AC_TRY_COMPILE([@%:@include "conftest_provider.h"], [CONFTEST_FIRE();], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include "conftest_provider.h"]], [[CONFTEST_FIRE();]])], [], [continue]) # DTrace is available on the system rb_cv_dtrace_available=yes${rb_dtrace_opt:+"(opt)"} diff --git a/tool/m4/ruby_dtrace_postprocess.m4 b/tool/m4/ruby_dtrace_postprocess.m4 index 9ef088b3f8..6fd6de7c9a 100644 --- a/tool/m4/ruby_dtrace_postprocess.m4 +++ b/tool/m4/ruby_dtrace_postprocess.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_DTRACE_POSTPROCESS], [AC_CACHE_CHECK(whether $DTRACE needs post processing, rb_cv_prog_dtrace_g, [ @@ -12,7 +12,7 @@ _PROBES $DTRACE ${DTRACE_OPT} -h -o conftest_provider.h -s conftest_provider.d >/dev/null 2>/dev/null && : }], [ - AC_TRY_COMPILE([@%:@include "conftest_provider.h"], [CONFTEST_FIRE();], [ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include "conftest_provider.h"]], [[CONFTEST_FIRE();]])],[ AS_IF([{ cp -p conftest.${ac_objext} conftest.${ac_objext}.save && $DTRACE ${DTRACE_OPT} -G -s conftest_provider.d conftest.${ac_objext} 2>/dev/null && diff --git a/tool/m4/ruby_func_attribute.m4 b/tool/m4/ruby_func_attribute.m4 index b265fd41da..bce26fc16a 100644 --- a/tool/m4/ruby_func_attribute.m4 +++ b/tool/m4/ruby_func_attribute.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- dnl RUBY_FUNC_ATTRIBUTE(attrib, macroname, cachevar, condition) AC_DEFUN([RUBY_FUNC_ATTRIBUTE], [dnl RUBY_DECL_ATTRIBUTE([$1], [$2], [$3], [$4], diff --git a/tool/m4/ruby_mingw32.m4 b/tool/m4/ruby_mingw32.m4 index f44fe5575c..98e922340b 100644 --- a/tool/m4/ruby_mingw32.m4 +++ b/tool/m4/ruby_mingw32.m4 @@ -1,13 +1,13 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_MINGW32], [AS_CASE(["$host_os"], [cygwin*], [ AC_CACHE_CHECK(for mingw32 environment, rb_cv_mingw32, -[AC_TRY_CPP([ +[AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ #ifndef __MINGW32__ # error #endif -], rb_cv_mingw32=yes,rb_cv_mingw32=no) +]])],[rb_cv_mingw32=yes],[rb_cv_mingw32=no]) rm -f conftest*]) AS_IF([test "$rb_cv_mingw32" = yes], [ target_os="mingw32" diff --git a/tool/m4/ruby_modular_gc.m4 b/tool/m4/ruby_modular_gc.m4 new file mode 100644 index 0000000000..661fce2e60 --- /dev/null +++ b/tool/m4/ruby_modular_gc.m4 @@ -0,0 +1,41 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_MODULAR_GC],[ +AC_ARG_WITH(modular-gc, + AS_HELP_STRING([--with-modular-gc=DIR], + [Enable replacement of Ruby's GC from a modular library in the specified directory.]), + [modular_gc_dir=$withval], [unset modular_gc_dir] +) + +AS_IF([test "$modular_gc_dir" = yes], [ + AC_MSG_ERROR(you must specify a directory when using --with-modular-gc) +]) + +AC_MSG_CHECKING([if building with modular GC support]) +AS_IF([test x"$modular_gc_dir" != x], [ + AC_MSG_RESULT([yes]) + + # Ensure that modular_gc_dir is always an absolute path so that Ruby + # never loads a modular GC from a relative path + AS_CASE(["$modular_gc_dir"], + [/*], [], + [test "$load_relative" = yes || modular_gc_dir="$prefix/$modular_gc_dir"] + ) + + # Ensure that modular_gc_dir always terminates with a / + AS_CASE(["$modular_gc_dir"], + [*/], [], + [modular_gc_dir="$modular_gc_dir/"] + ) + + AC_DEFINE([USE_MODULAR_GC], [1]) + + modular_gc_summary="yes (in $modular_gc_dir)" +], [ + AC_MSG_RESULT([no]) + AC_DEFINE([USE_MODULAR_GC], [0]) + + modular_gc_summary="no" +]) + +AC_SUBST(modular_gc_dir, "${modular_gc_dir}") +])dnl diff --git a/tool/m4/ruby_prepend_option.m4 b/tool/m4/ruby_prepend_option.m4 index 3e868a6831..3b7030a473 100644 --- a/tool/m4/ruby_prepend_option.m4 +++ b/tool/m4/ruby_prepend_option.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_PREPEND_OPTION], [# RUBY_PREPEND_OPTION($1) AS_CASE([" [$]{$1-} "], diff --git a/tool/m4/ruby_prog_gnu_ld.m4 b/tool/m4/ruby_prog_gnu_ld.m4 index 8d484f9440..b38fb3d527 100644 --- a/tool/m4/ruby_prog_gnu_ld.m4 +++ b/tool/m4/ruby_prog_gnu_ld.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_PROG_GNU_LD], [AC_CACHE_CHECK(whether the linker is GNU ld, rb_cv_prog_gnu_ld, [AS_IF([`$CC $CFLAGS $CPPFLAGS $LDFLAGS --print-prog-name=ld 2>&1` -v 2>&1 | grep "GNU ld" > /dev/null], [ diff --git a/tool/m4/ruby_prog_makedirs.m4 b/tool/m4/ruby_prog_makedirs.m4 new file mode 100644 index 0000000000..132ff7768f --- /dev/null +++ b/tool/m4/ruby_prog_makedirs.m4 @@ -0,0 +1,9 @@ +dnl -*- Autoconf -*- +m4_defun([RUBY_PROG_MAKEDIRS], + [m4_bpatsubst(m4_defn([AC_PROG_MKDIR_P]), + [MKDIR_P=\"$ac_install_sh -d\"], [ + AS_IF([test "x$MKDIR_P" = "xfalse"], [AC_MSG_ERROR([mkdir -p is required])]) + MKDIR_P="mkdir -p"]) + ]dnl + AC_SUBST(MAKEDIRS, ["$MKDIR_P"]) +) diff --git a/tool/m4/ruby_replace_funcs.m4 b/tool/m4/ruby_replace_funcs.m4 index d0612e29a0..33b3ca70c1 100644 --- a/tool/m4/ruby_replace_funcs.m4 +++ b/tool/m4/ruby_replace_funcs.m4 @@ -1,13 +1,13 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- dnl RUBY_REPLACE_FUNC [func] [included] AC_DEFUN([RUBY_REPLACE_FUNC], [dnl AC_CHECK_DECL([$1],dnl [AC_DEFINE(AS_TR_CPP(HAVE_[$1]))],dnl [AC_REPLACE_FUNCS($1)],dnl [$2])dnl -]) - +])dnl +dnl dnl RUBY_REPLACE_FUNCS [funcs] [included] -AC_DEFUN([RUBY_REPLACE_FUNCS] [dnl - m4_map_args_w([$1], [RUBY_REPLACE_FUNC(], [), [$2]])dnl -]) +AC_DEFUN([RUBY_REPLACE_FUNCS], [dnl + m4_map_args_w([$1], [RUBY_REPLACE_FUNC(], [, [$2])])dnl +])dnl diff --git a/tool/m4/ruby_replace_type.m4 b/tool/m4/ruby_replace_type.m4 index a7d2ffe938..3df0f3994f 100644 --- a/tool/m4/ruby_replace_type.m4 +++ b/tool/m4/ruby_replace_type.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- dnl RUBY_REPLACE_TYPE [typename] [default type] [macro type] [included] AC_DEFUN([RUBY_REPLACE_TYPE], [dnl AC_CHECK_TYPES([$1], @@ -6,6 +6,13 @@ AC_DEFUN([RUBY_REPLACE_TYPE], [dnl [n="patsubst([$2],["],[\\"])"], [$4]) AC_CACHE_CHECK([for convertible type of [$1]], rb_cv_[$1]_convertible, [ + AC_COMPILE_IFELSE( + [AC_LANG_BOOL_COMPILE_TRY([AC_INCLUDES_DEFAULT([$4])] + [typedef $n rbcv_conftest_target_type; + extern rbcv_conftest_target_type rbcv_conftest_var; + ], [sizeof(&*rbcv_conftest_var)])], + [rb_cv_[$1]_convertible=PTR], + [ u= t= AS_CASE(["$n "], [*" signed "*], [ ], @@ -37,6 +44,7 @@ AC_DEFUN([RUBY_REPLACE_TYPE], [dnl [ t=INT]) rb_cv_[$1]_convertible=${u}${t}]) + ]) AS_IF([test "${AS_TR_SH(ac_cv_type_[$1])}" = "yes"], [ n="$1" ], [ @@ -48,11 +56,13 @@ AC_DEFUN([RUBY_REPLACE_TYPE], [dnl AS_CASE(["${rb_cv_[$1]_convertible}"], [U*], [n="unsigned $n"]) ]) - AS_CASE("${rb_cv_[$1]_convertible}", [U*], [u=+1], [u=-1]) + AS_CASE("${rb_cv_[$1]_convertible}", [PTR], [u=], [U*], [u=+1], [u=-1]) AC_DEFINE_UNQUOTED(rb_[$1], $n) + AS_IF([test $u], [ AC_DEFINE_UNQUOTED([SIGNEDNESS_OF_]AS_TR_CPP($1), $u) AC_DEFINE_UNQUOTED([$3]2NUM[(v)], [${rb_cv_[$1]_convertible}2NUM(v)]) AC_DEFINE_UNQUOTED(NUM2[$3][(v)], [NUM2${rb_cv_[$1]_convertible}(v)]) AC_DEFINE_UNQUOTED(PRI_[$3]_PREFIX, [PRI_`echo ${rb_cv_[$1]_convertible} | sed ['s/^U//']`_PREFIX]) + ]) ])dnl diff --git a/tool/m4/ruby_require_funcs.m4 b/tool/m4/ruby_require_funcs.m4 new file mode 100644 index 0000000000..78372d57d8 --- /dev/null +++ b/tool/m4/ruby_require_funcs.m4 @@ -0,0 +1,13 @@ +dnl -*- Autoconf -*- +dnl RUBY_REQUIRE_FUNC [func] [included] +AC_DEFUN([RUBY_REQUIRE_FUNC], [ +# RUBY_REQUIRE_FUNC([$1], [$2]) + AC_CHECK_FUNCS([$1]) + AS_IF([test "$ac_cv_func_[]AS_TR_SH($1)" = yes], [], + [AC_MSG_ERROR($1[() must be supported])]) +])dnl +dnl +dnl RUBY_REQUIRE_FUNCS [funcs] [included] +AC_DEFUN([RUBY_REQUIRE_FUNCS], [dnl + m4_map_args_w([$1], [RUBY_REQUIRE_FUNC(], [, [$2])])dnl +])dnl diff --git a/tool/m4/ruby_rm_recursive.m4 b/tool/m4/ruby_rm_recursive.m4 index e33ba1d97a..b97701f88e 100644 --- a/tool/m4/ruby_rm_recursive.m4 +++ b/tool/m4/ruby_rm_recursive.m4 @@ -1,6 +1,6 @@ -# -*- Autoconf -*- -AC_DEFUN([RUBY_RM_RECURSIVE], [ -m4_version_prereq([2.70], [-1], [ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_RM_RECURSIVE], [dnl +m4_version_prereq([2.70], [], [dnl # suppress error messages, rm: cannot remove 'conftest.dSYM', from # AC_EGREP_CPP with CFLAGS=-g on Darwin. AS_CASE([$build_os], [darwin*], [ diff --git a/tool/m4/ruby_setjmp_type.m4 b/tool/m4/ruby_setjmp_type.m4 index e054a544c5..d26af34ea0 100644 --- a/tool/m4/ruby_setjmp_type.m4 +++ b/tool/m4/ruby_setjmp_type.m4 @@ -1,20 +1,17 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_SETJMP_TYPE], [ RUBY_CHECK_BUILTIN_SETJMP RUBY_CHECK_SETJMP(_setjmpex, [], [@%:@include <setjmpex.h>]) RUBY_CHECK_SETJMP(_setjmp) -RUBY_CHECK_SETJMP(sigsetjmp, [sigjmp_buf]) AC_MSG_CHECKING(for setjmp type) setjmp_suffix= -unset setjmp_sigmask AC_ARG_WITH(setjmp-type, - AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), + AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), [ AS_CASE([$withval], [__builtin_setjmp], [setjmp=__builtin_setjmp], [_setjmp], [ setjmp_prefix=_], - [sigsetjmp,*], [ setjmp_prefix=sig setjmp_sigmask=`expr "$withval" : 'sigsetjmp\(,.*\)'`], - [sigsetjmp], [ setjmp_prefix=sig], + [sigsetjmp*], [ AC_MSG_WARN(No longer use sigsetjmp; use setjmp instead); setjmp_prefix=], [setjmp], [ setjmp_prefix=], [setjmpex], [ setjmp_prefix= setjmp_suffix=ex], [''], [ unset setjmp_prefix], @@ -34,19 +31,13 @@ AS_IF([test ${setjmp_prefix+set}], [ ], [test "$ac_cv_func__setjmp" = yes], [ setjmp_prefix=_ setjmp_suffix= -], [test "$ac_cv_func_sigsetjmp" = yes], [ - AS_CASE([$target_os],[solaris*|cygwin*],[setjmp_prefix=],[setjmp_prefix=sig]) - setjmp_suffix= ], [ setjmp_prefix= setjmp_suffix= ]) -AS_IF([test x$setjmp_prefix:$setjmp_sigmask = xsig:], [ - setjmp_sigmask=,0 -]) -AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}${setjmp_sigmask}) -AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env)${setjmp_sigmask})]) +AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}) +AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env))]) AC_DEFINE_UNQUOTED([RUBY_LONGJMP(env,val)], [${setjmp_prefix}longjmp($setjmp_cast(env),val)]) -AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, ${setjmp_sigmask+${setjmp_prefix}}jmp_buf) +AS_CASE(["$GCC:$setjmp_prefix"], [yes:__builtin_], [], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, jmp_buf)) AS_IF([test x$setjmp_suffix = xex], [AC_DEFINE_UNQUOTED(RUBY_USE_SETJMPEX, 1)]) ])dnl diff --git a/tool/m4/ruby_stack_grow_direction.m4 b/tool/m4/ruby_stack_grow_direction.m4 index 74ec219322..8c6fdd5722 100644 --- a/tool/m4/ruby_stack_grow_direction.m4 +++ b/tool/m4/ruby_stack_grow_direction.m4 @@ -1,12 +1,12 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_STACK_GROW_DIRECTION], [ AS_VAR_PUSHDEF([stack_grow_dir], [rb_cv_stack_grow_dir_$1]) AC_CACHE_CHECK(stack growing direction on $1, stack_grow_dir, [ AS_CASE(["$1"], -[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*], [ $2=-1], +[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*|arm*|aarch*], [ $2=-1], [hppa*], [ $2=+1], [ - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ /* recurse to get rid of inlining */ static int stack_growup_p(addr, n) @@ -23,7 +23,7 @@ int main() int x; return stack_growup_p(&x, 10); } -], $2=-1, $2=+1, $2=0) +]])],[$2=-1],[$2=+1],[$2=0]) ]) eval stack_grow_dir=\$$2]) eval $2=\$stack_grow_dir diff --git a/tool/m4/ruby_thread.m4 b/tool/m4/ruby_thread.m4 new file mode 100644 index 0000000000..439c63bc22 --- /dev/null +++ b/tool/m4/ruby_thread.m4 @@ -0,0 +1,80 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_THREAD], [ +AC_ARG_WITH(thread, + AS_HELP_STRING([--with-thread=IMPLEMENTATION], [specify the thread implementation to use]), + [THREAD_MODEL=$withval], [ + THREAD_MODEL= + AS_CASE(["$target_os"], + [freebsd*], [ + AC_CACHE_CHECK([whether pthread should be enabled by default], + rb_cv_enable_pthread_default, + [AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ +@%:@include <osreldate.h> +@%:@if __FreeBSD_version < 502102 +@%:@error pthread should be disabled on this platform +@%:@endif + ]])], + rb_cv_enable_pthread_default=yes, + rb_cv_enable_pthread_default=no)]) + AS_IF([test $rb_cv_enable_pthread_default = yes], + [THREAD_MODEL=pthread], + [THREAD_MODEL=none]) + ], + [mingw*], [ + THREAD_MODEL=win32 + ], + [wasi*], [ + THREAD_MODEL=none + ], + [ + THREAD_MODEL=pthread + ] + ) +]) + +AS_IF([test x"$THREAD_MODEL" = xpthread], [ + AC_CHECK_HEADERS(pthread.h) + AS_IF([test x"$ac_cv_header_pthread_h" = xyes], [], [ + AC_MSG_WARN("Don't know how to find pthread header on your system -- thread support disabled") + THREAD_MODEL=none + ]) +]) +AS_IF([test x"$THREAD_MODEL" = xpthread], [ + THREAD_MODEL=none + for pthread_lib in thr pthread pthreads c c_r root; do + AC_CHECK_LIB($pthread_lib, pthread_create, + [THREAD_MODEL=pthread; break]) + done + AS_IF([test x"$THREAD_MODEL" = xpthread], [ + AC_DEFINE(_REENTRANT) + AC_DEFINE(_THREAD_SAFE) + AC_DEFINE(HAVE_LIBPTHREAD) + AC_CHECK_HEADERS(pthread_np.h, [], [], [@%:@include <pthread.h>]) + AS_CASE(["$pthread_lib:$target_os"], + [c:*], [], + [root:*], [], + [c_r:*|*:openbsd*|*:mirbsd*], [LIBS="-pthread $LIBS"], + [LIBS="-l$pthread_lib $LIBS"]) + ], [ + AC_MSG_WARN("Don't know how to find pthread library on your system -- thread support disabled") + ]) +]) + +AS_CASE(["$THREAD_MODEL"], +[pthread], [], +[win32], [], +[none], [], +[""], [AC_MSG_ERROR(thread model is missing)], + [AC_MSG_ERROR(unknown thread model $THREAD_MODEL)]) +AC_MSG_CHECKING(thread model) +AC_MSG_RESULT($THREAD_MODEL) + +THREAD_IMPL_H=thread_$THREAD_MODEL.h +AS_IF([test ! -f "$srcdir/$THREAD_IMPL_H"], + [AC_MSG_ERROR('$srcdir/$THREAD_IMPL_H' must exist)]) +THREAD_IMPL_SRC=thread_$THREAD_MODEL.c +AS_IF([test ! -f "$srcdir/$THREAD_IMPL_SRC"], + [AC_MSG_ERROR('$srcdir/$THREAD_IMPL_SRC' must exist)]) +AC_DEFINE_UNQUOTED(THREAD_IMPL_H, ["$THREAD_IMPL_H"]) +AC_DEFINE_UNQUOTED(THREAD_IMPL_SRC, ["$THREAD_IMPL_SRC"]) +])dnl diff --git a/tool/m4/ruby_try_cflags.m4 b/tool/m4/ruby_try_cflags.m4 index 86ab80e1e6..b397642aad 100644 --- a/tool/m4/ruby_try_cflags.m4 +++ b/tool/m4/ruby_try_cflags.m4 @@ -1,12 +1,41 @@ -# -*- Autoconf -*- -AC_DEFUN([RUBY_TRY_CFLAGS], [ - AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS]) +dnl -*- Autoconf -*- +dnl +dnl Autoconf 2.67 fails to detect `-Werror=old-style-definition` due +dnl to the old style definition of `main`. +m4_version_prereq([2.70], [], [ +m4_defun([AC_LANG_PROGRAM(C)], m4_bpatsubst(m4_defn([AC_LANG_PROGRAM(C)]), [main ()], [main (void)])) +])dnl +dnl +AC_DEFUN([_RUBY_TRY_CFLAGS], [ RUBY_WERROR_FLAG([ CFLAGS="[$]CFLAGS $1" - AC_TRY_COMPILE([$4], [$5], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS([$1], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)], + [$4], [$5]) +])dnl + +AC_DEFUN([_RUBY_TRY_CFLAGS_PREPEND], [ + RUBY_WERROR_FLAG([ + CFLAGS="$1 [$]CFLAGS" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS_PREPEND], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS_PREPEND([$1], [$2 AC_MSG_RESULT(yes)], [$3 - AC_MSG_RESULT(no)]) - ]) + AC_MSG_RESULT(no)], + [$4], [$5]) ])dnl diff --git a/tool/m4/ruby_try_cxxflags.m4 b/tool/m4/ruby_try_cxxflags.m4 new file mode 100644 index 0000000000..06f645f546 --- /dev/null +++ b/tool/m4/ruby_try_cxxflags.m4 @@ -0,0 +1,17 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_TRY_CXXFLAGS], [ + save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="[$]CXXFLAGS $1" + AC_MSG_CHECKING([whether ]$1[ is accepted as CXXFLAGS]) + RUBY_WERROR_FLAG([ + AC_LANG_PUSH([C++]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)]) + ]) + AC_LANG_POP([C++]) + CXXFLAGS="$save_CXXFLAGS" + save_CXXFLAGS= +])dnl diff --git a/tool/m4/ruby_try_ldflags.m4 b/tool/m4/ruby_try_ldflags.m4 index b275107ed9..c3a6be0fb3 100644 --- a/tool/m4/ruby_try_ldflags.m4 +++ b/tool/m4/ruby_try_ldflags.m4 @@ -1,10 +1,10 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_TRY_LDFLAGS], [ save_LDFLAGS="$LDFLAGS" LDFLAGS="[$]LDFLAGS $1" AC_MSG_CHECKING([whether $1 is accepted as LDFLAGS]) RUBY_WERROR_FLAG([ - AC_TRY_LINK([$4], [$5], + AC_LINK_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], [$2 AC_MSG_RESULT(yes)], [$3 diff --git a/tool/m4/ruby_type_attribute.m4 b/tool/m4/ruby_type_attribute.m4 index 6ddaa20c53..5ea1219c6e 100644 --- a/tool/m4/ruby_type_attribute.m4 +++ b/tool/m4/ruby_type_attribute.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- dnl RUBY_TYPE_ATTRIBUTE(attrib, macroname, cachevar, condition) AC_DEFUN([RUBY_TYPE_ATTRIBUTE], [dnl RUBY_DECL_ATTRIBUTE([$1], [$2], [$3], [$4], diff --git a/tool/m4/ruby_universal_arch.m4 b/tool/m4/ruby_universal_arch.m4 index a524bc2592..d3e0dd0b47 100644 --- a/tool/m4/ruby_universal_arch.m4 +++ b/tool/m4/ruby_universal_arch.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_UNIVERSAL_ARCH], [ # RUBY_UNIVERSAL_ARCH begin ARCH_FLAG=`expr " $CXXFLAGS " : ['.* \(-m[0-9][0-9]*\) ']` @@ -17,7 +17,7 @@ AS_IF([test ${target_archs+set}], [ cpu=$archs cpu=`echo $cpu | sed 's/-.*-.*//'` universal_binary="${universal_binary+$universal_binary,}$cpu" - universal_archnames="${universal_archnames} ${archs}=${cpu}" + universal_archnames="${universal_archnames:+$universal_archnames }${archs}=${cpu}" ARCH_FLAG="${ARCH_FLAG+$ARCH_FLAG }-arch $archs" ]) done @@ -39,8 +39,8 @@ AS_IF([test ${target_archs+set}], [ echo 'int main(){return 0;}' > conftest.c AS_IF([$CC $CFLAGS $ARCH_FLAG -o conftest conftest.c > /dev/null 2>&1], [ rm -fr conftest.* - ], [ - RUBY_DEFAULT_ARCH("$target_archs") + ], [test -z "$ARCH_FLAG"], [ + RUBY_DEFAULT_ARCH($target_archs) ]) ]) target_cpu=${target_archs} @@ -66,11 +66,14 @@ AS_IF([test ${target_archs+set}], [ #ifdef __ppc64__ "processor-name=powerpc64" #endif +#ifdef __arm64__ +"processor-name=arm64" +#endif EOF sed -n 's/^"processor-name=\(.*\)"/\1/p'` target="$target_cpu${target}" AC_MSG_RESULT([$target_cpu]) - ]) + ]) ]) target_archs="$target_cpu" ]) @@ -85,3 +88,35 @@ AS_IF([test "x${ARCH_FLAG}" != x], [ ]) # RUBY_UNIVERSAL_ARCH end ])dnl +dnl +AC_DEFUN([RUBY_UNIVERSAL_CHECK_HEADER_COND], [ dnl + AC_CACHE_CHECK([for $2 when $1], [$3], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [AC_INCLUDES_DEFAULT([$6])[ + @%:@if ]$1[ + @%:@include <]$2[> + @%:@endif]], [[]])], + [AS_VAR_SET($3, yes)], + [AS_VAR_SET($3, no)])) + AS_VAR_IF([$3], [yes], [dnl + printf "@%:@if %s\n" "$1" >>confdefs.h + AC_DEFINE_UNQUOTED(HAVE_[]AS_TR_CPP($2), 1)dnl + printf "@%:@endif\n" >>confdefs.h dnl + $4], [$5]) +])dnl +dnl +dnl RUBY_UNIVERSAL_CHECK_HEADER(CPU-LIST, HEADER, +dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND], +dnl [INCLUDES = DEFAULT-INCLUDES]) +AC_DEFUN([RUBY_UNIVERSAL_CHECK_HEADER], [ dnl + m4_if([$# dnl + ], [0], [], [ dnl + m4_foreach([rb_Header], [$1], + [AS_CASE([",$target_archs,"], [*,]rb_Header[,*], + [RUBY_UNIVERSAL_CHECK_HEADER_COND]([defined(__[]rb_Header[]__)], + [$2], [rb_cv_header_[]AS_TR_SH($2)_on_[]AS_TR_SH(rb_Header)], + [$3], [$4], [$5]) + ) + ]) + ])dnl +])dnl diff --git a/tool/m4/ruby_wasm_tools.m4 b/tool/m4/ruby_wasm_tools.m4 new file mode 100644 index 0000000000..efc017e771 --- /dev/null +++ b/tool/m4/ruby_wasm_tools.m4 @@ -0,0 +1,25 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_WASM_TOOLS], +[AS_CASE(["$target_os"], +[wasi*], [ + AC_CHECK_TOOL(WASMOPT, wasm-opt) + AS_IF([test x"${WASMOPT}" = x], [ + AC_MSG_ERROR([wasm-opt is required]) + ]) + AC_SUBST(wasmoptflags) + : ${wasmoptflags=-O3} + + AC_MSG_CHECKING([whether \$WASI_SDK_PATH is set]) + AS_IF([test x"${WASI_SDK_PATH}" = x], [ + AC_MSG_RESULT([no]) + AC_MSG_ERROR([WASI_SDK_PATH environment variable is required]) + ], [ + AC_MSG_RESULT([yes]) + CC="${CC:-${WASI_SDK_PATH}/bin/clang}" + LD="${LD:-${WASI_SDK_PATH}/bin/clang}" + AR="${AR:-${WASI_SDK_PATH}/bin/llvm-ar}" + RANLIB="${RANLIB:-${WASI_SDK_PATH}/bin/llvm-ranlib}" + OBJCOPY="${OBJCOPY:-${WASI_SDK_PATH}/bin/llvm-objcopy}" + ]) +]) +])dnl diff --git a/tool/m4/ruby_werror_flag.m4 b/tool/m4/ruby_werror_flag.m4 index e9155c8309..616a7f6abf 100644 --- a/tool/m4/ruby_werror_flag.m4 +++ b/tool/m4/ruby_werror_flag.m4 @@ -1,4 +1,4 @@ -# -*- Autoconf -*- +dnl -*- Autoconf -*- AC_DEFUN([RUBY_WERROR_FLAG], [dnl save_CFLAGS="$CFLAGS" CFLAGS="$CFLAGS $rb_cv_warnflags" diff --git a/tool/make-snapshot b/tool/make-snapshot index ad5fd879f3..4af6a855eb 100755 --- a/tool/make-snapshot +++ b/tool/make-snapshot @@ -1,5 +1,8 @@ #!/usr/bin/ruby -s # -*- coding: us-ascii -*- +require 'rubygems' +require 'rubygems/package' +require 'rubygems/package/tar_writer' require 'uri' require 'digest/sha1' require 'digest/sha2' @@ -7,18 +10,20 @@ require 'fileutils' require 'shellwords' require 'tmpdir' require 'pathname' +require 'date' +require 'yaml' +require 'json' require File.expand_path("../lib/vcs", __FILE__) require File.expand_path("../lib/colorize", __FILE__) STDOUT.sync = true $srcdir ||= nil -$exported = nil if ($exported ||= nil) == "" $archname = nil if ($archname ||= nil) == "" $keep_temp ||= nil $patch_file ||= nil $packages ||= nil $digests ||= nil -$s3 ||= nil +$no7z ||= nil $tooldir = File.expand_path("..", __FILE__) $unicode_version = nil if ($unicode_version ||= nil) == "" $colorize = Colorize.new @@ -28,18 +33,12 @@ def usage usage: #{File.basename $0} [option...] new-directory-to-save [version ...] options: -srcdir=PATH source directory path - -exported=PATH make snapshot from already exported working directory -archname=NAME make the basename of snapshots NAME -keep_temp keep temporary working directory -patch_file=PATCH apply PATCH file after export -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")}) -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")}) -unicode_version=VER Unicode version to generate encodings - -svn[=URL] make snapshot from SVN repository - (#{SVNURL}) - -git[=URL] make snapshot from GIT repository - (#{GITURL}) - -s3=PATH s3 bucket path(default=tmp) -help, --help show this message version: master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL @@ -49,17 +48,17 @@ end DIGESTS = %w[SHA1 SHA256 SHA512] PACKAGES = { + "tar" => %w".tar", "bzip" => %w".tar.bz2 bzip2 -c", "gzip" => %w".tar.gz gzip -c", "xz" => %w".tar.xz xz -c", - "zip" => %w".zip zip -qr", + "zip" => %w".zip zip -Xqr", } - -if system("7z", out: IO::NULL) +DEFAULT_PACKAGES = PACKAGES.keys - ["tar"] +if !$no7z and system("7z", out: IO::NULL) PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" - PACKAGES["zip"] = %w".zip 7z a -tzip -mx" << {out: IO::NULL} -end -if gzip = ENV.delete("GZIP") + PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL} +elsif gzip = ENV.delete("GZIP") PACKAGES["gzip"].concat(gzip.shellsplit) end @@ -69,21 +68,11 @@ if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?) end ENV["LC_ALL"] = ENV["LANG"] = "C" -SVNURL = URI.parse("https://svn.ruby-lang.org/repos/ruby/") # https git clone is disabled at git.ruby-lang.org/ruby.git. GITURL = URI.parse("https://github.com/ruby/ruby.git") RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ -ENV["VPATH"] ||= "include/ruby" -YACC = ENV["YACC"] ||= "bison" -ENV["BASERUBY"] ||= "ruby" -ENV["RUBY"] ||= "ruby" -ENV["MV"] ||= "mv" -ENV["RM"] ||= "rm -f" -ENV["MINIRUBY"] ||= "ruby" -ENV["PROGRAM"] ||= "ruby" -ENV["AUTOCONF"] ||= "autoconf" -ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" +ENV["TZ"] = "UTC" class String # for older ruby @@ -105,12 +94,12 @@ end $packages &&= $packages.split(/[, ]+/).tap {|pkg| if all = pkg.index("all") - pkg[all, 1] = PACKAGES.keys - pkg + pkg[all, 1] = DEFAULT_PACKAGES - pkg end pkg -= PACKAGES.keys pkg.empty? or abort "#{File.basename $0}: unknown packages - #{pkg.join(", ")}" } -$packages ||= PACKAGES.keys +$packages ||= DEFAULT_PACKAGES $digests &&= $digests.split(/[, ]+/).tap {|dig| dig -= DIGESTS @@ -119,22 +108,32 @@ $digests &&= $digests.split(/[, ]+/).tap {|dig| $digests ||= DIGESTS $patch_file &&= File.expand_path($patch_file) -path = ENV["PATH"].split(File::PATH_SEPARATOR) -%w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| - cmd, = ENV[var].shellsplit - unless path.any? {|dir| +PATH = ENV["PATH"].split(File::PATH_SEPARATOR) +def PATH.executable_env(var, command = nil) + command = ENV[var] ||= (command or return) + cmd, = command.shellsplit + unless any? {|dir| file = File.expand_path(cmd, dir) File.file?(file) and File.executable?(file) } abort "#{File.basename $0}: #{var} command not found - #{cmd}" end + command end +PATH.executable_env("MV", "mv") +PATH.executable_env("RM", "rm -f") +PATH.executable_env("AUTOCONF", "autoconf") + %w[BASERUBY RUBY MINIRUBY].each do |var| - %x[#{ENV[var]} --disable-gem -e1 2>&1] - if $?.success? - ENV[var] += ' --disable-gem' + cmd = PATH.executable_env(var, "ruby") + help = IO.popen("#{cmd} --help", err: %i[child out], &:read) + unless $?.success? and /ruby/ =~ help + abort "#{File.basename $0}: #{var} ruby not found - #{cmd}" end + IO.popen("#{cmd} --disable-gem -eexit", err: %i[child out], &:read) + cmd += ' --disable-gem' if $?.success? + ENV[var] = cmd end if defined?($help) or defined?($_help) @@ -145,27 +144,30 @@ unless destdir = ARGV.shift abort usage end revisions = ARGV.empty? ? [nil] : ARGV -unless tmp = $exported - FileUtils.mkpath(destdir) - destdir = File.expand_path(destdir) - tmp = Dir.mktmpdir("ruby-snapshot") - FileUtils.mkpath(tmp) - at_exit { - Dir.chdir "/" - FileUtils.rm_rf(tmp) - } unless $keep_temp + +if defined?($exported) + abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead" end +FileUtils.mkpath(destdir) +destdir = File.expand_path(destdir) +tmp = Dir.mktmpdir("ruby-snapshot") +FileUtils.mkpath(tmp) +at_exit { + Dir.chdir "/" + FileUtils.rm_rf(tmp) +} unless $keep_temp + def tar_create(tarball, dir) - require 'rubygems' - require 'rubygems/package' - require 'rubygems/package/tar_writer' header = Gem::Package::TarHeader dir_type = "5" uname = gname = "ruby" File.open(tarball, "wb") do |f| w = Gem::Package::TarWriter.new(f) - Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH) do |path| + list = Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH) + list.reject! {|name| name.end_with?("/.")} + list.sort_by! {|name| name.split("/")} + list.each do |path| next if File.basename(path) == "." s = File.stat(path) mode = 0644 @@ -199,10 +201,60 @@ rescue => e false end +def touch_all(time, pattern, opt, &cond) + Dir.glob(pattern, opt) do |n| + stat = File.stat(n) + if stat.file? or stat.directory? + next if cond and !yield(n, stat) + File.utime(time, time, n) + end + end +rescue + false +else + true +end + +class MAKE < Struct.new(:prog, :args) + def initialize(vars) + vars = vars.map {|arg| arg.join("=")} + super(ENV["MAKE"] || ENV["make"] || "make", vars) + end + + def run(target) + err = IO.pipe do |r, w| + begin + pid = Process.spawn(self.prog, *self.args, target, {:err => w, r => :close}) + w.close + r.read + ensure + Process.wait(pid) + end + end + if $?.success? + true + else + STDERR.puts err + $colorize.fail("#{target} failed") + false + end + end +end + +def measure + clock = Process::CLOCK_MONOTONIC + t0 = Process.clock_gettime(clock) + STDOUT.flush + result = yield + printf(" %6.3f", Process.clock_gettime(clock) - t0) + STDOUT.flush + result +end + def package(vcs, rev, destdir, tmp = nil) - patchlevel = false + pwd = Dir.pwd prerelease = false - if rev and revision = rev[/@(\d+)\z/, 1] + if rev and revision = rev[/@(\h+)\z/, 1] rev = $` end case rev @@ -217,29 +269,36 @@ def package(vcs, rev, destdir, tmp = nil) when /\Astable\z/ vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} url &&= vcs.branch(url) - when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + when /\A(\d+)\.(\d+)\.(\d+)-(preview|rc)(\d+)/ prerelease = true tag = "#{$4}#{$5}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") - when /\A(.*)\.(.*)\.(.*)-p(\d+)/ - patchlevel = true - tag = "p#{$4}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") - when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ - if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") - patchlevel = true - tag = "" - url = vcs.tag("v#{$1}_#{$2}_#{$3}") + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") + else + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") + end + when /\A(\d+)\.(\d+)\.(\d+)\z/ + tag = "" + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") else - url = vcs.branch("ruby_#{rev.tr('.', '_')}") + url = vcs.tag("v#{$1}_#{$2}_#{$3}") end + when /\A(\d+)\.(\d+)\z/ + url = vcs.branch("ruby_#{rev.tr('.', '_')}") else warn "#{$0}: unknown version - #{rev}" return end - if !revision and revision = vcs.get_revisions(url) + if info = vcs.get_revisions(url) + modified = info[2] + else + _, _, modified = VCS::Null.new(nil).get_revisions(url) + end + if !revision and info + revision = info url ||= vcs.branch(revision[3]) - revision = revision[0] + revision = revision[1] end version = nil unless revision @@ -249,41 +308,34 @@ def package(vcs, rev, destdir, tmp = nil) warn "#{$0}: #{rev} not found" return end - revision = vcs.get_revisions(url)[0] + revision = vcs.get_revisions(url)[1] end - v = nil - if $exported - if String === $exported - v = $exported - end - else - v = "ruby" - puts "Exporting #{rev}@#{revision}" - exported = tmp ? File.join(tmp, v) : v - unless vcs.export(revision, url, exported, true) {|line| print line} - warn("Export failed") - return - end - if $srcdir - Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file| - puts "copying #{file}" - dest = exported + file[$srcdir.size..-1] - FileUtils.mkpath(File.dirname(dest)) - begin - FileUtils.ln(file, dest, force: true) - next unless File.symlink?(dest) - File.unlink(dest) - rescue SystemCallError - end - begin - FileUtils.cp_r(file, dest, preserve: true) - rescue SystemCallError - end + + v = "ruby" + puts "Exporting #{rev}@#{revision}" + exported = tmp ? File.join(tmp, v) : v + unless vcs.export(revision, url, exported, true) {|line| print line} + warn("Export failed") + return + end + if $srcdir + Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file| + puts "copying #{file}" if $VERBOSE + dest = exported + file[$srcdir.size..-1] + FileUtils.mkpath(File.dirname(dest)) + begin + FileUtils.cp_r(file, dest) + FileUtils.chmod_R("a+rwX,go-w", dest) + rescue SystemCallError end end end - status = IO.read(File.dirname(__FILE__) + "/prereq.status") + Dir.glob("#{exported}/.*.yml") do |file| + FileUtils.rm(file, verbose: $VERBOSE) + end + + status = File.read(File.dirname(__FILE__) + "/prereq.status") Dir.chdir(tmp) if tmp if !File.directory?(v) @@ -292,68 +344,77 @@ def package(vcs, rev, destdir, tmp = nil) v = v[0] end - open("#{v}/revision.h", "wb") {|f| - short = vcs.short_revision(revision) - f.puts "#define RUBY_REVISION #{short.inspect}" - unless short == revision - f.puts "#define RUBY_FULL_REVISION #{revision.inspect}" - end + File.open("#{v}/revision.h", "wb") {|f| + f.puts vcs.revision_header(revision, modified) } - version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version ||= (versionhdr = File.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] version ||= begin - include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h") + include_ruby_versionhdr = File.read("#{v}/include/ruby/version.h") api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] [api_major_version, api_minor_version, version_teeny].join('.') end version or return - if patchlevel - unless tag.empty? - versionhdr ||= IO.read("#{v}/version.h") - patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] - tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision)) - end - elsif prerelease - versionhdr ||= IO.read("#{v}/version.h") - versionhdr.sub!(/^\#define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) - IO.write("#{v}/version.h", versionhdr) + if prerelease + versionhdr ||= File.read("#{v}/version.h") + versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace" + File.write("#{v}/version.h", versionhdr) else tag ||= vcs.revision_name(revision) end - unless v == $exported - if $archname - n = $archname - elsif tag.empty? - n = "ruby-#{version}" - else - n = "ruby-#{version}-#{tag}" - end - File.directory?(n) or File.rename v, n - v = n + + if $archname + n = $archname + elsif tag.empty? + n = "ruby-#{version}" + else + n = "ruby-#{version}-#{tag}" end - system(*%W"patch -d #{v} -p0 -i #{$patch_file}") if $patch_file - if !$exported or $patch_file - colors = %w[red yellow green cyan blue magenta] - "take a breath, and go ahead".scan(/./) do |c| - if c == ' ' - print c + File.directory?(n) or File.rename v, n + v = n + + if $patch_file && !system(*%W"patch -d #{v} -p0 -i #{$patch_file}") + puts $colorize.fail("patching failed") + return + end + + class << (clean = []) + def add(n) push(n) + n + end + def create(file, content = "", &block) + add(file) + if block + File.open(file, "wb", &block) else - colors.push(color = colors.shift) - print $colorize.decorate(c, color) + File.binwrite(file, content) end - sleep(c == "," ? 0.7 : 0.05) end - puts end - def (clean = []).add(n) push(n); n end + Dir.chdir(v) do unless File.exist?("ChangeLog") vcs.export_changelog(url, nil, revision, "ChangeLog") end - File.open(clean.add("cross.rb"), "w") do |f| + unless touch_all(modified, "**/*", File::FNM_DOTMATCH) + modified = nil + colors = %w[red yellow green cyan blue magenta] + "take a breath, and go ahead".scan(/./) do |c| + if c == ' ' + print c + else + colors.push(color = colors.shift) + print $colorize.decorate(c, color) + end + sleep(c == "," ? 0.7 : 0.05) + end + puts + end + + clean.create("cross.rb") do |f| f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)" f.puts "CROSS_COMPILING=true" f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)" @@ -361,9 +422,10 @@ def package(vcs, rev, destdir, tmp = nil) f.puts "Object.__send__(:remove_const, :RUBY_VERSION)" f.puts "RUBY_VERSION='#{version}'" end + puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE unless File.exist?("configure") print "creating configure..." - unless system([ENV["AUTOCONF"]]*2) + unless system(File.exist?(gen = "./autogen.sh") ? gen : [ENV["AUTOCONF"]]*2) puts $colorize.fail(" failed") return end @@ -372,25 +434,24 @@ def package(vcs, rev, destdir, tmp = nil) clean.add("autom4te.cache") clean.add("enc/unicode/data") print "creating prerequisites..." - if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + if File.file?("common.mk") && /^prereq/ =~ commonmk = File.read("common.mk") puts extout = clean.add('tmp') begin - status = IO.read("tool/prereq.status") + status = File.read("tool/prereq.status") rescue Errno::ENOENT # use fallback file end - File.open(clean.add("config.status"), "w") {|f| - f.print status - } + clean.create("config.status", status) + clean.create("noarch-fake.rb", "require_relative 'cross'\n") FileUtils.mkpath(hdrdir = "#{extout}/include/ruby") - File.open("#{hdrdir}/config.h", "w") {} + File.binwrite("#{hdrdir}/config.h", "") FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults") - File.open("#{defaults}/operating_system.rb", "w") {} - File.open("#{defaults}/ruby.rb", "w") {} + File.binwrite("#{defaults}/operating_system.rb", "") + File.binwrite("#{defaults}/ruby.rb", "") miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] - mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). + mk = (File.read("template/Makefile.in") rescue File.read("Makefile.in")). gsub(/^@.*\n/, '') vars = { "EXTOUT"=>extout, @@ -403,6 +464,9 @@ def package(vcs, rev, destdir, tmp = nil) "MAJOR"=>api_major_version, "MINOR"=>api_minor_version, "TEENY"=>version_teeny, + "VPATH"=>(ENV["VPATH"] || "include/ruby"), + "PROGRAM"=>(ENV["PROGRAM"] || "ruby"), + "BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"), } status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do vars[$2] ||= $3 @@ -411,41 +475,72 @@ def package(vcs, rev, destdir, tmp = nil) vars["UNICODE_VERSION"] = $unicode_version if $unicode_version args = vars.dup mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} - mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "") + commonmk.gsub!(/^!(?:include \$\(srcdir\)\/(.*))?/) do + if inc = $1 and File.exist?(inc) + File.binread(inc).gsub(/^!/, '# !') + else + "#" + end + end + mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') mk << <<-'APPEND' -prepare-package: touch-unicode-files -prepare-package: prereq update-download clean-cache $(CLEAN_CACHE) -clean-cache $(CLEAN_CACHE): after-update -update-download:: update-gems +update-download:: touch-unicode-files +prepare-package: prereq after-update +clean-cache: $(CLEAN_CACHE) after-update:: extract-gems extract-gems: update-gems update-gems: $(UNICODE_SRC_DATA_DIR)/.unicode-tables.time: touch-unicode-files: APPEND - open(clean.add("Makefile"), "w") do |f| - f.puts mk - end + clean.create("Makefile", mk) + clean.create("revision.tmp") + clean.create(".revision.time") ENV["CACHE_SAVE"] = "no" - system(ENV["MAKE"] || ENV["make"] || "make", "prepare-package", *args.map {|arg| arg.join("=")}) - clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk") + make = MAKE.new(args) + return unless make.run("update-download") + clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk", "ext/ripper/y.output", ".revision.time") + Dir.glob("**/*") do |dest| + next unless File.symlink?(dest) + orig = File.expand_path(File.readlink(dest), File.dirname(dest)) + File.unlink(dest) + FileUtils.cp_r(orig, dest) + end + File.utime(modified, modified, *Dir.glob(["tool/config.{guess,sub}", "gems/*.gem", "tool"])) + return unless make.run("prepare-package") + return unless make.run("clean-cache") + if modified + new_time = modified + 2 + touch_all(new_time, "**/*", File::FNM_DOTMATCH) do |name, stat| + stat.mtime > modified unless clean.include?(name) + end + modified = new_time + end print "prerequisites" else - system(*%W"#{YACC} -o parse.c parse.y") + system(*%W[#{PATH.executable_env("YACC", "bison")} -o parse.c parse.y]) end vcs.after_export(".") if exported - FileUtils.rm_rf(clean) unless $keep_temp - FileUtils.rm_rf(".downloaded-cache") + clean.concat(Dir.glob("ext/**/autom4te.cache")) + clean.add(".downloaded-cache") if File.exist?("gems/bundled_gems") gems = Dir.glob("gems/*.gem") gems -= File.readlines("gems/bundled_gems").map {|line| + next if /^\s*(?:#|$)/ =~ line name, version, _ = line.split(' ') "gems/#{name}-#{version}.gem" } - FileUtils.rm_f(gems) + clean.concat(gems) else - FileUtils.rm_rf("gems") + clean.add("gems") + end + FileUtils.rm_rf(clean) + if modified + touch_all(modified, "**/*/", 0) do |name, stat| + stat.mtime > modified + end + File.utime(modified, modified, ".") end unless $?.success? puts $colorize.fail(" failed") @@ -472,10 +567,12 @@ touch-unicode-files: if tarball next if tarball.empty? else - tarball = "#{$archname||v}.tar" + tarball = ext == ".tar" ? file : "#{$archname||v}.tar" print "creating tarball... #{tarball}" - if tar_create(tarball, v) + if measure {tar_create(tarball, v)} puts $colorize.pass(" done") + File.utime(modified, modified, tarball) if modified + next if tarball == file else puts $colorize.fail(" failed") tarball = "" @@ -483,7 +580,7 @@ touch-unicode-files: end end print "creating #{mesg} tarball... #{file}" - done = system(*cmd, tarball, out: file) + done = measure {system(*cmd, tarball, out: file)} else print "creating #{mesg} archive... #{file}" if Hash === cmd.last @@ -492,7 +589,7 @@ touch-unicode-files: else (cmd = cmd.dup) << file << v end - done = system(*cmd) + done = measure {system(*cmd)} end if done puts $colorize.pass(" done") @@ -503,41 +600,70 @@ touch-unicode-files: end end.compact ensure - FileUtils.rm_rf(v) if v and !$exported and !$keep_temp + FileUtils.rm_rf(tmp ? File.join(tmp, v) : v) if v and !$keep_temp + Dir.chdir(pwd) end -if [$srcdir, ($svn||=nil), ($git||=nil)].compact.size > 1 - abort "#{File.basename $0}: -srcdir, -svn, and -git are exclusive" +if [$srcdir, ($git||=nil)].compact.size > 1 + abort "#{File.basename $0}: -srcdir and -git are exclusive" end if $srcdir vcs = VCS.detect($srcdir) -elsif $svn - vcs = VCS::SVN.new($svn == true ? SVNURL : URI.parse($svn)) elsif $git - vcs = VCS::GIT.new($git == true ? GITURL : URI.parse($git)) + abort "#{File.basename $0}: use -srcdir with cloned local repository" else - vcs = VCS::SVN.new(SVNURL) + begin + vcs = VCS.detect(File.expand_path("../..", __FILE__)) + rescue VCS::NotFoundError + abort "#{File.expand_path("../..", __FILE__)}: cannot find git repository" + end end +release_date = Time.now.getutc +info = {} + success = true revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| if !name success = false next end - str = open(name, "rb") {|f| f.read} - if $s3 - %x[aws s3 cp #{name} s3://ftp.r-l.o/pub/#{$s3}/#{Pathname(name).basename}] - FileUtils.rm_f name - name = "https://s3.amazonaws.com/ftp.r-l.o/pub/#{$s3}/#{Pathname(name).basename}" + str = File.binread(name) + pathname = Pathname(name) + basename = pathname.basename.to_s + extname = pathname.extname.sub(/\A\./, '') + version = basename[/\Aruby-(.*)\.(?:tar|zip)/, 1] + key = basename[/\A(.*)\.(?:tar|zip)/, 1] + info[key] ||= Hash.new{|h,k|h[k]={}} + info[key]['version'] = version if version + info[key]['date'] = release_date.to_date + if version + info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" + info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" + else + info[key]['filename'][extname] = basename end + info[key]['size'][extname] = str.bytesize puts "* #{$colorize.pass(name)}" puts " SIZE: #{str.bytesize} bytes" $digests.each do |alg| - printf " %-8s%s\n", "#{alg}:", Digest.const_get(alg).hexdigest(str) + digest = Digest(alg).hexdigest(str) + info[key][alg.downcase][extname] = digest + printf " %-8s%s\n", "#{alg}:", digest end end +yaml = info.values.to_yaml +json = info.values.to_json +puts "#{$colorize.pass('YAML:')}" +puts yaml +puts "#{$colorize.pass('JSON:')}" +puts json +infodir = Pathname(destdir) + 'info' +infodir.mkpath +(infodir+'info.yml').write(yaml) +(infodir+'info.json').write(json) + exit false if !success # vim:fileencoding=US-ASCII sw=2 ts=4 noexpandtab ff=unix diff --git a/tool/make_hgraph.rb b/tool/make_hgraph.rb index 0f388814dd..174fa5dd2f 100644 --- a/tool/make_hgraph.rb +++ b/tool/make_hgraph.rb @@ -83,13 +83,12 @@ module ObjectSpace def self.module_refenreces_image klass, file dot = module_refenreces_dot(klass) - img = nil - IO.popen("dot -Tpng", 'r+'){|io| + img = IO.popen(%W"dot -Tpng", 'r+b') {|io| # io.puts dot io.close_write - img = io.read + io.read } - open(File.expand_path(file), 'w+'){|f| f.puts img} + File.binwrite(file, img) end end diff --git a/tool/merger.rb b/tool/merger.rb index f5ed9851c4..4c096087fc 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -2,9 +2,8 @@ # -*- ruby -*- exec "${RUBY-ruby}" "-x" "$0" "$@" && [ ] if false #!ruby -# This needs ruby 2.0, Subversion and Git. -# As a Ruby committer, run this in an SVN repository -# to commit a change. +# This needs ruby 2.0 and Git. +# As a Ruby committer, run this in a git repository to commit a change. require 'tempfile' require 'net/http' @@ -12,19 +11,14 @@ require 'uri' require 'shellwords' ENV['LC_ALL'] = 'C' +ORIGIN = 'git@git.ruby-lang.org:ruby.git' +GITHUB = 'git@github.com:ruby/ruby.git' -module Merger - REPOS = 'svn+ssh://svn@ci.ruby-lang.org/ruby/' -end - -class << Merger +class << Merger = Object.new def help puts <<-HELP \e[1msimple backport\e[0m - ruby #$0 1234 - -\e[1mbackport from other branch\e[0m - ruby #$0 17502 mvm + ruby #$0 1234abc \e[1mrevision increment\e[0m ruby #$0 revisionup @@ -33,16 +27,16 @@ class << Merger ruby #$0 teenyup \e[1mtagging major release\e[0m - ruby #$0 tag 2.2.0 + ruby #$0 tag 3.2.0 -\e[1mtagging patch release\e[0m (about 2.1.0 or later, it means X.Y.Z (Z > 0) release) +\e[1mtagging patch release\e[0m (for 2.1.0 or later, it means X.Y.Z (Z > 0) release) ruby #$0 tag \e[1mtagging preview/RC\e[0m - ruby #$0 tag 2.2.0-preview1 + ruby #$0 tag 3.2.0-preview1 \e[1mremove tag\e[0m - ruby #$0 removetag 2.2.9 + ruby #$0 removetag 3.2.9 \e[33;1m* all operations shall be applied to the working directory.\e[0m HELP @@ -53,11 +47,11 @@ class << Merger yield if block_given? STDERR.puts "\e[1;33m#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})\e[0m" case STDIN.gets - when /\Aa/i then exit + when /\Aa/i then exit 1 when /\Ar/i then redo when /\Ay/i then break when /\Ae/i then system(ENV['EDITOR'], editfile) - else exit + else exit 1 end end end @@ -65,17 +59,14 @@ class << Merger def version_up(teeny: false) now = Time.now now = now.localtime(9*60*60) # server is Japan Standard Time +09:00 - if svn_mode? - system('svn', 'revert', 'version.h') - else - system('git', 'checkout', 'HEAD', 'version.h') - end + system('git', 'checkout', 'HEAD', 'version.h') v, pl = version if teeny v[2].succ! end - if pl != '-1' # trunk does not have patchlevel + # We stopped bumping RUBY_PATCHLEVEL at Ruby 4.0.0. + if Integer(v[0]) <= 3 pl.succ! end @@ -112,7 +103,7 @@ class << Merger v, pl = version if relname abort "patchlevel is not -1 but '#{pl}' for preview or rc" if pl != '-1' && /-(?:preview|rc)/ =~ relname - abort "patchlevel is not 0 but '#{pl}' for the first release" if pl != '0' && /-(?:preview|rc)/ !~ relname + abort "patchlevel is not 0 but '#{pl}' for the first release" if pl != '0' && relname.end_with?(".0") pl = relname[/-(.*)\z/, 1] curver = "#{v.join('.')}#{("-#{pl}" if pl)}" if relname != curver @@ -123,32 +114,21 @@ class << Merger abort 'no relname is given and not in a release branch even if this is patch release' end end - tagname = "v#{v.join('_')}#{("_#{pl}" if v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl)}" - - if svn_mode? - if relname - branch_url = `svn info`[/URL: (.*)/, 1] - else - branch_url = "#{REPOS}branches/ruby_" - if v[0] < '2' || (v[0] == '2' && v[1] < '1') - abort 'patchlevel must be greater than 0 for patch release' if pl == '0' - branch_url << v.join('_') - else - abort 'teeny must be greater than 0 for patch release' if v[2] == '0' - branch_url << v.join('_').sub(/_\d+\z/, '') - end - end - tag_url = "#{REPOS}tags/#{tagname}" - unless system('svn', 'info', tag_url, out: IO::NULL, err: IO::NULL) - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('svn', 'cp', '-m', "add tag #{tagname}", branch_url, tag_url, interactive: true) + if /^(?:preview|rc)/ =~ pl + tagname = "v#{v.join('.')}-#{pl}" + elsif Integer(v[0]) >= 4 + tagname = "v#{v.join('.')}" else - unless execute('git', 'tag', tagname) - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('git', 'push', 'origin', tagname, interactive: true) + tagname = "v#{v.join('_')}" end + + unless execute('git', 'diff', '--exit-code') + abort 'uncommitted changes' + end + unless execute('git', 'tag', tagname) + abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' + end + execute('git', 'push', ORIGIN, tagname, interactive: true) end def remove_tag(relname) @@ -162,69 +142,52 @@ class << Merger unless relname raise ArgumentError, 'relname is not specified' end - if /^v/ !~ relname - tagname = "v#{relname.gsub(/[.-]/, '_')}" - else + if relname.start_with?('v') tagname = relname - end - - if svn_mode? - tag_url = "#{REPOS}tags/#{tagname}" - execute('svn', 'rm', '-m', "remove tag #{tagname}", tag_url, interactive: true) + elsif Integer(relname.split('.', 2).first) >= 4 + tagname = "v#{relname}" else - execute('git', 'tag', '-d', tagname) - execute('git', 'push', 'origin', ":#{tagname}", interactive: true) + tagname = "v#{relname.gsub(/[.-]/, '_')}" end + + execute('git', 'tag', '-d', tagname) + execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) + execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) end def update_revision_h - if svn_mode? - execute('svn', 'up') - end execute('ruby tool/file2lastrev.rb --revision.h . > revision.tmp') execute('tool/ifchange', '--timestamp=.revision.time', 'revision.h', 'revision.tmp') execute('rm', '-f', 'revision.tmp') end def stat - if svn_mode? - `svn stat` - else - `git status --short` - end + `git status --short` end def diff(file = nil) - if svn_mode? - command = %w[svn diff --diff-cmd=diff -x -upw] - else - command = %w[git diff --color] - end + command = %w[git diff --color HEAD] IO.popen(command + [file].compact, &:read) end def commit(file) - if svn_mode? - begin - execute('svn', 'ci', '-F', file) - ensure - execute('rm', '-f', 'subversion.commitlog') - end - else - current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip - execute('git', 'add', '.') && - execute('git', 'commit', '-F', file) && - execute('git', 'push', 'origin', current_branch) - end + current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip + execute('git', 'add', '.') && execute('git', 'commit', '-F', file) end - private - - def svn_mode? - return @svn_mode if defined?(@svn_mode) - @svn_mode = system("svn info", %i(out err) => IO::NULL) + def has_conflicts? + changes = IO.popen(%w[git status --porcelain -z]) { |io| io.readlines("\0", chomp: true) } + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + !conflict.empty? end + private + # Prints the version of Ruby found in version.h def version v = p = nil @@ -283,21 +246,21 @@ else case ARGV[0] when /--ticket=(.*)/ - tickets = $1.split(/,/).map{|num| " [Backport ##{num}]"}.join + tickets = $1.split(/,/) ARGV.shift else - tickets = '' + tickets = [] + detect_ticket = true end - revstr = ARGV[0].delete('^, :\-0-9a-fA-F') + revstr = ARGV[0].gsub(%r!https://github\.com/ruby/ruby/commit/|https://bugs\.ruby-lang\.org/projects/ruby-master/repository/git/revisions/!, '') + revstr = revstr.delete('^, :\-0-9a-fA-F') revs = revstr.split(/[,\s]+/) commit_message = '' revs.each do |rev| git_rev = nil case rev - when /\A\d{1,6}\z/ - svn_rev = rev when /\A\h{7,40}\z/ git_rev = rev when nil then @@ -308,28 +271,25 @@ else exit end - # Merge revision from Git patch or SVN - if git_rev - git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" - resp = Net::HTTP.get_response(URI(git_uri)) - if resp.code != '200' - abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" - end - patch = resp.body - - message = "\n\n#{(patch[/^Subject: (.*)\n\ndiff --git/m, 1] || "Message not found for revision: #{git_rev}\n")}" - puts '+ git apply' - IO.popen(['git', 'apply'], 'w') { |f| f.write(patch) } - else - default_merge_branch = (%r{^URL: .*/branches/ruby_1_8_} =~ `svn info` ? 'branches/ruby_1_8' : 'trunk') - svn_src = "#{Merger::REPOS}#{ARGV[1] || default_merge_branch}" - message = IO.popen(['svn', 'log', '-c', svn_rev, svn_src], &:read) + # Merge revision from Git patch + git_uri = "https://github.com/ruby/ruby/commit/#{git_rev}.patch" + resp = Net::HTTP.get_response(URI(git_uri)) + if resp.code != '200' + abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" + end + patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - cmd = ['svn', 'merge', '--accept=postpone', '-c', svn_rev, svn_src] - puts "+ #{cmd.join(' ')}" - system(*cmd) + if detect_ticket + tickets += patch.scan(/\[(?:Bug|Feature|Misc) #(\d+)\]/i).map(&:first) end + message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" + message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") + message = "\n\n#{message}" + + puts '+ git apply' + IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } + commit_message << message.sub(/\A-+\nr.*/, '').sub(/\n-+\n\z/, '').gsub(/^./, "\t\\&") end @@ -339,20 +299,22 @@ else Merger.version_up f = Tempfile.new 'merger.rb' - f.printf "merge revision(s) %s:%s", revstr, tickets + f.printf "merge revision(s) %s:%s", revs.join(', '), tickets.map{|num| " [Backport ##{num}]"}.join f.write commit_message f.flush f.close - Merger.interactive('conflicts resolved?', f.path) do - IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| - g << Merger.stat - g << "\n\n" - f.open - g << f.read - f.close - g << "\n\n" - g << Merger.diff + if Merger.has_conflicts? + Merger.interactive('conflicts resolved?', f.path) do + IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| + g << Merger.stat + g << "\n\n" + f.open + g << f.read + f.close + g << "\n\n" + g << Merger.diff + end end end diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat new file mode 100755 index 0000000000..d39568fe86 --- /dev/null +++ b/tool/missing-baseruby.bat @@ -0,0 +1,30 @@ +:"" == " +@echo off || ( + :warn + echo>&2.%~1 + goto :eof + :abort + exit /b 1 +)||( +:)"||( + # necessary libraries + require 'erb' + require 'fileutils' + require 'tempfile' + s = %^# +) +: ; call() { local call=${1#:}; shift; $call "$@"; } +: ; warn() { echo "$1" >&2; } +: ; abort () { exit 1; } + +call :warn "executable host ruby is required. use --with-baseruby option." +call :warn "Note that BASERUBY must be Ruby 3.1.0 or later." +call :abort +(goto :eof ^;) +verbose = true if ARGV[0] == "--verbose" +case +when !defined?(RubyVM::InstructionSequence) + abort(*(["BASERUBY must be CRuby"] if verbose)) +when RUBY_VERSION < s[%r[warn .*\KBASERUBY .*Ruby ([\d.]+)(?:\.0)?.*(?=\")],1] + abort(*(["#{$&}. Found: #{RUBY_VERSION}"] if verbose)) +end diff --git a/tool/mjit_archflag.sh b/tool/mjit_archflag.sh deleted file mode 100644 index 082fb4bcd0..0000000000 --- a/tool/mjit_archflag.sh +++ /dev/null @@ -1,40 +0,0 @@ -# -*- sh -*- - -quote() { - printf "#${indent}define $1" - shift - ${1+printf} ${1+' "%s"'$sep} ${1+"$@"} - echo -} - -archs="" -arch_flag="" - -parse_arch_flags() { - for arch in $1; do - archs="${archs:+$archs }${arch%=*}" - done - - while shift && [ "$#" -gt 0 ]; do - case "$1" in - -arch) - shift - archs="${archs:+$archs }$1" - ;; - *) - arch_flag="${arch_flag:+${arch_flag} }$1" - ;; - esac - done -} - -define_arch_flags() { - ${archs:+echo} ${archs:+'#if 0'} - for arch in $archs; do - echo "#elif defined __${arch}__" - quote "MJIT_ARCHFLAG " -arch "${arch}" - done - ${archs:+echo} ${archs:+'#else'} - quote "MJIT_ARCHFLAG /* ${arch_flag:-no flag} */" ${arch_flag} - ${archs:+echo} ${archs:+'#endif'} -} diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb new file mode 100644 index 0000000000..5aa07962f9 --- /dev/null +++ b/tool/mk_builtin_loader.rb @@ -0,0 +1,429 @@ +# Parse built-in script and make rbinc file + +require 'ripper' +require 'stringio' +require_relative 'ruby_vm/helpers/c_escape' + +SUBLIBS = {} +REQUIRED = {} +BUILTIN_ATTRS = %w[leaf inline_block use_block c_trace] + +module CompileWarning + @@warnings = 0 + + def warn(message) + @@warnings += 1 + super + end + + def self.reset + w, @@warnings = @@warnings, 0 + w.nonzero? + end +end + +Warning.extend CompileWarning + +def string_literal(lit, str = []) + while lit + case lit.first + when :string_concat, :string_embexpr, :string_content + _, *lit = lit + lit.each {|s| string_literal(s, str)} + return str + when :string_literal + _, lit = lit + when :@tstring_content + str << lit[1] + return str + else + raise "unexpected #{lit.first}" + end + end +end + +# e.g. [:symbol_literal, [:symbol, [:@ident, "inline", [19, 21]]]] +def symbol_literal(lit) + symbol_literal, symbol_lit = lit + raise "#{lit.inspect} was not :symbol_literal" if symbol_literal != :symbol_literal + symbol, ident_lit = symbol_lit + raise "#{symbol_lit.inspect} was not :symbol" if symbol != :symbol + ident, symbol_name, = ident_lit + raise "#{ident.inspect} was not :@ident" if ident != :@ident + symbol_name +end + +def inline_text argc, arg1 + raise "argc (#{argc}) of inline! should be 1" unless argc == 1 + arg1 = string_literal(arg1) + raise "1st argument should be string literal" unless arg1 + arg1.join("").rstrip +end + +def inline_attrs(args) + raise "args was empty" if args.empty? + args.each do |arg| + attr = symbol_literal(arg) + unless BUILTIN_ATTRS.include?(attr) + raise "attr (#{attr}) was not in: #{BUILTIN_ATTRS.join(', ')}" + end + end +end + +def make_cfunc_name inlines, name, lineno + case name + when /\[\]/ + name = '_GETTER' + when /\[\]=/ + name = '_SETTER' + else + name = name.tr('!?', 'EP') + end + + base = "builtin_inline_#{name}_#{lineno}" + if inlines[base] + 1000.times{|i| + name = "#{base}_#{i}" + return name unless inlines[name] + } + raise "too many functions in same line..." + else + base + end +end + +def collect_locals tree + _type, name, (line, _cols) = tree + if locals = LOCALS_DB[[name, line]] + locals + else + if false # for debugging + pp LOCALS_DB + raise "not found: [#{name}, #{line}]" + end + end +end + +def collect_builtin base, tree, name, bs, inlines, locals = nil + while tree + recv = sep = mid = args = nil + case tree.first + when :def + locals = collect_locals(tree[1]) + tree = tree[3] + next + when :defs + locals = collect_locals(tree[3]) + tree = tree[5] + next + when :class + name = 'class' + tree = tree[3] + next + when :sclass, :module + name = 'class' + tree = tree[2] + next + when :method_add_arg + _method_add_arg, mid, (_arg_paren, args) = tree + case mid.first + when :call + _, recv, sep, mid = mid + when :fcall + _, mid = mid + else + mid = nil + end + # w/ trailing comma: [[:method_add_arg, ...]] + # w/o trailing comma: [:args_add_block, [[:method_add_arg, ...]], false] + if args && args.first == :args_add_block + args = args[1] + end + when :vcall + _, mid = tree + when :command # FCALL + _, mid, (_, args) = tree + when :call, :command_call # CALL + _, recv, sep, mid, (_, args) = tree + end + + if mid + raise "unknown sexp: #{mid.inspect}" unless %i[@ident @const].include?(mid.first) + _, mid, (lineno,) = mid + if recv + func_name = nil + case recv.first + when :var_ref + _, recv = recv + if recv.first == :@const and recv[1] == "Primitive" + func_name = mid.to_s + end + when :vcall + _, recv = recv + if recv.first == :@ident and recv[1] == "__builtin" + func_name = mid.to_s + end + end + collect_builtin(base, recv, name, bs, inlines) unless func_name + else + func_name = mid[/\A__builtin_(.+)/, 1] + end + if func_name + cfunc_name = func_name + args.pop unless (args ||= []).last + argc = args.size + + if /(.+)[\!\?]\z/ =~ func_name + case $1 + when 'attr' + # Compile-time validation only. compile.c will parse them. + inline_attrs(args) + break + when 'cstmt' + text = inline_text argc, args.first + + func_name = "_bi#{lineno}" + cfunc_name = make_cfunc_name(inlines, name, lineno) + inlines[cfunc_name] = [lineno, text, locals, func_name] + argc -= 1 + when 'cexpr', 'cconst' + text = inline_text argc, args.first + code = "return #{text};" + + func_name = "_bi#{lineno}" + cfunc_name = make_cfunc_name(inlines, name, lineno) + + locals = [] if $1 == 'cconst' + inlines[cfunc_name] = [lineno, code, locals, func_name] + argc -= 1 + when 'cinit' + text = inline_text argc, args.first + func_name = nil # required + inlines[inlines.size] = [lineno, text, nil, nil] + argc -= 1 + when 'mandatory_only' + func_name = nil + when 'arg' + argc == 1 or raise "unexpected argument number #{argc}" + (arg = args.first)[0] == :symbol_literal or raise "symbol literal expected #{args}" + (arg = arg[1])[0] == :symbol or raise "symbol expected #{arg}" + (var = arg[1] and var = var[1]) or raise "argument name expected #{arg}" + func_name = nil + end + end + + if bs[func_name] && + bs[func_name] != [argc, cfunc_name] + raise "same builtin function \"#{func_name}\", but different arity (was #{bs[func_name]} but #{argc})" + end + + bs[func_name] = [argc, cfunc_name] if func_name + elsif /\Arequire(?:_relative)\z/ =~ mid and args.size == 1 and + (arg1 = args[0])[0] == :string_literal and + (arg1 = arg1[1])[0] == :string_content and + (arg1 = arg1[1])[0] == :@tstring_content and + sublib = arg1[1] + if File.exist?(f = File.join(@dir, sublib)+".rb") + puts "- #{@base}.rb requires #{sublib}" + if REQUIRED[sublib] + warn "!!! #{sublib} is required from #{REQUIRED[sublib]} already; ignored" + else + REQUIRED[sublib] = @base + (SUBLIBS[@base] ||= []) << sublib + end + ARGV.push(f) + end + end + break unless tree = args + end + + tree.each do |t| + collect_builtin base, t, name, bs, inlines, locals if Array === t + end + break + end +end + +# ruby mk_builtin_loader.rb TARGET_FILE.rb +# #=> generate TARGET_FILE.rbinc +# + +LOCALS_DB = {} # [method_name, first_line] = locals + +def collect_iseq iseq_ary + # iseq_ary.each_with_index{|e, i| p [i, e]} + label = iseq_ary[5] + first_line = iseq_ary[8] + type = iseq_ary[9] + locals = iseq_ary[10] + insns = iseq_ary[13] + + if type == :method + LOCALS_DB[[label, first_line].freeze] = locals + end + + insns.each{|insn| + case insn + when Integer + # ignore + when Array + # p insn.shift # insn name + insn.each{|op| + if Array === op && op[0] == "YARVInstructionSequence/SimpleDataFormat" + collect_iseq op + end + } + end + } +end + +def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) + f = StringIO.new + + # Avoid generating fetches of lvars we don't need. This is imperfect as it + # will match text inside strings or other false positives. + local_ptrs = [] + local_candidates = text.gsub(/\bLOCAL_PTR\(\K[a-zA-Z_][a-zA-Z0-9_]*(?=\))/) { + local_ptrs << $&; '' + }.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + + f.puts '{' + lineno += 1 + # locals is nil outside methods + locals&.reverse_each&.with_index{|param, i| + next unless Symbol === param + param = param.to_s + lvar = local_candidates.include?(param) + next unless lvar or local_ptrs.include?(param) + f.puts "VALUE *const #{param}__ptr = (VALUE *)&ec->cfp->ep[#{-3 - i}];" + f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" if lvar + lineno += lvar ? 2 : 1 + } + f.puts "#line #{body_lineno} \"#{line_file}\"" + lineno += 1 + + f.puts text + lineno += text.count("\n") + 1 + + f.puts "#line #{lineno + 2} \"#{ofile}\"" # TODO: restore line number. + f.puts "}" + f.puts + lineno += 3 + + return lineno, f.string +end + +def mk_builtin_header file + @dir = File.dirname(file) + base = File.basename(file, '.rb') + @base = base + ofile = "#{file}inc" + + # bs = { func_name => argc } + code = File.read(file) + begin + verbose, $VERBOSE = $VERBOSE, true + collect_iseq RubyVM::InstructionSequence.compile(code, base).to_a + ensure + $VERBOSE = verbose + end + if warnings = CompileWarning.reset + raise "#{warnings} warnings in #{file}" + end + collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) + + StringIO.open do |f| + if File::ALT_SEPARATOR + file = file.tr(File::ALT_SEPARATOR, File::SEPARATOR) + ofile = ofile.tr(File::ALT_SEPARATOR, File::SEPARATOR) + end + lineno = __LINE__ + f.puts "// -*- c -*-" + f.puts "// DO NOT MODIFY THIS FILE DIRECTLY." + f.puts "// auto-generated file" + f.puts "// by #{__FILE__}" + f.puts "// with #{file}" + f.puts '#include "internal/compilers.h" /* for MAYBE_UNUSED */' + f.puts '#include "internal/warnings.h" /* for COMPILER_WARNING_PUSH */' + f.puts '#include "ruby/ruby.h" /* for VALUE */' + f.puts '#include "builtin.h" /* for RB_BUILTIN_FUNCTION */' + f.puts 'struct rb_execution_context_struct; /* in vm_core.h */' + f.puts + lineno = __LINE__ - lineno - 1 + line_file = file + + inlines.each{|cfunc_name, (body_lineno, text, locals, func_name)| + if String === cfunc_name + f.puts "static VALUE #{cfunc_name}(struct rb_execution_context_struct *ec, const VALUE self)" + lineno += 1 + lineno, str = generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) + f.write str + else + # cinit! + f.puts "#line #{body_lineno} \"#{line_file}\"" + lineno += 1 + f.puts text + lineno += text.count("\n") + 1 + f.puts "#line #{lineno + 2} \"#{ofile}\"" # TODO: restore line number. + lineno += 1 + end + } + + if SUBLIBS[base] + f.puts "// sub libraries" + SUBLIBS[base].each do |sub| + f.puts %[#include #{(sub+".rbinc").dump}] + end + f.puts + end + + f.puts "void Init_builtin_#{base}(void)" + f.puts "{" + + table = "#{base}_table" + f.puts " // table definition" + f.puts " static const struct rb_builtin_function #{table}[] = {" + bs.each.with_index{|(func, (argc, cfunc_name)), i| + f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc})," + } + f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0)," + f.puts " };" + + f.puts + f.puts " // arity_check" + f.puts "COMPILER_WARNING_PUSH" + f.puts "#if GCC_VERSION_SINCE(5, 1, 0) || defined __clang__" + f.puts "COMPILER_WARNING_ERROR(-Wincompatible-pointer-types)" + f.puts "#endif" + bs.each{|func, (argc, cfunc_name)| + f.puts " if (0) rb_builtin_function_check_arity#{argc}(#{cfunc_name});" + } + f.puts "COMPILER_WARNING_POP" + + if SUBLIBS[base] + f.puts + f.puts " // sub libraries" + SUBLIBS[base].each do |sub| + f.puts " Init_builtin_#{sub}();" + end + end + + f.puts + f.puts " // load" + f.puts " rb_load_with_builtin_functions(#{base.dump}, #{table});" + + f.puts "}" + + begin + File.write(ofile, f.string) + rescue SystemCallError # EACCES, EPERM, EROFS, etc. + # Fall back to the current directory + File.write(File.basename(ofile), f.string) + end + end +end + +ARGV.each{|file| + # feature.rb => load_feature.inc + mk_builtin_header file +} diff --git a/tool/mk_call_iseq_optimized.rb b/tool/mk_call_iseq_optimized.rb deleted file mode 100644 index 9fd84aa696..0000000000 --- a/tool/mk_call_iseq_optimized.rb +++ /dev/null @@ -1,74 +0,0 @@ - -puts <<EOS -/* -*- c -*- */ -#if 1 /* enable or disable this optimization */ - -/* DO NOT EDIT THIS FILE DIRECTLY - * - * This file is generated by tool/mk_call_iseq_optimized.rb - */ - -EOS - -P = (0..3) -L = (0..5) - -def fname param, local - "vm_call_iseq_setup_normal_0start_#{param}params_#{local}locals" -end - -P.each{|param| - L.each{|local| - puts <<EOS -static VALUE -#{fname(param, local)}(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_call_info *ci, struct rb_call_cache *cc) -{ - RB_DEBUG_COUNTER_INC(ccf_iseq_fix); - return vm_call_iseq_setup_normal(ec, cfp, calling, cc->me, 0, #{param}, #{local}); -} - -EOS - # - } -} - -puts <<EOS -/* vm_call_iseq_handlers[param][local] */ -static const vm_call_handler vm_call_iseq_handlers[][#{L.to_a.size}] = { -#{P.map{|param| '{' + L.map{|local| fname(param, local)}.join(",\n ") + '}'}.join(",\n")} -}; - -static inline vm_call_handler -vm_call_iseq_setup_func(const struct rb_call_info *ci, const int param_size, const int local_size) -{ - if (UNLIKELY(ci->flag & VM_CALL_TAILCALL)) { - return &vm_call_iseq_setup_tailcall_0start; - } - else if (0) { /* to disable optimize */ - return &vm_call_iseq_setup_normal_0start; - } - else { - if (param_size <= #{P.end} && - local_size <= #{L.end}) { - VM_ASSERT(local_size >= 0); - return vm_call_iseq_handlers[param_size][local_size]; - } - return &vm_call_iseq_setup_normal_0start; - } -} - -#else - - -static inline vm_call_handler -vm_call_iseq_setup_func(const struct rb_call_info *ci, struct rb_call_cache *cc) -{ - if (UNLIKELY(ci->flag & VM_CALL_TAILCALL)) { - return &vm_call_iseq_setup_tailcall_0start; - } - else { - return &vm_call_iseq_setup_normal_0start; - } -} -#endif -EOS diff --git a/tool/mk_rbbin.rb b/tool/mk_rbbin.rb new file mode 100755 index 0000000000..991230f094 --- /dev/null +++ b/tool/mk_rbbin.rb @@ -0,0 +1,48 @@ +#!ruby -s + +OPTIMIZATION = { + inline_const_cache: true, + peephole_optimization: true, + tailcall_optimization: false, + specialized_instruction: true, + operands_unification: true, + instructions_unification: true, + frozen_string_literal: true, + debug_frozen_string_literal: false, + coverage_enabled: false, + debug_level: 0, +} + +file = File.basename(ARGV[0], ".rb") +name = "<internal:#{file}>" +iseq = RubyVM::InstructionSequence.compile(ARGF.read, name, name, **OPTIMIZATION) +puts <<C +/* -*- C -*- */ + +static const char #{file}_builtin[] = { +C +iseq.to_binary.bytes.each_slice(8) do |b| + print " ", b.map {|c| "0x%.2x," % c}.join(" ") + if $comment + print " /* ", b.pack("C*").gsub(/([[ -~]&&[^\\]])|(?m:.)/) { + (c = $1) ? "#{c} " : (c = $&.dump).size == 2 ? c : ". " + }, "*/" + end + puts +end +puts <<C +}; + +#include "ruby/ruby.h" +#include "vm_core.h" + +void +Init_#{file}(void) +{ + const char *builtin = #{file}_builtin; + size_t size = sizeof(#{file}_builtin); + VALUE code = rb_str_new_static(builtin, (long)size); + VALUE iseq = rb_funcallv(rb_cISeq, rb_intern_const("load_from_binary"), 1, &code); + rb_funcallv(iseq, rb_intern_const("eval"), 0, 0); +} +C diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb index 586eb9ec59..db74115730 100755 --- a/tool/mkconfig.rb +++ b/tool/mkconfig.rb @@ -1,4 +1,5 @@ #!./miniruby -s +# frozen-string-literal: true # This script, which is run when ruby is built, generates rbconfig.rb by # parsing information from config.status. rbconfig.rb contains build @@ -29,6 +30,7 @@ continued_name = nil continued_line = nil install_name = nil so_name = nil +platform = nil File.foreach "config.status" do |line| next if /^#/ =~ line name = nil @@ -62,8 +64,6 @@ File.foreach "config.status" do |line| when /^(?:X|(?:MINI|RUN|(?:HAVE_)?BASE|BOOTSTRAP|BTEST)RUBY(?:_COMMAND)?$)/; next when /^INSTALLDOC|TARGET$/; next when /^DTRACE/; next - when /^MJIT_SUPPORT/; # pass - when /^MJIT_/; next when /^(?:MAJOR|MINOR|TEENY)$/; vars[name] = val; next when /^LIBRUBY_D?LD/; next when /^RUBY_INSTALL_NAME$/; next vars[name] = (install_name = val).dup if $install_name @@ -87,7 +87,7 @@ File.foreach "config.status" do |line| unless $install_name $install_name = "ruby" val.gsub!(/\$\$/, '$') - val.scan(%r[\G[\s;]*(/(?:\\.|[^/])*/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*)\3((?:(?!\3)(?:\\.|.))*)\3([gi]*)]) do + val.scan(%r[\G[\s;]*(/(?:\\.|[^/])*+/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*+)\3((?:(?!\3)(?:\\.|.))*+)\3([gi]*)]) do |addr, cmd, sep, pat, rep, opt| if addr Regexp.new(addr[/\A\/(.*)\/\z/, 1]) =~ $install_name or next @@ -121,9 +121,17 @@ File.foreach "config.status" do |line| universal, val = val, 'universal' if universal when /^arch$/ if universal - val.sub!(/universal/, %q[#{arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || '\&'}]) + platform = val.sub(/universal/, '$(arch)') end - when /^oldincludedir$/ + when /^target_cpu$/ + if universal + val = 'cpu' + end + when /^target$/ + val = '"$(target_cpu)-$(target_vendor)-$(target_os)"' + when /^host(?:_(?:os|vendor|cpu|alias))?$/ + val = %["$(#{name.sub(/^host/, 'target')})"] + when /^includedir$/ val = '"$(SDKROOT)"'+val if /darwin/ =~ arch end v = " CONFIG[\"#{name}\"] #{eq} #{val}\n" @@ -160,8 +168,8 @@ def vars.expand(val, config = self) val.replace(newval) unless newval == val val end -prefix = vars.expand(vars["prefix"] ||= "") -rubyarchdir = vars.expand(vars["rubyarchdir"] ||= "") +prefix = vars.expand(vars["prefix"] ||= +"") +rubyarchdir = vars.expand(vars["rubyarchdir"] ||= +"") relative_archdir = rubyarchdir.rindex(prefix, 0) ? rubyarchdir[prefix.size..-1] : rubyarchdir puts %[\ @@ -184,17 +192,19 @@ print " # Ruby installed directory.\n" print " TOPDIR = File.dirname(__FILE__).chomp!(#{relative_archdir.dump})\n" print " # DESTDIR on make install.\n" print " DESTDIR = ", (drive ? "TOPDIR && TOPDIR[/\\A[a-z]:/i] || " : ""), "'' unless defined? DESTDIR\n" -print <<'ARCH' if universal +print <<"UNIVERSAL", <<'ARCH' if universal + universal = #{universal} +UNIVERSAL arch_flag = ENV['ARCHFLAGS'] || ((e = ENV['RC_ARCHS']) && e.split.uniq.map {|a| "-arch #{a}"}.join(' ')) arch = arch_flag && arch_flag[/\A\s*-arch\s+(\S+)\s*\z/, 1] + cpu = arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/] ARCH -print " universal = #{universal}\n" if universal print " # The hash configurations stored.\n" print " CONFIG = {}\n" print " CONFIG[\"DESTDIR\"] = DESTDIR\n" versions = {} -IO.foreach(File.join(srcdir, "version.h")) do |l| +File.foreach(File.join(srcdir, "version.h")) do |l| m = /^\s*#\s*define\s+RUBY_(PATCHLEVEL)\s+(-?\d+)/.match(l) if m versions[m[1]] = m[2] @@ -215,7 +225,7 @@ IO.foreach(File.join(srcdir, "version.h")) do |l| end end if versions.size != 4 - IO.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| + File.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| m = /^\s*#\s*define\s+RUBY_API_VERSION_(\w+)\s+(-?\d+)/.match(l) if m versions[m[1]] ||= m[2] @@ -246,14 +256,14 @@ end v_others.compact! if $install_name - if install_name and vars.expand("$(RUBY_INSTALL_NAME)") == $install_name + if install_name and vars.expand(+"$(RUBY_INSTALL_NAME)") == $install_name $install_name = install_name end v_fast << " CONFIG[\"ruby_install_name\"] = \"" + $install_name + "\"\n" v_fast << " CONFIG[\"RUBY_INSTALL_NAME\"] = \"" + $install_name + "\"\n" end if $so_name - if so_name and vars.expand("$(RUBY_SO_NAME)") == $so_name + if so_name and vars.expand(+"$(RUBY_SO_NAME)") == $so_name $so_name = so_name end v_fast << " CONFIG[\"RUBY_SO_NAME\"] = \"" + $so_name + "\"\n" @@ -267,10 +277,21 @@ EOS print <<EOS if $unicode_emoji_version CONFIG["UNICODE_EMOJI_VERSION"] = #{$unicode_emoji_version.dump} EOS -print <<EOS if /darwin/ =~ arch - CONFIG["SDKROOT"] = ENV["SDKROOT"] || "" # don't run xcrun everytime, usually useless. +print prefix.start_with?("/System/") ? <<EOS : <<EOS if /darwin/ =~ arch + if sdkroot = ENV["SDKROOT"] + sdkroot = sdkroot.dup + elsif File.exist?(File.join(CONFIG["prefix"], "include")) || + !(sdkroot = (IO.popen(%w[/usr/bin/xcrun --sdk macosx --show-sdk-path], in: IO::NULL, err: IO::NULL, &:read) rescue nil)) + sdkroot = +"" + else + sdkroot.chomp! + end + CONFIG["SDKROOT"] = sdkroot +EOS + CONFIG["SDKROOT"] = "" EOS print <<EOS + CONFIG["platform"] = #{platform || '"$(arch)"'} CONFIG["archdir"] = "$(rubyarchdir)" CONFIG["topdir"] = File.dirname(__FILE__) # Almost same with CONFIG. MAKEFILE_CONFIG has other variable @@ -283,9 +304,9 @@ print <<EOS # require 'rbconfig' # # print <<-END_OF_MAKEFILE - # prefix = \#{Config::MAKEFILE_CONFIG['prefix']} - # exec_prefix = \#{Config::MAKEFILE_CONFIG['exec_prefix']} - # bindir = \#{Config::MAKEFILE_CONFIG['bindir']} + # prefix = \#{RbConfig::MAKEFILE_CONFIG['prefix']} + # exec_prefix = \#{RbConfig::MAKEFILE_CONFIG['exec_prefix']} + # bindir = \#{RbConfig::MAKEFILE_CONFIG['bindir']} # END_OF_MAKEFILE # # => prefix = /usr/local @@ -295,7 +316,7 @@ print <<EOS # RbConfig.expand is used for resolving references like above in rbconfig. # # require 'rbconfig' - # p Config.expand(Config::MAKEFILE_CONFIG["bindir"]) + # p RbConfig.expand(RbConfig::MAKEFILE_CONFIG["bindir"]) # # => "/usr/local/bin" MAKEFILE_CONFIG = {} CONFIG.each{|k,v| MAKEFILE_CONFIG[k] = v.dup} @@ -330,11 +351,10 @@ print <<EOS RbConfig::expand(val) end - # :nodoc: # call-seq: # - # RbConfig.fire_update!(key, val) -> string - # RbConfig.fire_update!(key, val, mkconf, conf) -> string + # RbConfig.fire_update!(key, val) -> array + # RbConfig.fire_update!(key, val, mkconf, conf) -> array # # updates +key+ in +mkconf+ with +val+, and all values depending on # the +key+ in +mkconf+. @@ -346,7 +366,7 @@ print <<EOS # RbConfig::CONFIG.values_at("CC", "LDSHARED") # => ["gcc-8", "gcc-8 -shared"] # # returns updated keys list, or +nil+ if nothing changed. - def RbConfig.fire_update!(key, val, mkconf = MAKEFILE_CONFIG, conf = CONFIG) + def RbConfig.fire_update!(key, val, mkconf = MAKEFILE_CONFIG, conf = CONFIG) # :nodoc: return if mkconf[key] == val mkconf[key] = val keys = [key] @@ -374,6 +394,7 @@ print <<EOS ) end end +# Non-nil if configured for cross compiling. CROSS_COMPILING = nil unless defined? CROSS_COMPILING EOS diff --git a/tool/mkrunnable.rb b/tool/mkrunnable.rb index 01756a3529..ef358e2425 100755 --- a/tool/mkrunnable.rb +++ b/tool/mkrunnable.rb @@ -6,6 +6,7 @@ require './rbconfig' require 'fileutils' +require_relative 'lib/path' case ARGV[0] when "-n" @@ -18,77 +19,7 @@ else include FileUtils end -module Mswin - def ln_safe(src, dest, *opt) - cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] - cmd[1, 0] = opt - return if system("cmd", "/c", *cmd) - # TODO: use RUNAS or something - puts cmd.join(" ") - end - - def ln_dir_safe(src, dest) - ln_safe(src, dest, "/d") - end -end - -def clean_link(src, dest) - begin - link = File.readlink(dest) - rescue - else - return if link == src - File.unlink(dest) - end - yield src, dest -end - -def ln_safe(src, dest) - ln_sf(src, dest) -end - -alias ln_dir_safe ln_safe - -if !File.respond_to?(:symlink) && /mingw|mswin/ =~ (CROSS_COMPILING || RUBY_PLATFORM) - extend Mswin -end - -def clean_path(path) - path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') - nil while path.sub!(/[^\/]+\/\.\.\//, '') - path -end - -def relative_path_from(path, base) - path = clean_path(path) - base = clean_path(base) - path, base = [path, base].map{|s|s.split("/")} - until path.empty? or base.empty? or path[0] != base[0] - path.shift - base.shift - end - path, base = [path, base].map{|s|s.join("/")} - if /(\A|\/)\.\.\// =~ base - File.expand_path(path) - else - base.gsub!(/[^\/]+/, '..') - File.join(base, path) - end -end - -def ln_relative(src, dest) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_safe(s, d)} -end - -def ln_dir_relative(src, dest) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_dir_safe(s, d)} -end +include Path config = RbConfig::MAKEFILE_CONFIG.merge("prefix" => ".", "exec_prefix" => ".") config.each_value {|s| RbConfig.expand(s, config)} @@ -103,18 +34,25 @@ vendordir = config["vendordir"] rubylibdir = config["rubylibdir"] rubyarchdir = config["rubyarchdir"] archdir = "#{extout}/#{arch}" -[bindir, libdir, archdir].uniq.each do |dir| +exedir = bindir +if libdirname == "archlibdir" + exedir = exedir.sub(%r[/\K(?=[^/]+\z)]) {extout+"/"} +end +[exedir, libdir, archdir].uniq.each do |dir| File.directory?(dir) or mkdir_p(dir) end +unless exedir == bindir + ln_dir_relative(exedir, bindir) +end exeext = config["EXEEXT"] ruby_install_name = config["ruby_install_name"] rubyw_install_name = config["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name -[ruby_install_name, rubyw_install_name, goruby_install_name].map do |ruby| - ruby += exeext - if ruby and !ruby.empty? and !File.file?(target = "#{bindir}/#{ruby}") - ln_relative(ruby, target) +[ruby_install_name, rubyw_install_name, goruby_install_name].each do |ruby| + if ruby and !ruby.empty? + ruby += exeext + ln_relative(ruby, "#{exedir}/#{ruby}", true) end end so = config["LIBRUBY_SO"] diff --git a/tool/notes-github-pr.rb b/tool/notes-github-pr.rb new file mode 100644 index 0000000000..d69d479cdf --- /dev/null +++ b/tool/notes-github-pr.rb @@ -0,0 +1,138 @@ +#!/usr/bin/env ruby +# Add GitHub pull request reference / author info to git notes. + +require 'net/http' +require 'uri' +require 'tmpdir' +require 'json' +require 'yaml' + +# Conversion for people whose GitHub account name and SVN_ACCOUNT_NAME are different. +GITHUB_TO_SVN = { + 'amatsuda' => 'a_matsuda', + 'matzbot' => 'git', + 'jeremyevans' => 'jeremy', + 'znz' => 'kazu', + 'k-tsj' => 'ktsj', + 'nurse' => 'naruse', + 'ioquatix' => 'samuel', + 'suketa' => 'suke', + 'unak' => 'usa', +} + +EMAIL_YML_URL = 'https://raw.githubusercontent.com/ruby/git.ruby-lang.org/refs/heads/master/config/email.yml' +SVN_TO_EMAILS = YAML.safe_load(Net::HTTP.get_response(URI(EMAIL_YML_URL)).tap(&:value).body) + +class GitHub + ENDPOINT = URI.parse('https://api.github.com') + + def initialize(access_token) + @access_token = access_token + end + + # https://developer.github.com/changes/2019-04-11-pulls-branches-for-commit/ + def pulls(owner:, repo:, commit_sha:) + resp = get("/repos/#{owner}/#{repo}/commits/#{commit_sha}/pulls", accept: 'application/vnd.github.groot-preview+json') + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/pulls/#get-a-single-pull-request + def pull_request(owner:, repo:, number:) + resp = get("/repos/#{owner}/#{repo}/pulls/#{number}") + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/users/#get-a-single-user + def user(username:) + resp = get("/users/#{username}") + JSON.parse(resp.body) + end + + private + + def get(path, accept: 'application/vnd.github.v3+json') + Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: ENDPOINT.scheme == 'https') do |http| + headers = { 'Accept': accept, 'Authorization': "bearer #{@access_token}" } + http.get(path, headers).tap(&:value) + end + end +end + +module Git + class << self + def abbrev_ref(refname, repo_path:) + git('rev-parse', '--symbolic', '--abbrev-ref', refname, repo_path: repo_path).strip + end + + def rev_list(arg, first_parent: false, repo_path: nil) + git('rev-list', *[('--first-parent' if first_parent)].compact, arg, repo_path: repo_path).lines.map(&:chomp) + end + + def commit_message(sha) + git('log', '-1', '--pretty=format:%B', sha) + end + + def notes_message(sha) + git('log', '-1', '--pretty=format:%N', sha) + end + + def committer_name(sha) + git('log', '-1', '--pretty=format:%cn', sha) + end + + def committer_email(sha) + git('log', '-1', '--pretty=format:%cE', sha) + end + + private + + def git(*cmd, repo_path: nil) + env = {} + if repo_path + env['GIT_DIR'] = repo_path + end + out = IO.popen(env, ['git', *cmd], &:read) + unless $?.success? + abort "Failed to execute: git #{cmd.join(' ')}\n#{out}" + end + out + end + end +end + +github = GitHub.new(ENV.fetch('GITHUB_TOKEN')) + +repo_path, *rest = ARGV +rest.each_slice(3).map do |oldrev, newrev, _refname| + system('git', 'fetch', 'origin', 'refs/notes/commits:refs/notes/commits', exception: true) + + updated = false + Git.rev_list("#{oldrev}..#{newrev}", first_parent: true).each do |sha| + github.pulls(owner: 'ruby', repo: 'ruby', commit_sha: sha).each do |pull| + number = pull.fetch('number') + url = pull.fetch('html_url') + next unless url.start_with?('https://github.com/ruby/ruby/pull/') + + # "Merged" notes for "Squash and merge" + message = Git.commit_message(sha) + notes = Git.notes_message(sha) + if !message.include?(url) && !message.match(/[ (]##{number}[) ]/) && !notes.include?(url) + system('git', 'notes', 'append', '-m', "Merged: #{url}", sha, exception: true) + updated = true + end + + # "Merged-By" notes for "Rebase and merge" + if Git.committer_name(sha) == 'GitHub' && Git.committer_email(sha) == 'noreply@github.com' + username = github.pull_request(owner: 'ruby', repo: 'ruby', number: number).fetch('merged_by').fetch('login') + email = github.user(username: username).fetch('email') + email ||= SVN_TO_EMAILS[GITHUB_TO_SVN.fetch(username, username)]&.first + system('git', 'notes', 'append', '-m', "Merged-By: #{username}#{(" <#{email}>" if email)}", sha, exception: true) + updated = true + end + end + end + + if updated + system('git', 'push', 'origin', 'refs/notes/commits', exception: true) + end +end diff --git a/tool/notify-slack-commits.rb b/tool/notify-slack-commits.rb new file mode 100644 index 0000000000..73e22b9a03 --- /dev/null +++ b/tool/notify-slack-commits.rb @@ -0,0 +1,87 @@ +#!/usr/bin/env ruby + +require "net/https" +require "open3" +require "json" +require "digest/md5" + +SLACK_WEBHOOK_URLS = [ + ENV.fetch("SLACK_WEBHOOK_URL_ALERTS").chomp, # ruby-lang#alerts + ENV.fetch("SLACK_WEBHOOK_URL_COMMITS").chomp, # ruby-lang#commits + ENV.fetch("SLACK_WEBHOOK_URL_RUBY_JP").chomp, # ruby-jp#ruby-commits +] +GRAVATAR_OVERRIDES = { + "nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars0.githubusercontent.com/u/21976", + "noreply@github.com" => "https://avatars1.githubusercontent.com/u/9919", + "nurse@users.noreply.github.com" => "https://avatars1.githubusercontent.com/u/13423", + "svn-admin@ruby-lang.org" => "https://avatars1.githubusercontent.com/u/29403229", + "svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars1.githubusercontent.com/u/29403229", + "usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars2.githubusercontent.com/u/17790", + "usa@ruby-lang.org" => "https://avatars2.githubusercontent.com/u/17790", + "yui-knk@ruby-lang.org" => "https://avatars0.githubusercontent.com/u/5356517", + "znz@users.noreply.github.com" => "https://avatars3.githubusercontent.com/u/11857", +} + +def escape(s) + s.gsub(/[&<>]/, "&" => "&", "<" => "<", ">" => ">") +end + +ARGV.each_slice(3) do |oldrev, newrev, refname| + out, = Open3.capture2("git", "rev-parse", "--symbolic", "--abbrev-ref", refname) + branch = out.strip + + out, = Open3.capture2("git", "log", "--pretty=format:%H\n%h\n%cn\n%ce\n%ct\n%B", "--abbrev=10", "-z", "#{oldrev}..#{newrev}") + + attachments = [] + out.split("\0").reverse_each do |s| + sha, sha_abbr, committer, committeremail, committertime, body = s.split("\n", 6) + subject, body = body.split("\n", 2) + + # Append notes content to `body` if it's notes + if refname.match(%r[\Arefs/notes/\w+\z]) + # `--diff-filter=AM -M` to exclude rename by git's directory optimization + object = IO.popen(["git", "diff", "--diff-filter=AM", "-M", "--name-only", "#{sha}^..#{sha}"], &:read).chomp + if md = object.match(/\A(?<prefix>\h{2})\/?(?<rest>\h{38})\z/) + body = [body, IO.popen(["git", "notes", "show", md[:prefix] + md[:rest]], &:read)].join + end + end + + gravatar = GRAVATAR_OVERRIDES.fetch(committeremail) do + "https://www.gravatar.com/avatar/#{ Digest::MD5.hexdigest(committeremail.downcase) }" + end + + attachments << { + title: "#{ sha_abbr } (#{ branch }): #{ escape(subject) }", + title_link: "https://github.com/ruby/ruby/commit/#{ sha }", + text: escape((body || "").strip), + footer: committer, + footer_icon: gravatar, + ts: committertime.to_i, + color: '#24282D', + } + end + + # 100 attachments cannot be exceeded. 20 is recommended. https://api.slack.com/docs/message-attachments + attachments.each_slice(20).each do |attachments_group| + payload = { attachments: attachments_group } + + #Net::HTTP.post( + # URI.parse(SLACK_WEBHOOK_URL), + # JSON.generate(payload), + # "Content-Type" => "application/json" + #) + responses = SLACK_WEBHOOK_URLS.map do |url| + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.start do + req = Net::HTTP::Post.new(uri.path) + req.set_form_data(payload: payload.to_json) + http.request(req) + end + end + + results = responses.map { |resp| "#{resp.code} (#{resp.body})" }.join(', ') + puts "#{results} -- #{payload.to_json}" + end +end diff --git a/tool/outdate-bundled-gems.rb b/tool/outdate-bundled-gems.rb new file mode 100755 index 0000000000..47ee80bc89 --- /dev/null +++ b/tool/outdate-bundled-gems.rb @@ -0,0 +1,190 @@ +#!/usr/bin/ruby +require 'fileutils' +require 'rubygems' + +fu = FileUtils::Verbose + +until ARGV.empty? + case ARGV.first + when '--' + ARGV.shift + break + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Don't remove + fu = FileUtils::DryRun + when /\A--make=/ + # just to run when `make -n` + when /\A--mflags=(.*)/ + fu = FileUtils::DryRun if /\A-\S*n/ =~ $1 + when /\A--gem[-_]platform=(.*)/im + ## --gem-platform=PLATFORM Platform in RubyGems style + gem_platform = $1 + ruby_platform = nil + when /\A--ruby[-_]platform=(.*)/im + ## --ruby-platform=PLATFORM Platform in Ruby style + ruby_platform = $1 + gem_platform = nil + when /\A--ruby[-_]version=(.*)/im + ## --ruby-version=VERSION Ruby version to keep + ruby_version = $1 + when /\A--only=(?:(curdir|srcdir)|all)\z/im + ## --only=(curdir|srcdir|all) Specify directory to remove gems from + only = $1&.downcase + when /\A--all\z/im + ## --all Remove all gems not only bundled gems + all = true + when /\A--help\z/im + ## --help Print this message + puts "Usage: #$0 [options] [srcdir]" + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit + when /\A-/ + raise "#{$0}: unknown option: #{ARGV.first}" + else + break + end + ## + ARGV.shift +end + +gem_platform ||= Gem::Platform.new(ruby_platform).to_s if ruby_platform + +class Removal + attr_reader :base + + def initialize(base = nil) + @base = (File.join(base, "/") if base) + @remove = {} + end + + def prefixed(name) + @base ? File.join(@base, name) : name + end + + def stripped(name) + if @base && name.start_with?(@base) + name[@base.size..-1] + else + name + end + end + + def slash(name) + name.sub(%r[[^/]\K\z], '/') + end + + def exist?(name) + !@remove.fetch(name) {|k| @remove[k] = !File.exist?(prefixed(name))} + end + def directory?(name) + !@remove.fetch(slash(name)) {|k| @remove[k] = !File.directory?(prefixed(name))} + end + + def unlink(name) + @remove[stripped(name)] = :rm_f + end + def rmdir(name) + @remove[slash(stripped(name))] = :rm_rf + end + + def glob(pattern, *rest) + Dir.glob(prefixed(pattern), *rest) {|n| + yield stripped(n) + } + end + + def sorted + @remove.sort_by {|k, | [-k.count("/"), k]} + end + + def each_file + sorted.each {|k, v| yield prefixed(k) if v == :rm_f} + end + + def each_directory + sorted.each {|k, v| yield prefixed(k) if v == :rm_rf} + end +end + +srcdir = Removal.new(ARGV.shift) +curdir = !srcdir.base || File.identical?(srcdir.base, ".") ? srcdir : Removal.new + +bundled = File.readlines("#{srcdir.base}gems/bundled_gems"). + grep(/^(\w[^\#\s]+)\s+[^\#\s]+(?:\s+[^\#\s]+\s+([^\#\s]+))?/) {$~.captures}.to_h rescue nil + +srcdir.glob(".bundle/gems/*/") do |dir| + base = File.basename(dir) + next if !all && bundled && !bundled.key?(base[/\A.+(?=-)/]) + unless srcdir.exist?("gems/#{base}.gem") + srcdir.rmdir(dir) + end +end + +srcdir.glob(".bundle/.timestamp/*.revision") do |file| + unless bundled&.fetch(File.basename(file, ".revision"), nil) + srcdir.unlink(file) + end +end + +srcdir.glob(".bundle/specifications/*.gemspec") do |spec| + unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}/") + srcdir.unlink(spec) + end +end + +curdir.glob(".bundle/specifications/*.gemspec") do |spec| + unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}") + curdir.unlink(spec) + end +end + +curdir.glob(".bundle/gems/*/") do |dir| + base = File.basename(dir) + unless curdir.exist?(".bundle/specifications/#{base}.gemspec") or + curdir.exist?("#{dir}/.bundled.#{base}.gemspec") + curdir.rmdir(dir) + end +end + +curdir.glob(".bundle/{extensions,.timestamp}/*/") do |dir| + unless gem_platform and File.fnmatch?(gem_platform, File.basename(dir)) + curdir.rmdir(dir) + end +end + +if gem_platform + curdir.glob(".bundle/{extensions,.timestamp}/#{gem_platform}/*/") do |dir| + unless ruby_version and File.fnmatch?(ruby_version, File.basename(dir, '-static')) + curdir.rmdir(dir) + end + end +end + +if ruby_version + curdir.glob(".bundle/extensions/#{gem_platform || '*'}/#{ruby_version}/*/") do |dir| + unless curdir.exist?(".bundle/specifications/#{File.basename(dir)}.gemspec") + curdir.rmdir(dir) + end + end + + curdir.glob(".bundle/.timestamp/#{gem_platform || '*'}/#{ruby_version}/.*.time") do |stamp| + dir = stamp[%r[/\.([^/]+)\.time\z], 1].gsub('.-.', '/')[%r[\A[^/]+/[^/]+]] + unless curdir.directory?(File.join(".bundle", dir)) + curdir.unlink(stamp) + end + end +end + +unless only == "curdir" + srcdir.each_file {|f| fu.rm_f(f)} + srcdir.each_directory {|d| fu.rm_rf(d)} +end +unless only == "srcdir" or curdir.equal?(srcdir) + curdir.each_file {|f| fu.rm_f(f)} + curdir.each_directory {|d| fu.rm_rf(d)} +end diff --git a/tool/prereq.status b/tool/prereq.status index 6de00c8a92..6aca615e90 100644 --- a/tool/prereq.status +++ b/tool/prereq.status @@ -41,4 +41,5 @@ s,@rubylibprefix@,,g s,@srcdir@,.,g s/@[A-Za-z][A-Za-z0-9_]*@//g -s/{\$([A-Za-z]*)}//g +s/{\$([^(){}]*)}//g +s/^!/#!/ diff --git a/tool/pure_parser.rb b/tool/pure_parser.rb deleted file mode 100755 index 4d5e86e543..0000000000 --- a/tool/pure_parser.rb +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/ruby -pi -BEGIN { - require_relative 'lib/colorize' - - colorize = Colorize.new - file = ARGV.shift - unless /\Abison .* (\d+)\.\d+/ =~ IO.popen(ARGV+%w[--version], &:read) - puts colorize.fail("not bison") - exit - end - exit if $1.to_i >= 3 - ARGV.clear - ARGV.push(file) -} -$_.sub!(/^%define\s+api\.pure/, '%pure-parser') diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 9046249bfb..874c3ef1d9 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -3,6 +3,8 @@ # Used by the "make install" target to install Ruby. # See common.mk for more details. +ENV["SDKROOT"] ||= "" if /darwin/ =~ RUBY_PLATFORM + begin load "./rbconfig.rb" rescue LoadError @@ -20,13 +22,13 @@ require 'fileutils' require 'shellwords' require 'optparse' require 'optparse/shellwords' -require 'ostruct' require 'rubygems' begin require "zlib" rescue LoadError $" << "zlib.rb" end +require_relative 'lib/path' INDENT = " "*36 STDOUT.sync = true @@ -39,6 +41,7 @@ def parse_args(argv = ARGV) $make = 'make' $mflags = [] $install = [] + $installed = {} $installed_list = nil $exclude = [] $dryrun = false @@ -49,14 +52,17 @@ def parse_args(argv = ARGV) $dir_mode = nil $script_mode = nil $strip = false + $debug_symbols = nil $cmdtype = (if File::ALT_SEPARATOR == '\\' File.exist?("rubystub.exe") ? 'exe' : 'cmd' end) mflags = [] + gnumake = false opt = OptionParser.new opt.on('-n', '--dry-run') {$dryrun = true} opt.on('--dest-dir=DIR') {|dir| $destdir = dir} opt.on('--extout=DIR') {|dir| $extout = (dir unless dir.empty?)} + opt.on('--ext-build-dir=DIR') {|v| $ext_build_dir = v } opt.on('--make=COMMAND') {|make| $make = make} opt.on('--mantype=MAN') {|man| $mantype = man} opt.on('--make-flags=FLAGS', '--mflags', Shellwords) do |v| @@ -88,12 +94,28 @@ def parse_args(argv = ARGV) opt.on('--html-output [DIR]') {|dir| $htmldir = dir} opt.on('--cmd-type=TYPE', %w[cmd plain]) {|cmd| $cmdtype = (cmd unless cmd == 'plain')} opt.on('--[no-]strip') {|strip| $strip = strip} + opt.on('--gnumake') {gnumake = true} + opt.on('--debug-symbols=SUFFIX', /\w+/) {|name| $debug_symbols = ".#{name}"} + + unless $install_procs.empty? + w = (w = ENV["COLUMNS"] and (w = w.to_i) > 80) ? w - 30 : 50 + opt.on("\n""Types for --install and --exclude:") + mesg = +" " + $install_procs.each_key do |t| + if mesg.size + t.size > w + opt.on(mesg) + mesg = +" " + end + mesg << " " << t.to_s + end + opt.on(mesg) + end opt.order!(argv) do |v| case v when /\AINSTALL[-_]([-\w]+)=(.*)/ argv.unshift("--#{$1.tr('_', '-')}=#{$2}") - when /\A\w[-\w+]*=\z/ + when /\A\w[-\w]*=/ mflags << v when /\A\w[-\w+]*\z/ $install << v.intern @@ -110,6 +132,7 @@ def parse_args(argv = ARGV) $make, *rest = Shellwords.shellwords($make) $mflags.unshift(*rest) unless rest.empty? $mflags.unshift(*mflags) + $mflags.reject! {|v| /\A-[OW]/ =~ v} if gnumake def $mflags.set?(flag) grep(/\A-(?!-).*#{flag.chr}/i) { return true } @@ -127,6 +150,7 @@ def parse_args(argv = ARGV) end $destdir ||= $mflags.defined?("DESTDIR") + $destdir = File.expand_path($destdir) unless $destdir.empty? if $extout ||= $mflags.defined?("EXTOUT") RbConfig.expand($extout) end @@ -135,7 +159,7 @@ def parse_args(argv = ARGV) if $installed_list ||= $mflags.defined?('INSTALLED_LIST') RbConfig.expand($installed_list, RbConfig::CONFIG) - $installed_list = open($installed_list, "ab") + $installed_list = File.open($installed_list, "ab") $installed_list.sync = true end @@ -144,11 +168,26 @@ def parse_args(argv = ARGV) $dir_mode ||= $prog_mode | 0700 $script_mode ||= $prog_mode + if $ext_build_dir.nil? + raise OptionParser::MissingArgument.new("--ext-build-dir=DIR") + end +end + +Compressors = {".gz"=>"gzip", ".bz2"=>"bzip2"} +def Compressors.for(type) + ext = File.extname(type) + if compress = fetch(ext, nil) + [type.chomp(ext), ext, compress] + else + [type, *find {|_, z| system(z, in: IO::NULL, out: IO::NULL)}] + end end $install_procs = Hash.new {[]} def install?(*types, &block) - $install_procs[:all] <<= block + unless types.delete(:nodefault) + $install_procs[:all] <<= block + end types.each do |type| $install_procs[type] <<= block end @@ -170,9 +209,12 @@ def install(src, dest, options = {}) options = options.clone strip = options.delete(:strip) options[:preserve] = true + srcs = Array(src).select {|s| !$installed[$made_dirs[dest] ? File.join(dest, s) : dest]} + return if srcs.empty? + src = srcs if Array === src d = with_destdir(dest) - super(src, d, options) - srcs = Array(src) + super(src, d, **options) + srcs.each {|s| $installed[$made_dirs[dest] ? File.join(dest, s) : dest] = true} if strip d = srcs.map {|s| File.join(d, File.basename(s))} if $made_dirs[dest] strip_file(d) @@ -189,15 +231,20 @@ def ln_sf(src, dest) end $made_dirs = {} + +def dir_creating(dir) + $made_dirs.fetch(dir) do + $made_dirs[dir] = true + $installed_list.puts(File.join(dir, "")) if $installed_list + yield if defined?(yield) + end +end + def makedirs(dirs) dirs = fu_list(dirs) dirs.collect! do |dir| realdir = with_destdir(dir) - realdir unless $made_dirs.fetch(dir) do - $made_dirs[dir] = true - $installed_list.puts(File.join(dir, "")) if $installed_list - File.directory?(realdir) - end + realdir unless dir_creating(dir) {File.directory?(realdir)} end.compact! super(dirs, :mode => $dir_mode) unless dirs.empty? end @@ -253,7 +300,7 @@ def install_recursive(srcdir, dest, options = {}) elsif stat.symlink? # skip else - files << [src, d, false] if File.fnmatch?(glob, f) and !skip[f] + files << [src, d, false] if File.fnmatch?(glob, f, File::FNM_EXTGLOB) and !skip[f] end end paths.insert(0, *files) @@ -261,7 +308,8 @@ def install_recursive(srcdir, dest, options = {}) end for src, d, dir in found if dir - makedirs(d) + next + # makedirs(d) else makedirs(d[/.*(?=\/)/m]) if block_given? @@ -274,11 +322,11 @@ def install_recursive(srcdir, dest, options = {}) end def open_for_install(path, mode) - data = open(realpath = with_destdir(path), "rb") {|f| f.read} rescue nil + data = File.binread(realpath = with_destdir(path)) rescue nil newdata = yield unless $dryrun unless newdata == data - open(realpath, "wb", mode) {|f| f.write newdata} + File.open(realpath, "wb", mode) {|f| f.write newdata} end File.chmod(mode, realpath) end @@ -329,15 +377,22 @@ rubyw_install_name = CONFIG["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name bindir = CONFIG["bindir", true] +if CONFIG["libdirname"] == "archlibdir" + archbindir = bindir.sub(%r[/\K(?=[^/]+\z)]) {CONFIG["config_target"] + "/"} +end libdir = CONFIG[CONFIG.fetch("libdirname", "libdir"), true] rubyhdrdir = CONFIG["rubyhdrdir", true] archhdrdir = CONFIG["rubyarchhdrdir"] || (rubyhdrdir + "/" + CONFIG['arch']) rubylibdir = CONFIG["rubylibdir", true] archlibdir = CONFIG["rubyarchdir", true] -sitelibdir = CONFIG["sitelibdir"] -sitearchlibdir = CONFIG["sitearchdir"] -vendorlibdir = CONFIG["vendorlibdir"] -vendorarchlibdir = CONFIG["vendorarchdir"] +if CONFIG["sitedir"] + sitelibdir = CONFIG["sitelibdir"] + sitearchlibdir = CONFIG["sitearchdir"] +end +if CONFIG["vendordir"] + vendorlibdir = CONFIG["vendorlibdir"] + vendorarchlibdir = CONFIG["vendorarchdir"] +end mandir = CONFIG["mandir", true] docdir = CONFIG["docdir", true] enable_shared = CONFIG["ENABLE_SHARED"] == 'yes' @@ -348,109 +403,13 @@ load_relative = CONFIG["LIBRUBY_RELATIVE"] == 'yes' rdoc_noinst = %w[created.rid] -install?(:local, :arch, :bin, :'bin-arch') do - prepare "binary commands", bindir - - install ruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - if rubyw_install_name and !rubyw_install_name.empty? - install rubyw_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - if File.exist? goruby_install_name+exeext - install goruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - if enable_shared and dll != lib - install dll, bindir, :mode => $prog_mode, :strip => $strip - end -end - -install?(:local, :arch, :lib, :'lib-arch') do - prepare "base libraries", libdir - - install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc - install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" - if dll == lib and dll != arc - for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] - ln_sf(dll, File.join(libdir, link)) - end - end - - prepare "arch files", archlibdir - install "rbconfig.rb", archlibdir, :mode => $data_mode - if CONFIG["ARCHFILE"] - for file in CONFIG["ARCHFILE"].split - install file, archlibdir, :mode => $data_mode - end - end -end - -install?(:local, :arch, :data) do - pc = CONFIG["ruby_pc"] - if pc and File.file?(pc) and File.size?(pc) - prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") - install pc, pkgconfigdir, :mode => $data_mode - end -end - -install?(:ext, :arch, :'ext-arch') do - prepare "extension objects", archlibdir - noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split - install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) - prepare "extension objects", sitearchlibdir - prepare "extension objects", vendorarchlibdir - if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and - !(extso = extso.gsub(/\\\n/, '').split).empty? - libpathenv = CONFIG["LIBPATHENV"] - dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] - prepare "external libraries", dest - for file in extso - install file, dest, :mode => $prog_mode - end - end -end -install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do - prepare "extension headers", archhdrdir - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.obj", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.pch", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.pdb", :mode => $data_mode) -end -install?(:ext, :comm, :'ext-comm') do - prepare "extension scripts", rubylibdir - install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) - prepare "extension scripts", sitelibdir - prepare "extension scripts", vendorlibdir -end -install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do - hdrdir = rubyhdrdir + "/ruby" - prepare "extension headers", hdrdir - install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) -end - -install?(:doc, :rdoc) do - if $rdocdir - ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") - prepare "rdoc", ridatadir - install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :html) do - if $htmldir - prepare "html-docs", docdir - install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :capi) do - prepare "capi-docs", docdir - install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode -end - prolog_script = <<EOS bindir="#{load_relative ? '${0%/*}' : bindir.gsub(/\"/, '\\\\"')}" EOS -if CONFIG["LIBRUBY_RELATIVE"] != 'yes' and libpathenv = CONFIG["LIBPATHENV"] +if !load_relative and libpathenv = CONFIG["LIBPATHENV"] pathsep = File::PATH_SEPARATOR prolog_script << <<EOS -libdir="#{load_relative ? '$\{bindir%/bin\}/lib' : libdir.gsub(/\"/, '\\\\"')}" +libdir="#{libdir.gsub(/\"/, '\\\\"')}" export #{libpathenv}="$libdir${#{libpathenv}:+#{pathsep}$#{libpathenv}}" EOS end @@ -461,7 +420,7 @@ PROLOG_SCRIPT["cmd"] = <<EOS :""||{ ""=> %q<-*- ruby -*- @"%~dp0#{ruby_install_name}" -x "%~f0" %* @exit /b %ERRORLEVEL% -};{#\n#{prolog_script.gsub(/(?=\n)/, ' #')}>,\n} +};{ #\n#{prolog_script.gsub(/(?=\n)/, ' #')}>,\n} EOS PROLOG_SCRIPT.default = (load_relative || /\s/ =~ bindir) ? <<EOS : PROLOG_SCRIPT["exe"] @@ -473,7 +432,12 @@ _=_\\ =end EOS -installer = Struct.new(:ruby_shebang, :ruby_bin, :ruby_install_name, :stub, :trans) +installer = Struct.new(:ruby_shebang, :ruby_bin, :ruby_install_name, :stub, :trans) do + def transform(name) + RbConfig.expand(trans[name]) + end +end + $script_installer = Class.new(installer) do ruby_shebang = File.join(bindir, ruby_install_name) if File::ALT_SEPARATOR @@ -482,7 +446,7 @@ $script_installer = Class.new(installer) do if trans = CONFIG["program_transform_name"] exp = [] trans.gsub!(/\$\$/, '$') - trans.scan(%r[\G[\s;]*(/(?:\\.|[^/])*/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*)\3((?:(?!\3)(?:\\.|.))*)\3([gi]*)]) do + trans.scan(%r[\G[\s;]*(/(?:\\.|[^/])*+/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*+)\3((?:(?!\3)(?:\\.|.))*+)\3([gi]*)]) do |addr, cmd, sep, pat, rep, opt| addr &&= Regexp.new(addr[/\A\/(.*)\/\z/, 1]) case cmd @@ -518,9 +482,9 @@ $script_installer = Class.new(installer) do end def install(src, cmd) - cmd = cmd.sub(/[^\/]*\z/m) {|n| RbConfig.expand(trans[n])} + cmd = cmd.sub(/[^\/]*\z/m) {|n| transform(n)} - shebang, body = open(src, "rb") do |f| + shebang, body = File.open(src, "rb") do |f| next f.gets, f.read end shebang or raise "empty file - #{src}" @@ -554,183 +518,269 @@ $script_installer = Class.new(installer) do break new(ruby_shebang, ruby_bin, ruby_install_name, nil, trans) end -install?(:local, :comm, :bin, :'bin-comm') do - prepare "command scripts", bindir - - install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| - $script_installer.install(src, cmd) - end -end - -install?(:local, :comm, :lib) do - prepare "library scripts", rubylibdir - noinst = %w[*.txt *.rdoc *.gemspec] - install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) -end - -install?(:local, :comm, :hdr, :'comm-hdr') do - prepare "common headers", rubyhdrdir - - noinst = [] - unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ - noinst << "win32.h" - end - noinst = nil if noinst.empty? - install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.h", :mode => $data_mode) -end - -install?(:local, :comm, :man) do - mdocs = Dir["#{srcdir}/man/*.[1-9]"] - prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} - - case $mantype - when /\.(?:(gz)|bz2)\z/ - compress = $1 ? "gzip" : "bzip2" - suffix = $& - end - mandir = File.join(mandir, "man") - has_goruby = File.exist?(goruby_install_name+exeext) - require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype - mdocs.each do |mdoc| - next unless File.file?(mdoc) and open(mdoc){|fh| fh.read(1) == '.'} - base = File.basename(mdoc) - if base == "goruby.1" - next unless has_goruby - end - - destdir = mandir + (section = mdoc[/\d+$/]) - destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) - destfile = File.join(destdir, "#{destname}.#{section}") - - if /\Adoc\b/ =~ $mantype - if compress - w = open(mdoc) {|f| - stdin = STDIN.dup - STDIN.reopen(f) - begin - destfile << suffix - IO.popen(compress, &:read) - ensure - STDIN.reopen(stdin) - stdin.close - end - } - open_for_install(destfile, $data_mode) {w} - else - install mdoc, destfile, :mode => $data_mode - end - else - class << (w = []) - alias print push +module RbInstall + def self.no_write(options = nil) + u = File.umask(0022) + if $dryrun + fu = ::Object.class_eval do + fu = remove_const(:FileUtils) + const_set(:FileUtils, fu::NoWrite) + fu end - open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} - w = w.join("") - if compress - require 'tmpdir' - Dir.mktmpdir("man") {|d| - dest = File.join(d, File.basename(destfile)) - File.open(dest, "wb") {|f| f.write w} - if system(compress, dest) - w = File.open(dest+suffix, "rb") {|f| f.read} - destfile << suffix - end - } + # RubyGems 3.0.0 or later supports `dir_mode`, but it uses + # `File` method to apply it, not `FileUtils`. + dir_mode = options.delete(:dir_mode) if options + end + yield + ensure + options[:dir_mode] = dir_mode if dir_mode + if fu + ::Object.class_eval do + remove_const(:FileUtils) + const_set(:FileUtils, fu) end - open_for_install(destfile, $data_mode) {w} end + File.umask(u) end -end -module RbInstall module Specs class FileCollector - def initialize(gemspec) + def self.for(srcdir, type, gemspec) + relative_base = (File.dirname(gemspec) if gemspec.include?("/")) + const_get(type.capitalize).new(gemspec, srcdir, relative_base) + end + + attr_reader :gemspec, :srcdir, :relative_base + def initialize(gemspec, srcdir, relative_base) @gemspec = gemspec - @base_dir = File.dirname(gemspec) + @srcdir = srcdir + @relative_base = relative_base end def collect - (ruby_libraries + built_libraries).sort + requirable_features.sort end private - def type - /\/(ext|lib)?\/.*?\z/ =~ @base_dir - $1 + + def features_from_makefile(makefile_path) + makefile = File.read(makefile_path) + + name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + return [] if name.nil? || name.empty? + + feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = feature.sub("$(TARGET)", name) + + target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? + + Array(feature) end - def ruby_libraries - case type - when "ext" - prefix = "#{$extout}/common/" - base = "#{prefix}#{relative_base}" - when "lib" - base = @base_dir - prefix = base.sub(/lib\/.*?\z/, "") + "lib/" + class Ext < self + def requirable_features + # install ext only when it's configured + return [] unless File.exist?(makefile_path) + + ruby_features + ext_features end - if base - Dir.glob("#{base}{.rb,/**/*.rb}").collect do |ruby_source| - remove_prefix(prefix, ruby_source) + private + + def ruby_features + Dir.glob("**/*.rb", base: "#{makefile_dir}/lib") + end + + def ext_features + features_from_makefile(makefile_path) + end + + def makefile_path + if File.exist?("#{makefile_dir}/Makefile") + "#{makefile_dir}/Makefile" + else + # for out-of-place build + "#{$ext_build_dir}/#{relative_base}/Makefile" end - else - [remove_prefix(File.dirname(@gemspec) + '/', @gemspec.gsub(/gemspec/, 'rb'))] + end + + def makefile_dir + "#{root}/#{relative_base}" + end + + def root + File.expand_path($ext_build_dir, srcdir) end end - def built_libraries - case type - when "ext" - prefix = "#{$extout}/#{CONFIG['arch']}/" - base = "#{prefix}#{relative_base}" - dlext = CONFIG['DLEXT'] - Dir.glob("#{base}{.#{dlext},/**/*.#{dlext}}").collect do |built_library| - remove_prefix(prefix, built_library) + class Lib < self + def requirable_features + ruby_features + ext_features + end + + private + + def ruby_features + gemname = File.basename(gemspec, ".gemspec") + base = relative_base || gemname + # for lib/net/net-smtp.gemspec + if m = /.*(?=-(.*)\z)/.match(gemname) + base = File.join(base, *m.to_a.select {|n| !base.include?(n)}) end - when "lib" - [] - else - [] + files = Dir.glob("#{base}{.rb,/**/*.rb}", base: root) + if !relative_base and files.empty? # no files at the toplevel + # pseudo gem like ruby2_keywords + files << "#{gemname}.rb" + end + + case gemname + when "net-http" + files << "net/https.rb" + when "optparse" + files << "optionparser.rb" + end + + files + end + + def ext_features + loaded_gemspec = load_gemspec("#{root}/#{gemspec}") + extension = loaded_gemspec.extensions.first + return [] unless extension + + extconf = File.expand_path(extension, srcdir) + ext_build_dir = File.dirname(extconf) + makefile_path = "#{ext_build_dir}/Makefile" + return [] unless File.exist?(makefile_path) + + features_from_makefile(makefile_path) + end + + def root + "#{srcdir}/lib" end end - def relative_base - /\/#{Regexp.escape(type)}\/(.*?)\z/ =~ @base_dir - $1 + class UnpackedGem < self + def collect + base = @srcdir or return [] + Dir.glob("**/*", File::FNM_DOTMATCH, base: base).select do |n| + next if n == "." + next if File.fnmatch?("*.gemspec", n, File::FNM_DOTMATCH|File::FNM_PATHNAME) + !File.directory?(File.join(base, n)) + end + end end + end + end + + class DirPackage + attr_reader :spec - def remove_prefix(prefix, string) - string.sub(/\A#{Regexp.escape(prefix)}/, "") + attr_accessor :dir_mode + attr_accessor :prog_mode + attr_accessor :data_mode + + def initialize(spec, dir_map = nil) + @spec = spec + @src_dir = File.dirname(@spec.loaded_from) + @dir_map = dir_map + end + + def extract_files(destination_dir, pattern = "*") + return if @src_dir == destination_dir + File.chmod(0700, destination_dir) unless $dryrun + mode = pattern == File.join(spec.bindir, '*') ? prog_mode : data_mode + destdir = without_destdir(destination_dir) + if @dir_map + (dir_map = @dir_map.map {|k, v| Regexp.quote(k) unless k == v}).compact! + dir_map = %r{\A(?:#{dir_map.join('|')})(?=/)} + end + spec.files.each do |f| + next unless File.fnmatch(pattern, f) + src = File.join(@src_dir, dir_map =~ f ? "#{@dir_map[$&]}#{$'}" : f) + dest = File.join(destdir, f) + makedirs(dest[/.*(?=\/)/m]) + install src, dest, :mode => mode end + File.chmod(dir_mode, destination_dir) unless $dryrun end end class UnpackedInstaller < Gem::Installer - module DirPackage - def extract_files(destination_dir, pattern = "*") - path = File.dirname(@gem.path) - return if path == destination_dir - File.chmod(0700, destination_dir) - mode = pattern == "bin/*" ? $script_mode : $data_mode - spec.files.each do |f| - src = File.join(path, f) - dest = File.join(without_destdir(destination_dir), f) - makedirs(dest[/.*(?=\/)/m]) - install src, dest, :mode => mode - end - File.chmod($dir_mode, destination_dir) + # This method is mostly copied from old version of Gem::Installer#install + def install_with_default_gem + verify_gem_home + + # The name and require_paths must be verified first, since it could contain + # ruby code that would be eval'ed in #ensure_loadable_spec + verify_spec + + ensure_loadable_spec + + if options[:install_as_default] + Gem.ensure_default_gem_subdirectories gem_home + else + Gem.ensure_gem_subdirectories gem_home end - end - def initialize(spec, *options) - super(spec.loaded_from, *options) - @package.extend(DirPackage).spec = spec - end + return true if @force - def write_cache_file + ensure_dependencies_met unless @ignore_dependencies + + run_pre_install_hooks + + # Set loaded_from to ensure extension_dir is correct + if @options[:install_as_default] + spec.loaded_from = default_spec_file + else + spec.loaded_from = spec_file + end + + # Completely remove any previous gem files + FileUtils.rm_rf gem_dir + FileUtils.rm_rf spec.extension_dir + + dir_mode = options[:dir_mode] + FileUtils.mkdir_p gem_dir, mode: dir_mode && 0o755 + + if @options[:install_as_default] + extract_bin + write_default_spec + else + extract_files + + build_extensions + write_build_info_file + run_post_build_hooks + end + + generate_bin + generate_plugins + + unless @options[:install_as_default] + write_spec + write_cache_file + end + + File.chmod(dir_mode, gem_dir) if dir_mode + + say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? + + Gem::Specification.add_spec(spec) unless @install_dir + + load_plugin + + run_post_install_hooks + + spec + rescue Errno::EACCES => e + # Permission denied - /path/to/foo + raise Gem::FilePermissionError, e.message.split(" - ").last end - def build_extensions + def write_cache_file end def shebang(bin_file_name) @@ -753,11 +803,6 @@ module RbInstall super end - def generate_bin_script(filename, bindir) - return if same_bin_script?(filename, bindir) - super - end - def same_bin_script?(filename, bindir) path = File.join(bindir, formatted_program_filename(filename)) begin @@ -766,99 +811,398 @@ module RbInstall end false end - end -end -class Gem::Installer - install = instance_method(:install) - define_method(:install) do - spec.post_install_message = nil - begin - u = File.umask(0022) - install.bind(self).call - ensure - File.umask(u) + def write_spec + super unless $dryrun + $installed_list.puts(without_destdir(spec_file)) if $installed_list end - end - generate_bin_script = instance_method(:generate_bin_script) - define_method(:generate_bin_script) do |filename, bindir| - generate_bin_script.bind(self).call(filename, bindir) - File.chmod($script_mode, File.join(bindir, formatted_program_filename(filename))) - end -end + def write_default_spec + super unless $dryrun + $installed_list.puts(without_destdir(default_spec_file)) if $installed_list + end -# :startdoc: + def install + spec.post_install_message = nil + dir_creating(without_destdir(gem_dir)) + RbInstall.no_write(options) { install_with_default_gem } + end -install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do - install_default_gem('lib', srcdir) -end -install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do - install_default_gem('ext', srcdir) + # Now build-ext builds all extensions including bundled gems. + def build_extensions + end + + def generate_bin_script(filename, bindir) + return if same_bin_script?(filename, bindir) + name = formatted_program_filename(filename) + unless $dryrun + super + script = File.join(bindir, name) + File.chmod($script_mode, script) + File.unlink("#{script}.lock") rescue nil + end + $installed_list.puts(File.join(without_destdir(bindir), name)) if $installed_list + end + + def verify_gem_home # :nodoc: + end + + def ensure_writable_dir(dir) + $made_dirs.fetch(d = without_destdir(dir)) do + $made_dirs[d] = true + super unless $dryrun + $installed_list.puts(d+"/") if $installed_list + end + end + + def load_plugin + # Suppress warnings for constant re-assignment + verbose, $VERBOSE = $VERBOSE, nil + super + ensure + $VERBOSE = verbose + end + end end -def load_gemspec(file) +def load_gemspec(file, base = nil, files: nil) file = File.realpath(file) code = File.read(file, encoding: "utf-8:-") - code.gsub!(/`git.*?`/m, '""') - code.gsub!(/%x\[git.*?\]/m, '""') + + code.gsub!(/^ *#.*/, "") + spec_files = files ? files.map(&:dump).join(", ") : "" + code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split(\([^\)]*\))?/m) do + "[" + spec_files + "]" + end \ + or + code.gsub!(/IO\.popen\(.*git.*?\)/) do + "[" + spec_files + "] || itself" + end + spec = eval(code, binding, file) + # for out-of-place build + collected_files = files ? spec.files.concat(files).uniq : spec.files + spec.files = collected_files.map do |f| + if !File.exist?(File.join(base || ".", f)) && f.end_with?(".rb") + "lib/#{f}" + else + f + end + end unless Gem::Specification === spec raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." end - spec.loaded_from = file + spec.loaded_from = base ? File.join(base, File.basename(file)) : file + spec.date = RUBY_RELEASE_DATE + spec end -def install_default_gem(dir, srcdir) +def install_default_gem(dir, srcdir, bindir) gem_dir = Gem.default_dir - directories = Gem.ensure_gem_subdirectories(gem_dir, :mode => $dir_mode) - prepare "default gems from #{dir}", gem_dir, directories - - spec_dir = File.join(gem_dir, directories.grep(/^spec/)[0]) - default_spec_dir = "#{spec_dir}/default" - makedirs(default_spec_dir) - - gems = Dir.glob("#{srcdir}/#{dir}/**/*.gemspec").map {|src| - spec = load_gemspec(src) - file_collector = RbInstall::Specs::FileCollector.new(src) - files = file_collector.collect - next if files.empty? - spec.files = files - spec + install_dir = with_destdir(gem_dir) + prepare "default gems from #{dir}", gem_dir + RbInstall.no_write do + # Record making directories + makedirs(Gem.ensure_default_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) + end + + options = { + :install_dir => with_destdir(gem_dir), + :bin_dir => with_destdir(bindir), + :ignore_dependencies => true, + :dir_mode => $dir_mode, + :data_mode => $data_mode, + :prog_mode => $script_mode, + :wrappers => true, + :format_executable => true, + :install_as_default => true, + } + default_spec_dir = Gem.default_specifications_dir + + base = "#{srcdir}/#{dir}" + gems = Dir.glob("**/*.gemspec", base: base).map {|src| + files = RbInstall::Specs::FileCollector.for(srcdir, dir, src).collect + if files.empty? + next + end + load_gemspec("#{base}/#{src}", files: files) } gems.compact.sort_by(&:name).each do |gemspec| old_gemspecs = Dir[File.join(with_destdir(default_spec_dir), "#{gemspec.name}-*.gemspec")] if old_gemspecs.size > 0 - old_gemspecs.each {|spec| FileUtils.rm spec } + old_gemspecs.each {|spec| rm spec } end full_name = "#{gemspec.name}-#{gemspec.version}" + gemspec.loaded_from = File.join srcdir, gemspec.spec_name + + package = RbInstall::DirPackage.new gemspec, {gemspec.bindir => 'libexec'} + ins = RbInstall::UnpackedInstaller.new(package, options) puts "#{INDENT}#{gemspec.name} #{gemspec.version}" - gemspec_path = File.join(default_spec_dir, "#{full_name}.gemspec") - open_for_install(gemspec_path, $data_mode) do - gemspec.to_ruby + ins.install + end +end + +def mdoc_file?(mdoc) + /^\.Nm / =~ File.read(mdoc, 1024) +end + +# :startdoc: + +install?(:local, :arch, :bin, :'bin-arch') do + prepare "binary commands", (dest = archbindir || bindir) + + def (bins = []).add(name) + push(name) + name + end + + install bins.add(ruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + if rubyw_install_name and !rubyw_install_name.empty? + install bins.add(rubyw_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support + # to load and execute the second .wasm file. Both are required to execute ruby + if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" + install bins.add(ruby_install_name+".wasm"), dest, :mode => $prog_mode, :strip => $strip + end + if File.exist? goruby_install_name+exeext + install bins.add(goruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + if enable_shared and dll != lib + install bins.add(dll), dest, :mode => $prog_mode, :strip => $strip + end + if archbindir + prepare "binary command links", bindir + relpath = Path.relative(archbindir, bindir) + bins.each do |f| + ln_sf(File.join(relpath, f), File.join(bindir, f)) + end + end +end + +install?(:local, :arch, :lib, :'lib-arch') do + prepare "base libraries", libdir + + install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc + install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" + if dll == lib and dll != arc + for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] + ln_sf(dll, File.join(libdir, link)) end + end - unless gemspec.executables.empty? then - bin_dir = File.join(gem_dir, 'gems', full_name, gemspec.bindir) - makedirs(bin_dir) + prepare "arch files", archlibdir + install "rbconfig.rb", archlibdir, :mode => $data_mode + if CONFIG["ARCHFILE"] + for file in CONFIG["ARCHFILE"].split + install file, archlibdir, :mode => $data_mode + end + end +end + +install?(:local, :arch, :data) do + pc = CONFIG["ruby_pc"] + if pc and File.file?(pc) and File.size?(pc) + prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") + install pc, pkgconfigdir, :mode => $data_mode + if (pkgconfig_base = CONFIG["libdir", true]) != libdir + prepare "pkgconfig data link", File.join(pkgconfig_base, "pkgconfig") + ln_sf(File.join("..", Path.relative(pkgconfigdir, pkgconfig_base), pc), + File.join(pkgconfig_base, "pkgconfig", pc)) + end + end +end - gemspec.executables.map {|exec| - install File.join(srcdir, 'libexec', exec), - File.join(bin_dir, exec) - } +install?(:ext, :arch, :'ext-arch') do + prepare "extension objects", archlibdir + noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split + install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) + prepare "extension objects", sitearchlibdir + prepare "extension objects", vendorarchlibdir + if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and + !(extso = extso.gsub(/\\\n/, '').split).empty? + libpathenv = CONFIG["LIBPATHENV"] + dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] + prepare "external libraries", dest + for file in extso + install file, dest, :mode => $prog_mode end end end +install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do + prepare "extension headers", archhdrdir + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:ext, :comm, :'ext-comm') do + prepare "extension scripts", rubylibdir + install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) + prepare "extension scripts", sitelibdir + prepare "extension scripts", vendorlibdir +end + +install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do + hdrdir = rubyhdrdir + "/ruby" + prepare "extension headers", hdrdir + install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:doc, :rdoc) do + if $rdocdir + ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") + prepare "rdoc", ridatadir + install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :html) do + if $htmldir + prepare "html-docs", docdir + install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :capi) do + prepare "capi-docs", docdir + install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode +end + +install?(:local, :comm, :bin, :'bin-comm') do + prepare "command scripts", bindir + + install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| + $script_installer.install(src, cmd) + end +end + +install?(:local, :comm, :lib) do + prepare "library scripts", rubylibdir + noinst = %w[*.txt *.rdoc *.gemspec] + install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) +end + +install?(:local, :comm, :hdr, :'comm-hdr') do + prepare "common headers", rubyhdrdir + + noinst = [] + unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ + noinst << "win32.h" + end + noinst = nil if noinst.empty? + install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) +end + +install?(:local, :comm, :man) do + mdocs = Dir["#{srcdir}/man/*.[1-9]"] + prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} + + mantype, suffix, compress = Compressors.for($mantype) + has_goruby = File.exist?(goruby_install_name+exeext) + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ mantype + mdocs.each do |mdoc| + next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' + base = File.basename(mdoc) + if base == "goruby.1" + next unless has_goruby + end + + destdir = File.join(mandir, "man" + (section = mdoc[/\d+$/])) + destname = $script_installer.transform(base.chomp(".#{section}")) + destfile = File.join(destdir, "#{destname}.#{section}") + + if /\Adoc\b/ =~ mantype or !mdoc_file?(mdoc) + if compress + begin + w = IO.popen(compress, "rb", in: mdoc, &:read) + rescue + else + destfile << suffix + end + end + if w + open_for_install(destfile, $data_mode) {w} + else + install mdoc, destfile, :mode => $data_mode + end + else + class << (w = []) + alias print push + end + File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + w = w.join("") + if compress + begin + w = IO.popen(compress, "r+b") do |f| + Thread.start {f.write w; f.close_write} + f.read + end + rescue + else + destfile << suffix + end + end + open_for_install(destfile, $data_mode) {w} + end + end +end + +install?(:dbg, :nodefault) do + prepare "debugger commands", bindir + prepare "debugger scripts", rubylibdir + conf = MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) + Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| + cmd = $script_installer.transform(File.basename(src, ".in")) + open_for_install(File.join(bindir, cmd), $script_mode) { + RbConfig.expand(File.read(src), conf) + } + end + Dir.glob(File.join(srcdir, "misc/lldb_*")) do |src| + if File.directory?(src) + install_recursive src, File.join(rubylibdir, File.basename(src)) + else + install src, rubylibdir + end + end + install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") + if $debug_symbols + { + ruby_install_name => archbindir || bindir, + rubyw_install_name => archbindir || bindir, + goruby_install_name => archbindir || bindir, + dll => libdir, + }.each do |src, dest| + next if src.empty? + src += $debug_symbols + if File.directory?(src) + install_recursive src, File.join(dest, src) + end + end + end +end + +install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do + install_default_gem('lib', srcdir, bindir) +end + +install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do + install_default_gem('ext', srcdir, bindir) +end + install?(:ext, :comm, :gem, :'bundled-gems') do gem_dir = Gem.default_dir - directories = Gem.ensure_gem_subdirectories(gem_dir, :mode => $dir_mode) - prepare "bundled gems", gem_dir, directories install_dir = with_destdir(gem_dir) + prepare "bundled gems", gem_dir + RbInstall.no_write do + # Record making directories + makedirs(Gem.ensure_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) + end + installed_gems = {} + skipped = {} options = { :install_dir => install_dir, :bin_dir => with_destdir(bindir), @@ -866,56 +1210,92 @@ install?(:ext, :comm, :gem, :'bundled-gems') do :ignore_dependencies => true, :dir_mode => $dir_mode, :data_mode => $data_mode, - :prog_mode => $prog_mode, + :prog_mode => $script_mode, :wrappers => true, :format_executable => true, } - gem_ext_dir = "#$extout/gems/#{CONFIG['arch']}" + extensions_dir = Gem::StubSpecification.gemspec_stub("", gem_dir, gem_dir).extensions_dir - dirs = Gem::Util.glob_files_in_dir "*/", "#{srcdir}/gems" - Gem::Specification.each_gemspec(dirs) do |path| - spec = load_gemspec(path) - next unless spec.platform == Gem::Platform::RUBY - next unless spec.full_name == path[srcdir.size..-1][/\A\/gems\/([^\/]+)/, 1] - spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" - if File.directory?(ext = "#{gem_ext_dir}/#{spec.full_name}") - spec.extensions[0] ||= "-" + specifications_dir = File.join(gem_dir, "specifications") + build_dir = Gem::StubSpecification.gemspec_stub("", ".bundle", ".bundle").extensions_dir + + # We are about to build extensions, and want to configure extensions with the + # newly installed ruby. + Gem.instance_variable_set(:@ruby, with_destdir(File.join(bindir, ruby_install_name))) + # Prevent fake.rb propagation. It conflicts with the natural mkmf configs of + # the newly installed ruby. + ENV.delete('RUBYOPT') + + collector = RbInstall::Specs::FileCollector::UnpackedGem + File.foreach("#{srcdir}/gems/bundled_gems") do |name| + next if /^\s*(?:#|$)/ =~ name + next unless /^(\S+)\s+(\S+).*/ =~ name + gem = $1 + gem_name = "#$1-#$2" + path = [ + # gemspec that removed duplicated dependencies of bundled gems + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec", + # gemspec for C ext gems, It has the original dependencies + # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec", + # original gemspec generated by rubygems + "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + ].find { |gemspec| File.exist?(gemspec) } + if path.nil? + skipped[gem_name] = "gemspec not found" + next + end + base = "#{srcdir}/.bundle/gems/#{gem_name}" + files = collector.new(path, base, nil).collect + files.delete("#{gem}.gemspec") + files.delete("#{gem_name}.gemspec") + spec = load_gemspec(path, base, files: files) + unless spec.platform == Gem::Platform::RUBY + skipped[gem_name] = "not ruby platform (#{spec.platform})" + next + end + unless spec.full_name == gem_name + skipped[gem_name] = "full name unmatch #{spec.full_name}" + next end - ins = RbInstall::UnpackedInstaller.new(spec, options) + # Skip install C ext bundled gem if it is build failed or not found + if !spec.extensions.empty? && !File.exist?("#{build_dir}/#{gem_name}/gem.build_complete") + skipped[gem_name] = "extensions not found or build failed #{spec.full_name}" + next + end + spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + + package = RbInstall::DirPackage.new spec + ins = RbInstall::UnpackedInstaller.new(package, options) puts "#{INDENT}#{spec.name} #{spec.version}" ins.install - File.chmod($data_mode, File.join(install_dir, "specifications", "#{spec.full_name}.gemspec")) - unless spec.extensions.empty? - install_recursive(ext, spec.extension_dir) + install_recursive("#{build_dir}/#{gem_name}", "#{extensions_dir}/#{gem_name}") do |src, dest| + # puts "#{INDENT} #{dest[extensions_dir.size+gem_name.size+2..-1]}" + install src, dest, :mode => (File.executable?(src) ? $prog_mode : $data_mode) end installed_gems[spec.full_name] = true end installed_gems, gems = Dir.glob(srcdir+'/gems/*.gem').partition {|gem| installed_gems.key?(File.basename(gem, '.gem'))} unless installed_gems.empty? + prepare "bundled gem cache", gem_dir+"/cache" install installed_gems, gem_dir+"/cache" end - next if gems.empty? - if defined?(Zlib) - Gem.instance_variable_set(:@ruby, with_destdir(File.join(bindir, ruby_install_name))) - silent = Gem::SilentUI.new + unless gems.empty? + skipped.default = "not found in bundled_gems" + puts "skipped bundled gems:" gems.each do |gem| - inst = Gem::Installer.new(gem, options) - inst.spec.extension_dir = with_destdir(inst.spec.extension_dir) - begin - Gem::DefaultUserInteraction.use_ui(silent) {inst.install} - rescue Gem::InstallError - next - end - gemname = File.basename(gem) - puts "#{INDENT}#{gemname}" - end - # fix directory permissions - # TODO: Gem.install should accept :dir_mode option or something - File.chmod($dir_mode, *Dir.glob(install_dir+"/**/")) - # fix .gemspec permissions - File.chmod($data_mode, *Dir.glob(install_dir+"/specifications/*.gemspec")) - else - puts "skip installing bundled gems because of lacking zlib" + gem = File.basename(gem) + printf " %-31s %s\n", gem, skipped[gem.chomp(".gem")] + end + end +end + +install?('modular-gc') do + if modular_gc_dir = CONFIG['modular_gc_dir'] and !modular_gc_dir.empty? + dlext = CONFIG['DLEXT', true] + modular_gc_dir = File.expand_path(modular_gc_dir, CONFIG['prefix']) + prepare "modular GC library", modular_gc_dir + install Dir.glob("gc/*/librubygc.*.#{dlext}"), modular_gc_dir end end @@ -943,5 +1323,9 @@ installs.each do |block| Dir.chdir(dir) end end +unless installs.empty? or $destdir.empty? + require_relative 'lib/colorize' + puts "Installed under #{Colorize.new.info($destdir)}" +end # vi:set sw=2: diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests new file mode 100644 index 0000000000..4bcb5707a5 --- /dev/null +++ b/tool/rbs_skip_tests @@ -0,0 +1,57 @@ +# Running tests of RBS gem may fail because of various reasons. +# You can skip tests of RBS gem using this file, instead of pushing a new commit to `ruby/rbs` repository. +# +# The most frequently seen reason is the incompatibilities introduced to the unreleased version, including +# +# * Strict argument type check is introduced +# * A required method parameter is added +# * A method/class is removed +# +# Feel free to skip the tests with this file for that case. +# +# Syntax: +# +# $(test-case-name) ` ` $(optional comment) # Skipping single test case +# $(test-class-name) ` ` $(optional comment) # Skipping a test class +# + +## Failed tests because of testing environment + +test_collection_install(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__stdlib_source(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__dependency_no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__rbs_dependency_and__gem_dependency(RBS::CliTest) running tests without Bundler +test_collection_install_frozen(RBS::CliTest) running tests without Bundler +test_collection_install_gemspec(RBS::CliTest) running tests without Bundler +test_collection_update(RBS::CliTest) running tests without Bundler + +NetSingletonTest depending on external resources +NetInstanceTest depending on external resources +TestHTTPRequest depending on external resources +TestSingletonNetHTTPResponse depending on external resources +TestInstanceNetHTTPResponse depending on external resources + +test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` + +# Failing because ObjectSpace.count_nodes has been removed +test_count_nodes(ObjectSpaceTest) + +## Unknown failures + +# NoMethodError: undefined method 'inspect' for an instance of RBS::UnitTest::Convertibles::ToInt +test_compile(RegexpSingletonTest) +test_linear_time?(RegexpSingletonTest) +test_new(RegexpSingletonTest) + +## Failed tests caused by unreleased version of Ruby +test_source_location(MethodInstanceTest) +test_source_location(ProcInstanceTest) +test_source_location(UnboundMethodInstanceTest) + +# Errno::ENOENT: No such file or directory - bundle +test_collection_install__pathname_set(RBS::CliTest) +test_collection_install__set_pathname__manifest(RBS::CliTest) diff --git a/tool/rbs_skip_tests_windows b/tool/rbs_skip_tests_windows new file mode 100644 index 0000000000..db12c69419 --- /dev/null +++ b/tool/rbs_skip_tests_windows @@ -0,0 +1,111 @@ +ARGFTest Failing on Windows + +RactorSingletonTest Hangs up on Windows +RactorInstanceTest Hangs up on Windows + +# NotImplementedError: fileno() function is unimplemented on this machine +test_fileno(DirInstanceTest) +test_fchdir(DirSingletonTest) +test_for_fd(DirSingletonTest) + +# ArgumentError: user root doesn't exist +test_home(DirSingletonTest) + +# NameError: uninitialized constant Etc::CS_PATH +test_confstr(EtcSingletonTest) + +# NameError: uninitialized constant Etc::SC_ARG_MAX +test_sysconf(EtcSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-udw6rx/chmod +test_chmod(FileInstanceTest) +test_chmod(FileInstanceTest) +test_truncate(FileInstanceTest) + +# Errno::EISDIR: Is a directory @ rb_sysopen - C:/a/ruby/ruby/src/gems/src/rbs/test/stdlib +test_directory?(FileSingletonTest) + +# NotImplementedError: lutime() function is unimplemented on this machine +test_lutime(FileSingletonTest) + +# NotImplementedError: mkfifo() function is unimplemented on this machine +test_mkfifo(FileSingletonTest) + +# Returns `nil` on Windows +test_getgrgid(EtcSingletonTest) +test_getgrnam(EtcSingletonTest) +test_getpwnam(EtcSingletonTest) +test_getpwuid(EtcSingletonTest) + +# Returns `false` +test_setgid?(FileSingletonTest) +test_setuid?(FileSingletonTest) +test_sticky?(FileSingletonTest) + +test_world_readable?(FileSingletonTest) # Returns `420` +test_world_readable?(FileStatInstanceTest) # Returns `420` +test_world_writable?(FileSingletonTest) # Returns `nil` +test_dev_major(FileStatInstanceTest) # Returns `nil` +test_dev_minor(FileStatInstanceTest) # Returns `nil` +test_rdev_major(FileStatInstanceTest) # Returns `nil` +test_rdev_minor(FileStatInstanceTest) # Returns `nil` + +# ArgumentError: wrong number of arguments (given -403772944, expected 0+) +test_curry(MethodInstanceTest) + +# ArgumentError: no output encoding given +test_tolocale(KconvSingletonTest) + +# Errno::EINVAL: Invalid argument - : +test_system(KernelInstanceTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_load(OpenSSLConfigSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - +test_parse(OpenSSLConfigSingletonTest) +test_parse_config(OpenSSLConfigSingletonTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_each(OpenSSLConfigTest) +test_lookup_and_set(OpenSSLConfigTest) +test_sections(OpenSSLConfigTest) + +# OpenSSL::SSL::SSLError: SSL_connect returned=1 errno=0 peeraddr=185.199.108.153:443 state=error: certificate verify failed (unable to get local issuer certificate) +test_URI_open(OpenURISingletonTest) + +# ArgumentError: both textmode and binmode specified +test_binwrite(PathnameInstanceTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-delete-test20250813-10156-mb3e9i +test_delete(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-binwrite-test20250813-10156-sh8145 +test_open(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_truncate(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_unlink(PathnameInstanceTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /etc/resolv.conf +test_parse_resolv_conf(ResolvDNSConfigSingletonTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvInstanceTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvSingletonTest) + +# ArgumentError: unsupported signal 'SIGUSR2' +test_trap(SignalSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-mgr4tx +test_create(TempfileSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-hp9nzu +test_initialize(TempfileSingletonTest) +test_new(TempfileSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-f8z9pn/test.gz +test_open(ZlibGzipReaderSingletonTest) + +# Errno::EACCES: Permission denied @ rb_file_s_rename +# D:/a/ruby/ruby/src/lib/rubygems/util/atomic_file_writer.rb:42:in 'File.rename' +test_write_binary(GemSingletonTest) diff --git a/tool/rbuninstall.rb b/tool/rbuninstall.rb index eb324c966e..60f5241a4f 100755 --- a/tool/rbuninstall.rb +++ b/tool/rbuninstall.rb @@ -21,16 +21,35 @@ BEGIN { end $dirs = [] $files = [] + COLUMNS = $tty && (ENV["COLUMNS"]&.to_i || begin require 'io/console/size'; rescue; else IO.console_size&.at(1); end)&.then do |n| + n-1 if n > 1 + end + if COLUMNS + $column = 0 + def message(str = nil) + $stdout.print "\b \b" * $column + if str + if str.size > COLUMNS + str = "..." + str[(-COLUMNS+3)..-1] + end + $stdout.print str + end + $stdout.flush + $column = str&.size || 0 + end + else + alias message puts + end } list = ($_.chomp!('/') ? $dirs : $files) -$_ = File.join($destdir, $_) if $destdir list << $_ END { status = true - $\ = ors = (!$dryrun and $tty) ? "\e[K\r" : "\n" + $\ = nil $files.each do |file| - print "rm #{file}" + message "rm #{file}" unless $dryrun + file = File.join($destdir, file) if $destdir begin File.unlink(file) rescue Errno::ENOENT @@ -44,28 +63,37 @@ END { $dirs.each do |dir| unlink[dir] = true end + nonempty = {} while dir = $dirs.pop - print "rmdir #{dir}" + dir = File.dirname(dir) while File.basename(dir) == '.' + message "rmdir #{dir}" unless $dryrun + realdir = $destdir ? File.join($destdir, dir) : dir begin begin unlink.delete(dir) - Dir.rmdir(dir) + Dir.rmdir(realdir) rescue Errno::ENOTDIR - raise unless File.symlink?(dir) - File.unlink(dir) + raise unless File.symlink?(realdir) + File.unlink(realdir) end - rescue Errno::ENOENT, Errno::ENOTEMPTY + rescue Errno::ENOTEMPTY + nonempty[dir] = true + rescue Errno::ENOENT rescue status = false puts $! else + nonempty.delete(dir) parent = File.dirname(dir) $dirs.push(parent) unless parent == dir or unlink[parent] end end end - $\ = nil - print ors.chomp + message + unless nonempty.empty? + puts "Non empty director#{nonempty.size == 1 ? 'y' : 'ies'}:" + nonempty.each_key {|dir| print " #{dir}\n"} + end exit(status) } diff --git a/tool/rdoc-srcdir b/tool/rdoc-srcdir new file mode 100755 index 0000000000..ecc49b4b2c --- /dev/null +++ b/tool/rdoc-srcdir @@ -0,0 +1,30 @@ +#!ruby -W0 + +%w[tsort rdoc].each do |lib| + path = Dir.glob("#{File.dirname(__dir__)}/.bundle/gems/#{lib}-*").first + $LOAD_PATH.unshift("#{path}/lib") +end +require 'rdoc/rdoc' + +# Make only the output directory relative to the invoked directory. +invoked = Dir.pwd + +# Load options and parse files from srcdir. +Dir.chdir(File.dirname(__dir__)) + +options = RDoc::Options.load_options +options.title = options.title.sub(/Ruby \K.*version/) { + File.read("include/ruby/version.h") + .scan(/^ *# *define +RUBY_API_VERSION_(MAJOR|MINOR) +(\d+)/) + .sort # "MAJOR" < "MINOR", fortunately + .to_h.values.join(".") +} +options.parse ARGV + ["#{invoked}/rbconfig.rb"] + +options.singleton_class.define_method(:finish) do + super() + @op_dir = File.expand_path(@op_dir, invoked) +end + +# Do not hide errors when generating documents of Ruby itself. +RDoc::RDoc.new.document options diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index 245e2a66c0..95a9688cb2 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -10,13 +10,7 @@ require 'optparse' require 'abbrev' require 'pp' require 'shellwords' -begin - require 'readline' -rescue LoadError - module Readline; end -end - -VERSION = '0.0.1' +require 'reline' opts = OptionParser.new target_version = nil @@ -24,10 +18,9 @@ repo_path = nil api_key = nil ssl_verify = true opts.on('-k REDMINE_API_KEY', '--key=REDMINE_API_KEY', 'specify your REDMINE_API_KEY') {|v| api_key = v} -opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 2.1)') {|v| target_version = v} +opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 3.1)') {|v| target_version = v} opts.on('-r RUBY_REPO_PATH', '--repository=RUBY_REPO_PATH', 'specify repository path') {|v| repo_path = v} opts.on('--[no-]ssl-verify', TrueClass, 'use / not use SSL verify') {|v| ssl_verify = v} -opts.version = VERSION opts.parse!(ARGV) http_options = {use_ssl: true} @@ -35,17 +28,17 @@ http_options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify $openuri_options = {} $openuri_options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify -TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (raise 'need to specify TARGET_VERSION') +TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (puts opts.help; raise 'need to specify TARGET_VERSION') RUBY_REPO_PATH = repo_path || ENV['RUBY_REPO_PATH'] BACKPORT_CF_KEY = 'cf_5' STATUS_CLOSE = 5 -REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (raise 'need to specify REDMINE_API_KEY') +REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (puts opts.help; raise 'need to specify REDMINE_API_KEY') REDMINE_BASE = 'https://bugs.ruby-lang.org' @query = { 'f[]' => BACKPORT_CF_KEY, "op[#{BACKPORT_CF_KEY}]" => '~', - "v[#{BACKPORT_CF_KEY}][]" => "#{TARGET_VERSION}: REQUIRED", + "v[#{BACKPORT_CF_KEY}][]" => "\"#{TARGET_VERSION}: REQUIRED\"", 'limit' => 40, 'status_id' => STATUS_CLOSE, 'sort' => 'updated_on' @@ -70,34 +63,34 @@ COLORS = { } class String - def color(fore=nil, back=nil, bold: false, underscore: false) + def color(fore=nil, back=nil, opts={}, bold: false, underscore: false) seq = "" - if bold - seq << "\e[1m" + if bold || opts[:bold] + seq = seq + "\e[1m" end - if underscore - seq << "\e[2m" + if underscore || opts[:underscore] + seq = seq + "\e[2m" end if fore c = COLORS[fore] raise "unknown foreground color #{fore}" unless c - seq << "\e[#{c}m" + seq = seq + "\e[#{c}m" end if back c = COLORS[back] raise "unknown background color #{back}" unless c - seq << "\e[#{c + 10}m" + seq = seq + "\e[#{c + 10}m" end if seq.empty? self else - seq << self << "\e[0m" + seq = seq + self + "\e[0m" end end end class StringScanner - # lx: limit of x (colmns of screen) + # lx: limit of x (columns of screen) # ly: limit of y (rows of screen) def getrows(lx, ly) cp1 = charpos @@ -162,86 +155,13 @@ def more(sio) end end -class << Readline - def readline(prompt = '') - console = IO.console - console.binmode - _, lx = console.winsize - if /mswin|mingw/ =~ RUBY_PLATFORM or /^(?:vt\d\d\d|xterm)/i =~ ENV["TERM"] - cls = "\r\e[2K" - else - cls = "\r" << (" " * lx) - end - cls << "\r" << prompt - console.print prompt - console.flush - line = '' - while true - case c = console.getch - when "\r", "\n" - puts - HISTORY << line - return line - when "\C-?", "\b" # DEL/BS - print "\b \b" if line.chop! - when "\C-u" - print cls - line.clear - when "\C-d" - return nil if line.empty? - line << c - when "\C-p" - HISTORY.pos -= 1 - line = HISTORY.current - print cls - print line - when "\C-n" - HISTORY.pos += 1 - line = HISTORY.current - print cls - print line - else - if c >= " " - print c - line << c - end - end - end - end - - HISTORY = [] - def HISTORY.<<(val) - HISTORY.push(val) - @pos = self.size - self - end - def HISTORY.pos - @pos ||= 0 - end - def HISTORY.pos=(val) - @pos = val - if @pos < 0 - @pos = -1 - elsif @pos >= self.size - @pos = self.size - end - end - def HISTORY.current - @pos ||= 0 - if @pos < 0 || @pos >= self.size - '' - else - self[@pos] - end - end -end unless defined?(Readline.readline) - -def find_svn_log(pattern) - `svn log --xml --stop-on-copy --search="#{pattern}" #{RUBY_REPO_PATH}` +def find_git_log(pattern) + `git #{RUBY_REPO_PATH ? "-C #{RUBY_REPO_PATH.shellescape}" : ""} log --grep="#{pattern}"` end -def find_git_log(pattern) - `git #{RUBY_REPO_PATH ? "-C #{RUBY_REPO_PATH.shellecape}" : ""} log --grep="#{pattern}"` +def has_commit(commit, branch) + base = RUBY_REPO_PATH ? ["-C", RUBY_REPO_PATH.shellescape] : nil + system("git", *base, "merge-base", "--is-ancestor", commit, branch) end def show_last_journal(http, uri) @@ -269,18 +189,14 @@ def backport_command_string @changesets = @changesets.select do |c| next false if c.match(/\A\d{1,6}\z/) # skip SVN revision - # check if the Git revision is included in trunk - begin - uri = URI("#{REDMINE_BASE}/projects/ruby-trunk/repository/git/revisions/#{c}") - uri.read($openuri_options) - true - rescue - false - end + # check if the Git revision is included in master + has_commit(c, "origin/master") + end.sort_by do |changeset| + Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) end @changesets.define_singleton_method(:validated){true} end - " #{merger_path} --ticket=#{@issue} #{@changesets.sort.join(',')}" + "#{merger_path} --ticket=#{@issue} #{@changesets.join(',')}" end def status_char(obj) @@ -295,22 +211,23 @@ end console = IO.console row, = console.winsize @query['limit'] = row - 2 -puts "Backporter #{VERSION}".color(bold: true) + " for #{TARGET_VERSION}" +puts "Redmine Backporter".color(bold: true) + " for Ruby #{TARGET_VERSION}" class CommandSyntaxError < RuntimeError; end commands = { "ls" => proc{|args| raise CommandSyntaxError unless /\A(\d+)?\z/ =~ args - uri = URI(REDMINE_BASE+'/projects/ruby-trunk/issues.json?'+URI.encode_www_form(@query.dup.merge('page' => ($1 ? $1.to_i : 1)))) + uri = URI(REDMINE_BASE+'/projects/ruby-master/issues.json?'+URI.encode_www_form(@query.dup.merge('page' => ($1 ? $1.to_i : 1)))) # puts uri res = JSON(uri.read($openuri_options)) @issues = issues = res["issues"] from = res["offset"] + 1 total = res["total_count"] + closed = issues.count { |x, _| x["status"]["name"] == "Closed" } to = from + issues.size - 1 - puts "#{from}-#{to} / #{total}" + puts "#{from}-#{to} / #{total} (closed: #{closed})" issues.each_with_index do |x, i| - id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]]) + id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]], bold: x["status"]["name"] == "Closed") puts "#{'%2d' % i} #{id} #{x["priority"]["name"][0]} #{status_char(x["status"])} #{x["subject"][0,80]}" end }, @@ -349,7 +266,7 @@ eom i["custom_fields"].each do |x| sio.puts "%-10s: %s" % [x["name"], x["value"]] end - #res["attachements"].each do |x| + #res["attachments"].each do |x| #end sio.puts i["description"] sio.puts @@ -377,12 +294,9 @@ eom "rel" => proc{|args| # this feature requires custom redmine which allows add_related_issue API case args - when /\Ar?(\d+)\z/ # SVN - rev = $1 - uri = URI("#{REDMINE_BASE}/projects/ruby-trunk/repository/trunk/revisions/#{rev}/issues.json") when /\A\h{7,40}\z/ # Git rev = args - uri = URI("#{REDMINE_BASE}/projects/ruby-trunk/repository/git/revisions/#{rev}/issues.json") + uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/git/revisions/#{rev}/issues.json") else raise CommandSyntaxError end @@ -423,9 +337,10 @@ eom }, "done" => proc{|args| - raise CommandSyntaxError unless /\A(\d+)?(?:\s*-- +(.*))?\z/ =~ args - notes = $2 + raise CommandSyntaxError unless /\A(\d+)?(?: *by (\h+))?(?:\s*-- +(.*))?\z/ =~ args + notes = $3 notes.strip! if notes + rev = $2 if $1 i = $1.to_i i = @issues[i]["id"] if @issues && i < @issues.size @@ -436,19 +351,17 @@ eom next end - if system("svn info #{RUBY_REPO_PATH&.shellescape}", %i(out err) => IO::NULL) # SVN - if (log = find_svn_log("##@issue]")) && (/revision="(?<rev>\d+)/ =~ log) - rev = "r#{rev}" - end - else # Git - if log = find_git_log("##@issue]") - /^commit (?<rev>\h{40})$/ =~ log + if rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") + notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." + elsif rev.nil? && (log = find_git_log("##@issue]")) && !(revs = log.scan(/^commit (\h{40})$/).flatten).empty? + commits = revs.map { |rev| "commit:#{rev}" }.join(", ") + if merged_revs = log[/merge revision\(s\) ([^:]+)(?=:)/] + merged_revs.sub!(/\Amerge/, 'merged') + merged_revs.gsub!(/\h{8,40}/, 'commit:\0') + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits} #{merged_revs}." + else + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits}." end - end - if log && rev - str = log[/merge revision\(s\) ([^:]+)(?=:)/] - str.insert(5, "d") - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev} #{str}." if notes str << "\n" str << notes @@ -562,7 +475,7 @@ list = Abbrev.abbrev(commands.keys) @changesets = nil while true begin - l = Readline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " + l = Reline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " rescue Interrupt break end diff --git a/tool/release.sh b/tool/release.sh index 04c361a48a..d467d8f24b 100755 --- a/tool/release.sh +++ b/tool/release.sh @@ -1,38 +1,27 @@ -#!/bin/sh +#!/bin/bash +# Bash version 3.2+ is required for regexp +# Usage: +# tool/release.sh 3.0.0 +# tool/release.sh 3.0.0-rc1 -RUBYDIR=/home/ftp/pub/ruby -EXTS='.tar.gz .tar.bz2 .tar.xz .zip' +EXTS='.tar.gz .tar.xz .zip' +if [[ -n $AWS_ACCESS_KEY_ID ]]; then + AWS_CLI_OPTS="" +else + AWS_CLI_OPTS="--profile ruby" +fi -releases=`ls ruby-*|grep -o 'ruby-[0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\(-\(preview\|rc\|p\)[0-9]\{1,4\}\)\?'|uniq` +ver=$1 +if [[ $ver =~ ^([1-9]\.[0-9])\.([0-9]|[1-9][0-9]|0-(preview[1-9]|rc[1-9]))$ ]]; then + : +else + echo $ver is not valid release version + exit 1 +fi -# check files -for r in $releases -do - echo "checking files for $r..." - for ext in $EXTS - do - if ! [ -f $r$ext ];then - echo "ERROR: $r$ext not found" - exit 1 - fi - done - echo "files are ok" -done - -# version directory -for r in $releases -do - xy=`echo $r|grep -o '[0-9]\.[0-9]'` - preview=`echo $r|grep -o -- '-\(preview\|rc\)'` - dir="${RUBYDIR}/$xy" - echo "$dir" - mkdir -p $dir - for ext in $EXTS - do - cp $r$ext $dir/$r$ext - ln -sf $xy/$r$ext ${RUBYDIR}/$r$ext - if [ x$preview = x ];then - ln -sf $xy/$r$ext ${RUBYDIR}/ruby-$xy-stable$ext - fi - done +short=${BASH_REMATCH[1]} +echo $ver +echo $short +for ext in $EXTS; do + aws $AWS_CLI_OPTS s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext done diff --git a/tool/releng/gen-mail.rb b/tool/releng/gen-mail.rb new file mode 100755 index 0000000000..17fa499d69 --- /dev/null +++ b/tool/releng/gen-mail.rb @@ -0,0 +1,55 @@ +#!/usr/bin/env ruby +require "open-uri" +require "yaml" + +lang = ARGV.shift +unless lang + abort "usage: #$1 {en,ja,release.md} | pbcopy" +end + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end +confirm_w_r_l_o_wd + +releases = YAML.safe_load_file('_data/releases.yml', permitted_classes: [Date]) + +case lang +when "en", "ja" + url = "https://hackmd.io/@naruse/ruby-relnote-#{lang}/download" + src = URI(url).read +else # the path of the Release note in markdown is given + src = File.read(lang) +end +src.gsub!(/[ \t]+$/, "") +src.sub!(/(?<!\n)\z/, "\n") +src.sub!(/^breaks: false\n/, '') + +if /^\{% assign release = site.data.releases \| where: "version", "([^"]+)" \| first %\}/ =~ src + version = $1 +else + abort %[#{url} doesn't include `{% assign release = site.data.releases | where: "version", "<version>" | first %}`] +end + +release = releases.find{|rel|rel['version'] == version} +unless release + abort "#{version} is not found in '_data/releases.yml'" +end + +src.gsub!(/^{% assign .*\n/, '') +src.gsub!(/\{\{(.*?)\}\}/) do + var = $1.strip + case var + when /\Arelease\.(.*)/ + val = release.dig(*$1.split('.')) + raise "invalid variable '#{var}'" unless val + else + raise "unknown variable '#{var}'" + end + val +end +puts src diff --git a/tool/releng/gen-release-note.rb b/tool/releng/gen-release-note.rb new file mode 100755 index 0000000000..5afd11b796 --- /dev/null +++ b/tool/releng/gen-release-note.rb @@ -0,0 +1,36 @@ +#!/usr/bin/env ruby +require 'open-uri' +require 'time' +require 'yaml' + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end +confirm_w_r_l_o_wd + +%w[ + https://hackmd.io/@naruse/ruby-relnote-en/download + https://hackmd.io/@naruse/ruby-relnote-ja/download +].each do |url| + src = URI(url).read + src.gsub!(/[ \t]+$/, "") + src.sub!(/\s+\z/, "\n") + src.sub!(/^breaks: false\n/, '') + if /^\{% assign release = site.data.releases \| where: "version", "([^"]+)" \| first %\}/ =~ src + version = $1 + else + abort %[#{url} doesn't include `{% assign release = site.data.releases | where: "version", "<version>" | first %}`] + end + puts "#{url} -> #{version}" + + + # Write release note article + path = Time.parse(src[/^date: (.*)/, 1]). + strftime("./#{src[/^lang: (\w+)/, 1]}/news/_posts/%Y-%m-%d-ruby-#{version.tr('.', '-')}-released.md") + puts path + File.write(path, src) +end diff --git a/tool/releng/update-www-meta.rb b/tool/releng/update-www-meta.rb new file mode 100755 index 0000000000..0dd5b25631 --- /dev/null +++ b/tool/releng/update-www-meta.rb @@ -0,0 +1,200 @@ +#!/usr/bin/env ruby +require "open-uri" +require "yaml" +require_relative "../ruby-version" + +class Tarball + attr_reader :version, :size, :sha1, :sha256, :sha512 + + def initialize(version, url, size, sha1, sha256, sha512) + @url = url + @size = size + @sha1 = sha1 + @sha256 = sha256 + @sha512 = sha512 + @version = version + @xy = version[/\A\d+\.\d+/] + end + + def gz?; @url.end_with?('.gz'); end + def zip?; @url.end_with?('.zip'); end + def xz?; @url.end_with?('.xz'); end + + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end + + def to_md + <<eom +* <https://cache.ruby-lang.org/pub/ruby/#{@xy}/ruby-#{@version}.#{ext}> + + SIZE: #{@size} bytes + SHA1: #{@sha1} + SHA256: #{@sha256} + SHA512: #{@sha512} +eom + end + + # * /home/naruse/obj/ruby-trunk/tmp/ruby-2.6.0-preview3.tar.gz + # SIZE: 17116009 bytes + # SHA1: 21f62c369661a2ab1b521fd2fa8191a4273e12a1 + # SHA256: 97cea8aa63dfa250ba6902b658a7aa066daf817b22f82b7ee28f44aec7c2e394 + # SHA512: 1e2042324821bb4e110af7067f52891606dcfc71e640c194ab1c117f0b941550e0b3ac36ad3511214ac80c536b9e5cfaf8789eec74cf56971a832ea8fc4e6d94 + def self.parse(wwwdir, version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + teeny = Integer($3) + + uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" + info = YAML.load(URI(uri).read) + if info.size != 1 + raise "unexpected info.yml '#{uri}'" + end + tarballs = [] + info[0]["size"].each_key do |ext| + url = info[0]["url"][ext] + size = info[0]["size"][ext] + sha1 = info[0]["sha1"][ext] + sha256 = info[0]["sha256"][ext] + sha512 = info[0]["sha512"][ext] + tarball = Tarball.new(version, url, size, sha1, sha256, sha512) + tarballs << tarball + end + + if teeny == 0 + # show diff shortstat + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) + rubydir = File.expand_path(File.join(__FILE__, '../../../')) + puts %`git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + files_changed, insertions, deletions = stat.scan(/\d+/) + end + + xy = version[/\A\d+\.\d+/] + #puts "## Download\n\n" + #tarballs.each do |tarball| + # puts tarball.to_md + #end + update_branches_yml(version, xy, wwwdir) + update_downloads_yml(version, xy, wwwdir) + update_releases_yml(version, xy, tarballs, wwwdir, files_changed, insertions, deletions) + end + + def self.update_branches_yml(ver, xy, wwwdir) + filename = "_data/branches.yml" + data = File.read(File.join(wwwdir, filename)) + if data.include?("\n- name: #{xy}\n") + data.sub!(/\n- name: #{Regexp.escape(xy)}\n(?: .*\n)*/) do |node| + unless ver.include?("-") + # assume this is X.Y.0 release + node.sub!(/^ status: preview\n/, " status: normal maintenance\n") + node.sub!(/^ date:\n/, " date: #{Time.now.year}-12-25\n") + end + node + end + else + if ver.include?("-") + status = "preview" + year = nil + else + status = "normal maintenance" + year = Time.now.year + end + entry = <<eom +- name: #{xy} + status: #{status} + date:#{ year && " #{year}-12-25" } + eol_date: + +eom + data.sub!(/(?=^- name)/, entry) + end + File.write(File.join(wwwdir, filename), data) + end + + def self.update_downloads_yml(ver, xy, wwwdir) + filename = "_data/downloads.yml" + data = File.read(File.join(wwwdir, filename)) + + if /^preview:\n\n(?: .*\n)* - #{Regexp.escape(xy)}\./ =~ data + if ver.include?("-") + data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + else + data.sub!(/^ - #{Regexp.escape(xy)}\..*\n/, "") + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + else + unless data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + if ver.include?("-") + data.sub!(/(?<=^preview:\n\n)/, " - #{ver}\n") + else + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + end + end + File.write(File.join(wwwdir, filename), data) + end + + def self.update_releases_yml(ver, xy, ary, wwwdir, files_changed, insertions, deletions) + filename = "_data/releases.yml" + data = File.read(File.join(wwwdir, filename)) + + date = Time.now.utc # use utc to use previous day in midnight + entry = <<eom +- version: #{ver} + tag: #{RubyVersion.tag(ver)} + date: #{date.strftime("%Y-%m-%d")} + post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ + stats: + files_changed: #{files_changed} + insertions: #{insertions} + deletions: #{deletions} + url: + gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz + zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip + xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz + size: + gz: #{ary.find{|x|x.gz? }.size} + zip: #{ary.find{|x|x.zip?}.size} + xz: #{ary.find{|x|x.xz? }.size} + sha1: + gz: #{ary.find{|x|x.gz? }.sha1} + zip: #{ary.find{|x|x.zip?}.sha1} + xz: #{ary.find{|x|x.xz? }.sha1} + sha256: + gz: #{ary.find{|x|x.gz? }.sha256} + zip: #{ary.find{|x|x.zip?}.sha256} + xz: #{ary.find{|x|x.xz? }.sha256} + sha512: + gz: #{ary.find{|x|x.gz? }.sha512} + zip: #{ary.find{|x|x.zip?}.sha512} + xz: #{ary.find{|x|x.xz? }.sha512} +eom + + if data.include?("\n- version: #{ver}\n") + elsif data.sub!(/\n# #{Regexp.escape(xy)} series\n/, "\\&\n#{entry}") + else + data.sub!(/^$/, "\n# #{xy} series\n\n#{entry}") + end + File.write(File.join(wwwdir, filename), data) + end +end + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end + +def main + if ARGV.size != 1 + abort "usage: #$1 <version>" + end + confirm_w_r_l_o_wd + version = ARGV.shift + Tarball.parse(Dir.pwd, version) +end + +main diff --git a/tool/ruby-version.rb b/tool/ruby-version.rb new file mode 100755 index 0000000000..3bbec576e1 --- /dev/null +++ b/tool/ruby-version.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby + +module RubyVersion + def self.tag(version) + major_version = Integer(version.split('.', 2)[0]) + if major_version >= 4 + "v#{version}" + else + "v#{version.tr('.-', '_')}" + end + end + + # Return the previous version to be used for release diff links. + # For a ".0" version, it returns the previous ".0" version. + # For a non-".0" version, it returns the previous teeny version. + def self.previous(version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + major = Integer($1) + minor = Integer($2) + teeny = Integer($3) + + if teeny != 0 + "#{major}.#{minor}.#{teeny-1}" + elsif minor != 0 # && teeny == 0 + "#{major}.#{minor-1}.#{teeny}" + else # minor == 0 && teeny == 0 + case major + when 3 + "2.7.0" + when 4 + "3.4.0" + else + raise "it doesn't know what is the previous version of '#{version}'" + end + end + end +end + +if __FILE__ == $0 + case ARGV[0] + when "tag" + print RubyVersion.tag(ARGV[1]) + when "previous" + print RubyVersion.previous(ARGV[1]) + when "previous-tag" + print RubyVersion.tag(RubyVersion.previous(ARGV[1])) + else + "#{$0}: unexpected command #{ARGV[0].inspect}" + end +end diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb index 25c10947ed..f6c0e39600 100644 --- a/tool/ruby_vm/controllers/application_controller.rb +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -16,10 +15,11 @@ require_relative '../models/typemap' require_relative '../loaders/vm_opts_h' class ApplicationController - def generate i, destdir + def generate i, destdir, basedir path = Pathname.new i dst = destdir ? Pathname.new(destdir).join(i) : Pathname.new(i) - dumper = RubyVM::Dumper.new dst + base = basedir ? Pathname.new(basedir) : Pathname.pwd + dumper = RubyVM::Dumper.new dst, base.expand_path return [path, dumper] end end diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb index 3e2bf2e02c..628cb0428b 100644 --- a/tool/ruby_vm/helpers/c_escape.rb +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -17,7 +16,10 @@ module RubyVM::CEscape # generate comment, with escaps. def commentify str - return "/* #{str.strip.b.gsub '*/', '*\\/'} */" + unless str = str.dump[/\A"\K.*(?="\z)/] + raise Encoding::CompatibilityError, "must be ASCII-compatible (#{str.encoding})" + end + return "/* #{str.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" end # Mimic gensym of CL. @@ -46,7 +48,7 @@ module RubyVM::CEscape # I believe this is the fastest implementation done in pure-ruby. # Constants cached, gsub skips block evaluation, string literal optimized. buf = str.b - buf.gsub! %r/./n, RString2CStr + buf.gsub! %r/./nm, RString2CStr return %'"#{buf}"' end @@ -60,7 +62,7 @@ module RubyVM::CEscape "\x18"=>"\\x18", "\x19"=>"\\x19", "\x1A"=>"\\x1a", "\e"=>"\\x1b", "\x1C"=>"\\x1c", "\x1D"=>"\\x1d", "\x1E"=>"\\x1e", "\x1F"=>"\\x1f", " "=> " ", "!"=> "!", "\""=> "\\\"", "#"=> "#", - "$"=> "$", "%"=> "%", "&"=> "&", "'"=> "\\'", + "$"=> "$", "%"=> "%", "&"=> "&", "'"=> "'", "("=> "(", ")"=> ")", "*"=> "*", "+"=> "+", ","=> ",", "-"=> "-", "."=> ".", "/"=> "/", "0"=> "0", "1"=> "1", "2"=> "2", "3"=> "3", diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb index 00c301c01c..f0758dd44f 100644 --- a/tool/ruby_vm/helpers/dumper.rb +++ b/tool/ruby_vm/helpers/dumper.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -25,18 +24,15 @@ class RubyVM::Dumper end def new_erb spec - path = Pathname.new(__FILE__).relative_path_from(Pathname.pwd).dirname + path = Pathname.new(__FILE__) + path = (path.relative_path_from(Pathname.pwd) rescue path).dirname path += '../views' - path += spec - src = path.read mode: 'rt:utf-8:utf-8' + path += Pathname.pwd.join(spec).expand_path.to_s.sub("#{@base}/", '') + src = path.expand_path.read mode: 'rt:utf-8:utf-8' rescue Errno::ENOENT raise "don't know how to generate #{path}" else - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%-') - else - erb = ERB.new(src, nil, '%-') - end + erb = ERB.new(src, trim_mode: '%-') erb.filename = path.to_path return erb end @@ -84,10 +80,11 @@ class RubyVM::Dumper . join end - def initialize dst + def initialize dst, base @erb = {} @empty = new_binding @file = cstr dst.to_path + @base = base end def render partial, opts = { :locals => {} } diff --git a/tool/ruby_vm/helpers/scanner.rb b/tool/ruby_vm/helpers/scanner.rb index 593271a415..8998abb2d3 100644 --- a/tool/ruby_vm/helpers/scanner.rb +++ b/tool/ruby_vm/helpers/scanner.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -20,7 +19,8 @@ class RubyVM::Scanner attr_reader :__LINE__ def initialize path - src = Pathname.new(__FILE__).relative_path_from(Pathname.pwd).dirname + src = Pathname.new(__FILE__) + src = (src.relative_path_from(Pathname.pwd) rescue src).dirname src += path @__LINE__ = 1 @__FILE__ = src.to_path diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb index a7d27ad06a..d45d0ba83c 100644 --- a/tool/ruby_vm/loaders/insns_def.rb +++ b/tool/ruby_vm/loaders/insns_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -11,6 +10,7 @@ # details. require_relative '../helpers/scanner' +require_relative './vm_opts_h' json = [] scanner = RubyVM::Scanner.new '../../../insns.def' @@ -20,20 +20,20 @@ grammar = %r' (?<keyword> typedef | extern | static | auto | register | struct | union | enum ){0} (?<C> (?: \g<block> | [^{}]+ )* ){0} - (?<block> \{ \g<ws>* ^ \g<C> $ \g<ws>* \} ){0} + (?<block> \{ \g<ws>* \g<C> \g<ws>* \} ){0} (?<ws> \g<comment> | \s ){0} (?<ident> [_a-zA-Z] [0-9_a-zA-Z]* ){0} (?<type> (?: \g<keyword> \g<ws>+ )* \g<ident> ){0} - (?<arg> \g<type> \g<ws>+ \g<ident> | \.\.\. ){0} + (?<arg> \g<type> \g<ws>+ \g<ident> ){0} (?<argv> (?# empty ) | void | - \g<arg> (?: \g<ws>* , \g<ws>* \g<arg> \g<ws>* )* ){0} + (?: \.\.\. | \g<arg>) (?: \g<ws>* , \g<ws>* \g<arg> \g<ws>* )* ){0} (?<pragma> \g<ws>* // \s* attr \g<ws>+ (?<pragma:type> \g<type> ) \g<ws>+ (?<pragma:name> \g<ident> ) \g<ws>* = \g<ws>* (?<pragma:expr> .+?; ) \g<ws>* ){0} - (?<insn> DEFINE_INSN \g<ws>+ + (?<insn> DEFINE_INSN(_IF\((?<insn:if>\w+)\))? \g<ws>+ (?<insn:name> \g<ident> ) \g<ws>* [(] \g<ws>* (?<insn:opes> \g<argv> ) \g<ws>* [)] \g<ws>* [(] \g<ws>* (?<insn:pops> \g<argv> ) \g<ws>* [)] \g<ws>* @@ -52,9 +52,14 @@ until scanner.eos? do l1 = scanner.scan!(/\G#{grammar}\g<insn>/o) name = scanner["insn:name"] + opt = scanner["insn:if"] ope = split.(scanner["insn:opes"]) pop = split.(scanner["insn:pops"]) ret = split.(scanner["insn:rets"]) + if ope.include?("...") + raise sprintf("parse error at %s:%d:%s: operands cannot be variadic", + scanner.__FILE__, scanner.__LINE__, name) + end attrs = [] while l2 = scanner.scan(/\G#{grammar}\g<pragma>/o) do @@ -67,21 +72,23 @@ until scanner.eos? do end l3 = scanner.scan!(/\G#{grammar}\g<block>/o) - json << { - name: name, - location: [path, l1], - signature: { + if opt.nil? || RubyVM::VmOptsH[opt] + json << { name: name, - ope: ope, - pop: pop, - ret: ret, - }, - attributes: attrs, - expr: { - location: [path, l3], - expr: scanner["block"], - }, - } + location: [path, l1], + signature: { + name: name, + ope: ope, + pop: pop, + ret: ret, + }, + attributes: attrs, + expr: { + location: [path, l3], + expr: scanner["block"], + }, + } + end end RubyVM::InsnsDef = json diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb index aa6fd79e79..0750f1823a 100644 --- a/tool/ruby_vm/loaders/opt_insn_unif_def.rb +++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb index 29aef8a325..e08509a433 100644 --- a/tool/ruby_vm/loaders/opt_operand_def.rb +++ b/tool/ruby_vm/loaders/opt_operand_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb index 3f05c270ee..d626ea0296 100644 --- a/tool/ruby_vm/loaders/vm_opts_h.rb +++ b/tool/ruby_vm/loaders/vm_opts_h.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index cc16a5f898..177b701b92 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -21,6 +20,13 @@ class RubyVM::Attribute @key = opts[:name] @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] @type = opts[:type] + @ope_decls = @insn.operands.map do |operand| + decl = operand[:decl] + if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA' + decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci') + end + decl + end end def name @@ -32,22 +38,20 @@ class RubyVM::Attribute end def declaration - opes = @insn.opes - if opes.empty? + if @ope_decls.empty? argv = "void" else - argv = opes.map {|o| o[:decl] }.join(', ') + argv = @ope_decls.join(', ') end sprintf '%s %s(%s)', @type, name, argv end def definition - opes = @insn.opes - if opes.empty? + if @ope_decls.empty? argv = "void" else - argv = opes.map {|o| "MAYBE_UNUSED(#{o[:decl]})" }.join(",\n ") - argv = "\n #{argv}\n" if opes.size > 1 + argv = @ope_decls.map {|decl| "MAYBE_UNUSED(#{decl})" }.join(",\n ") + argv = "\n #{argv}\n" if @ope_decls.size > 1 end sprintf "%s\n%s(%s)", @type, name, argv end diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instruction.rb index e0fac5ff91..f87dd74179 100755..100644 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -15,8 +14,8 @@ require_relative 'c_expr' require_relative 'typemap' require_relative 'attribute' -class RubyVM::BareInstructions - attr_reader :template, :name, :opes, :pops, :rets, :decls, :expr +class RubyVM::BareInstruction + attr_reader :template, :name, :operands, :pops, :rets, :decls, :expr def initialize opts = {} @template = opts[:template] @@ -24,7 +23,7 @@ class RubyVM::BareInstructions @loc = opts[:location] @sig = opts[:signature] @expr = RubyVM::CExpr.new opts[:expr] - @opes = typesplit @sig[:ope] + @operands = typesplit @sig[:ope] @pops = typesplit @sig[:pop].reject {|i| i == '...' } @rets = typesplit @sig[:ret].reject {|i| i == '...' } @attrs = opts[:attributes].map {|i| @@ -33,6 +32,7 @@ class RubyVM::BareInstructions h[a.key] = a } @attrs_orig = @attrs.dup + check_attribute_consistency predefine_attributes end @@ -50,7 +50,7 @@ class RubyVM::BareInstructions def call_attribute name return sprintf 'attr_%s_%s(%s)', name, @name, \ - @opes.map {|i| i[:name] }.compact.join(', ') + @operands.map {|i| i[:name] }.compact.join(', ') end def has_attribute? k @@ -64,7 +64,7 @@ class RubyVM::BareInstructions end def width - return 1 + opes.size + return 1 + operands.size end def declarations @@ -97,7 +97,7 @@ class RubyVM::BareInstructions end def operands_info - opes.map {|o| + operands.map {|o| c, _ = RubyVM::Typemap.fetch o[:type] next c }.join @@ -107,10 +107,6 @@ class RubyVM::BareInstructions /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr end - def always_leaf? - @attrs.fetch('leaf').expr.expr == 'true;' - end - def handle_canary stmt # Stack canary is basically a good thing that we want to add, however: # @@ -132,15 +128,39 @@ class RubyVM::BareInstructions end def has_ope? var - return @opes.any? {|i| i[:name] == var[:name] } + return @operands.any? {|i| i[:name] == var[:name] } end def has_pop? var return @pops.any? {|i| i[:name] == var[:name] } end + def use_call_data? + @use_call_data ||= + @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } + end + + def zjit_profile? + @attrs.fetch('zjit_profile').expr.expr != 'false;' + end + private + def check_attribute_consistency + if has_attribute?('sp_inc') \ + && use_call_data? \ + && !has_attribute?('comptime_sp_inc') + # As the call cache caches information that can only be obtained at + # runtime, we do not need it when compiling from AST to bytecode. This + # attribute defines an expression that computes the stack pointer + # increase based on just the call info to avoid reserving space for the + # call cache at compile time. In the expression, all call data operands + # are mapped to their call info counterpart. Additionally, all mentions + # of `cd` in the operand name are replaced with `ci`. + raise "Please define attribute `comptime_sp_inc` for `#{@name}`" + end + end + def generate_attribute t, k, v @attrs[k] ||= RubyVM::Attribute.new \ insn: self, \ @@ -155,18 +175,19 @@ class RubyVM::BareInstructions # Beware: order matters here because some attribute depends another. generate_attribute 'const char*', 'name', "insn_name(#{bin})" generate_attribute 'enum ruby_vminsn_type', 'bin', bin - generate_attribute 'rb_num_t', 'open', opes.size + generate_attribute 'rb_num_t', 'open', operands.size generate_attribute 'rb_num_t', 'popn', pops.size generate_attribute 'rb_num_t', 'retn', rets.size generate_attribute 'rb_num_t', 'width', width generate_attribute 'rb_snum_t', 'sp_inc', rets.size - pops.size generate_attribute 'bool', 'handles_sp', default_definition_of_handles_sp generate_attribute 'bool', 'leaf', default_definition_of_leaf + generate_attribute 'bool', 'zjit_profile', false end def default_definition_of_handles_sp # Insn with ISEQ should yield it; can handle sp. - return opes.any? {|o| o[:type] == 'ISEQ' } + return operands.any? {|o| o[:type] == 'ISEQ' } end def default_definition_of_leaf @@ -203,13 +224,13 @@ class RubyVM::BareInstructions new h.merge(:template => h) } - def self.fetch name + def self.find(name) @instances.find do |insn| insn.name == name end or raise IndexError, "instruction not found: #{name}" end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb index 073112f545..095ff4f1d9 100644 --- a/tool/ruby_vm/models/c_expr.rb +++ b/tool/ruby_vm/models/c_expr.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -36,6 +35,10 @@ class RubyVM::CExpr end def inspect - sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + if @__LINE__ + sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + else + sprintf "#<%s %s>", @__FILE__, @expr + end end end diff --git a/tool/ruby_vm/models/instructions.rb b/tool/ruby_vm/models/instructions.rb index 1198c7a4a6..7be7064b14 100644 --- a/tool/ruby_vm/models/instructions.rb +++ b/tool/ruby_vm/models/instructions.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -10,13 +9,15 @@ # conditions mentioned in the file COPYING are met. Consult the file for # details. -require_relative 'bare_instructions' -require_relative 'operands_unifications' -require_relative 'instructions_unifications' +require_relative 'bare_instruction' +require_relative 'operands_unification' +require_relative 'instructions_unification' +require_relative 'trace_instruction' +require_relative 'zjit_instruction' -RubyVM::Instructions = RubyVM::BareInstructions.to_a + \ - RubyVM::OperandsUnifications.to_a + \ - RubyVM::InstructionsUnifications.to_a - -require_relative 'trace_instructions' +RubyVM::Instructions = RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all + + RubyVM::TraceInstruction.all + + RubyVM::ZJITInstruction.all RubyVM::Instructions.freeze diff --git a/tool/ruby_vm/models/instructions_unifications.rb b/tool/ruby_vm/models/instructions_unification.rb index 214ba5fcc2..5c798e6d54 100644 --- a/tool/ruby_vm/models/instructions_unifications.rb +++ b/tool/ruby_vm/models/instructions_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,9 +11,9 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_insn_unif_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::InstructionsUnifications +class RubyVM::InstructionsUnification include RubyVM::CEscape attr_reader :name @@ -23,7 +22,7 @@ class RubyVM::InstructionsUnifications @location = opts[:location] @name = namegen opts[:signature] @series = opts[:signature].map do |i| - RubyVM::BareInstructions.fetch i # Misshit is fatal + RubyVM::BareInstruction.find(i) # Misshit is fatal end end @@ -37,7 +36,7 @@ class RubyVM::InstructionsUnifications new h end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/operands_unifications.rb b/tool/ruby_vm/models/operands_unification.rb index ee4e3a695d..ce118648ca 100644 --- a/tool/ruby_vm/models/operands_unifications.rb +++ b/tool/ruby_vm/models/operands_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,16 +11,16 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_operand_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::OperandsUnifications < RubyVM::BareInstructions +class RubyVM::OperandsUnification < RubyVM::BareInstruction include RubyVM::CEscape attr_reader :preamble, :original, :spec def initialize opts = {} name = opts[:signature][0] - @original = RubyVM::BareInstructions.fetch name + @original = RubyVM::BareInstruction.find(name) template = @original.template parts = compose opts[:location], opts[:signature], template[:signature] json = template.dup @@ -38,8 +37,8 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions end def operand_shift_of var - before = @original.opes.find_index var - after = @opes.find_index var + before = @original.operands.find_index var + after = @operands.find_index var raise "no #{var} for #{@name}" unless before and after return before - after end @@ -50,7 +49,7 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions case val when '*' then next nil else - type = @original.opes[i][:type] + type = @original.operands[i][:type] expr = RubyVM::Typemap.typecast_to_VALUE type, val next "#{ptr}[#{i}] == #{expr}" end @@ -85,7 +84,7 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions def compose location, spec, template name = namegen spec *, argv = *spec - opes = @original.opes + opes = @original.operands if opes.size != argv.size raise sprintf("operand size mismatch for %s (%s's: %d, given: %d)", name, template[:name], opes.size, argv.size) @@ -130,12 +129,12 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions new h end - def self.to_a + def self.all @instances end def self.each_group - to_a.group_by(&:original).each_pair do |k, v| + all.group_by(&:original).each_pair do |k, v| yield k, v end end diff --git a/tool/ruby_vm/models/trace_instructions.rb b/tool/ruby_vm/models/trace_instruction.rb index 4ed4c8cb42..6a3ad53c44 100644 --- a/tool/ruby_vm/models/trace_instructions.rb +++ b/tool/ruby_vm/models/trace_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -11,9 +10,9 @@ # details. require_relative '../helpers/c_escape' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::TraceInstructions +class RubyVM::TraceInstruction include RubyVM::CEscape attr_reader :name @@ -61,11 +60,11 @@ class RubyVM::TraceInstructions private - @instances = RubyVM::Instructions.map {|i| new i } + @instances = (RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all).map {|i| new(i) } - def self.to_a + def self.all @instances end - - RubyVM::Instructions.push(*to_a) end diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb index 65ddfea41d..68ef5a41a5 100644 --- a/tool/ruby_vm/models/typemap.rb +++ b/tool/ruby_vm/models/typemap.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,11 +11,11 @@ RubyVM::Typemap = { "..." => %w[. TS_VARIABLE], - "CALL_CACHE" => %w[E TS_CALLCACHE], - "CALL_INFO" => %w[C TS_CALLINFO], + "CALL_DATA" => %w[C TS_CALLDATA], "CDHASH" => %w[H TS_CDHASH], - "GENTRY" => %w[G TS_GENTRY], "IC" => %w[K TS_IC], + "IVC" => %w[A TS_IVC], + "ICVARC" => %w[J TS_ICVARC], "ID" => %w[I TS_ID], "ISE" => %w[T TS_ISE], "ISEQ" => %w[S TS_ISEQ], @@ -25,6 +24,7 @@ RubyVM::Typemap = { "lindex_t" => %w[L TS_LINDEX], "rb_insn_func_t" => %w[F TS_FUNCPTR], "rb_num_t" => %w[N TS_NUM], + "RB_BUILTIN" => %w[R TS_BUILTIN], } # :FIXME: should this method be here? diff --git a/tool/ruby_vm/models/zjit_instruction.rb b/tool/ruby_vm/models/zjit_instruction.rb new file mode 100644 index 0000000000..04764e4c61 --- /dev/null +++ b/tool/ruby_vm/models/zjit_instruction.rb @@ -0,0 +1,56 @@ +require_relative '../helpers/c_escape' +require_relative 'bare_instruction' + +# Profile YARV instructions to optimize code generated by ZJIT +class RubyVM::ZJITInstruction + include RubyVM::CEscape + + attr_reader :name + + def initialize(orig) + @orig = orig + @name = as_tr_cpp "zjit @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute?(*) + return false + end + + @instances = RubyVM::BareInstruction.all.filter(&:zjit_profile?).map {|i| new(i) } + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb index 8325dd364f..ad8603b1a8 100644 --- a/tool/ruby_vm/scripts/insns2vm.rb +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -15,10 +14,10 @@ require_relative '../controllers/application_controller.rb' module RubyVM::Insns2VM def self.router argv - options = { destdir: nil } + options = { destdir: nil, basedir: nil } targets = generate_parser(options).parse argv return targets.map do |i| - next ApplicationController.new.generate i, options[:destdir] + next ApplicationController.new.generate i, options[:destdir], options[:basedir] end end @@ -84,6 +83,14 @@ module RubyVM::Insns2VM options[:destdir] = dir end + this.on "--basedir=DIR", <<-'begin' do |dir| + Change the base directory from the current working directory + to the given path. Used for searching the source template. + begin + raise "directory was not found in '#{dir}'" unless Dir.exist?(dir) + options[:basedir] = dir + end + this.on "-V", "--[no-]verbose", <<-'end' Please let us ignore this and be modest. end diff --git a/tool/ruby_vm/tests/.gitkeep b/tool/ruby_vm/tests/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 --- a/tool/ruby_vm/tests/.gitkeep +++ /dev/null diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb new file mode 100644 index 0000000000..8bb28db1c1 --- /dev/null +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -0,0 +1,71 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%# +% +% stack_increase = proc do |i| +% if i.has_attribute?('sp_inc') +% '-127' +% else +% sprintf("%4d", i.rets.size - i.pops.size) +% end +% end +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); +PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); + +rb_snum_t +comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) +{ + static const signed char t[] = { +% insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#endif + }; + signed char c = t[insn]; + + ASSERT_VM_INSTRUCTION_SIZE(t); + if (c != -127) { + return c; + } + else switch(insn) { + default: + UNREACHABLE; +% RubyVM::Instructions.each do |i| +% next unless i.has_attribute?('sp_inc') +% attr_function = +% if i.has_attribute?('comptime_sp_inc') +% "attr_comptime_sp_inc_#{i.name}" +% else +% "attr_sp_inc_#{i.name}" +% end + case <%= i.bin %>: + return <%= attr_function %>(<%= + i.operands.map.with_index do |v, j| + if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc') + v = v.dup + v[:type] = 'CALL_INFO' + end + i.cast_from_VALUE v, "opes[#{j}]" + end.join(", ") + %>); +% end + } +} + +int +comptime_insn_stack_increase(int depth, int insn, const VALUE *opes) +{ + enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn; + return depth + (int)comptime_insn_stack_increase_dispatch(itype, opes); +} diff --git a/tool/ruby_vm/views/_insn_entry.erb b/tool/ruby_vm/views/_insn_entry.erb index cdadd93abc..6ec33461c4 100644 --- a/tool/ruby_vm/views/_insn_entry.erb +++ b/tool/ruby_vm/views/_insn_entry.erb @@ -20,11 +20,11 @@ INSN_ENTRY(<%= insn.name %>) <%= render_c_expr konst -%> % end % -% insn.opes.each_with_index do |ope, i| +% insn.operands.each_with_index do |ope, i| <%= ope[:decl] %> = (<%= ope[:type] %>)GET_OPERAND(<%= i + 1 %>); % end # define INSN_ATTR(x) <%= insn.call_attribute(' ## x ## ') %> - bool leaf = INSN_ATTR(leaf); + const bool MAYBE_UNUSED(leaf) = INSN_ATTR(leaf); % insn.pops.reverse_each.with_index.reverse_each do |pop, i| <%= pop[:decl] %> = <%= insn.cast_from_VALUE pop, "TOPN(#{i})"%>; % end @@ -35,13 +35,13 @@ INSN_ENTRY(<%= insn.name %>) % end /* ### Instruction preambles. ### */ - if (! leaf) ADD_PC(INSN_ATTR(width)); + ADD_PC(INSN_ATTR(width)); % if insn.handles_sp? POPN(INSN_ATTR(popn)); % end -<%= insn.handle_canary "SETUP_CANARY()" -%> +<%= insn.handle_canary "SETUP_CANARY(leaf)" -%> COLLECT_USAGE_INSN(INSN_ATTR(bin)); -% insn.opes.each_with_index do |ope, i| +% insn.operands.each_with_index do |ope, i| COLLECT_USAGE_OPERAND(INSN_ATTR(bin), <%= i %>, <%= ope[:name] %>); % end % unless body.empty? @@ -55,7 +55,7 @@ INSN_ENTRY(<%= insn.name %>) /* ### Instruction trailers. ### */ CHECK_VM_STACK_OVERFLOW_FOR_INSN(VM_REG_CFP, INSN_ATTR(retn)); -<%= insn.handle_canary "CHECK_CANARY()" -%> +<%= insn.handle_canary "CHECK_CANARY(leaf, INSN_ATTR(bin))" -%> % if insn.handles_sp? % insn.rets.reverse_each do |ret| PUSH(<%= insn.cast_to_VALUE ret %>); @@ -64,9 +64,10 @@ INSN_ENTRY(<%= insn.name %>) INC_SP(INSN_ATTR(sp_inc)); % insn.rets.reverse_each.with_index do |ret, i| TOPN(<%= i %>) = <%= insn.cast_to_VALUE ret %>; + VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_NONE)); + VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_MOVED)); % end % end - if (leaf) ADD_PC(INSN_ATTR(width)); # undef INSN_ATTR /* ### Leave the instruction. ### */ diff --git a/tool/ruby_vm/views/_insn_leaf_info.erb b/tool/ruby_vm/views/_insn_leaf_info.erb new file mode 100644 index 0000000000..f30366ffda --- /dev/null +++ b/tool/ruby_vm/views/_insn_leaf_info.erb @@ -0,0 +1,18 @@ +MAYBE_UNUSED(static bool insn_leaf(int insn, const VALUE *opes)); +static bool +insn_leaf(int insn, const VALUE *opes) +{ + switch (insn) { +% RubyVM::Instructions.each do |insn| +% next if insn.is_a?(RubyVM::TraceInstruction) || insn.is_a?(RubyVM::ZJITInstruction) + case <%= insn.bin %>: + return attr_leaf_<%= insn.name %>(<%= + insn.operands.map.with_index do |ope, i| + "(#{ope[:type]})opes[#{i}]" + end.join(', ') + %>); +% end + default: + return false; + } +} diff --git a/tool/ruby_vm/views/_insn_len_info.erb b/tool/ruby_vm/views/_insn_len_info.erb index b292f42fb5..b29a405918 100644 --- a/tool/ruby_vm/views/_insn_len_info.erb +++ b/tool/ruby_vm/views/_insn_len_info.erb @@ -5,17 +5,32 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% CONSTFUNC(MAYBE_UNUSED(static int insn_len(VALUE insn))); +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const uint8_t rb_vm_insn_len_info[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const uint8_t rb_vm_insn_len_info[] = { +% insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#endif +}; + +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_len_info); +#endif + int insn_len(VALUE i) { - static const char t[] = { -% RubyVM::Instructions.each_slice 23 do |a| - <%= a.map(&:width).join(', ') -%>, -% end - }; - - ASSERT_VM_INSTRUCTION_SIZE(t); - return t[i]; + return rb_vm_insn_len_info[i]; } diff --git a/tool/ruby_vm/views/_insn_name_info.erb b/tool/ruby_vm/views/_insn_name_info.erb index 767346093e..2862908631 100644 --- a/tool/ruby_vm/views/_insn_name_info.erb +++ b/tool/ruby_vm/views/_insn_name_info.erb @@ -6,33 +6,54 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.name } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% next_offset = 0 +% name_offset = proc do |i| +% offset = sprintf("%4d", next_offset) +% next_offset += i.name.length + 1 # insn.name + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_name(VALUE insn))); +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const int rb_vm_max_insn_name_size; +extern const char rb_vm_insn_name_base[]; +extern const unsigned short rb_vm_insn_name_offset[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + #ifdef RUBY_VM_INSNS_INFO -const int rb_vm_max_insn_name_size = <%= a.map(&:size).max %>; -#endif +%# "trace_" is longer than "zjit_", so USE_ZJIT doesn't impact the max name size. +const int rb_vm_max_insn_name_size = <%= RubyVM::Instructions.map { |i| i.name.size }.max %>; -const char * -insn_name(VALUE i) -{ - static const char x[] = -% a.each do |i| - <%=cstr i%> "\0" +const char rb_vm_insn_name_base[] = +% insns.each do |i| + <%= cstr i.name %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each do |i| + <%= cstr i.name %> "\0" % end - ; +#endif + ; - static const unsigned short y[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%4d", i) }.join(', ') %>, +const unsigned short rb_vm_insn_name_offset[] = { +% insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, % end - }; +#endif +}; - ASSERT_VM_INSTRUCTION_SIZE(y); +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_name_offset); +#endif - return &x[y[i]]; +const char * +insn_name(VALUE i) +{ + return &rb_vm_insn_name_base[rb_vm_insn_name_offset[i]]; } diff --git a/tool/ruby_vm/views/_insn_operand_info.erb b/tool/ruby_vm/views/_insn_operand_info.erb index 69d361521a..410869fcd3 100644 --- a/tool/ruby_vm/views/_insn_operand_info.erb +++ b/tool/ruby_vm/views/_insn_operand_info.erb @@ -6,32 +6,55 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.operands_info } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% operands_info = proc { |i| sprintf("%-6s", cstr(i.operands_info)) } +% +% next_offset = 0 +% op_offset = proc do |i| +% offset = sprintf("%3d", next_offset) +% next_offset += i.operands_info.length + 1 # insn.operands_info + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_op_types(VALUE insn))); CONSTFUNC(MAYBE_UNUSED(static int insn_op_type(VALUE insn, long pos))); -const char * -insn_op_types(VALUE i) -{ - static const char x[] = -% a.each_slice 5 do |d| - <%= d.map {|i| sprintf("%-6s", cstr(i)) }.join(' "\0" ') %> "\0" +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const char rb_vm_insn_op_base[]; +extern const unsigned short rb_vm_insn_op_offset[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const char rb_vm_insn_op_base[] = +% insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" % end - ; +#endif + ; - static const unsigned short y[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%3d", i) }.join(', ') %>, +const unsigned short rb_vm_insn_op_offset[] = { +% insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, % end - }; +#endif +}; - ASSERT_VM_INSTRUCTION_SIZE(y); +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_op_offset); +#endif - return &x[y[i]]; +const char * +insn_op_types(VALUE i) +{ + return &rb_vm_insn_op_base[rb_vm_insn_op_offset[i]]; } int diff --git a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb b/tool/ruby_vm/views/_insn_sp_pc_dependency.erb deleted file mode 100644 index 95528fbbf4..0000000000 --- a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb +++ /dev/null @@ -1,27 +0,0 @@ -%# -*- C -*- -%# Copyright (c) 2019 Takashi Kokubun. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -%# -PUREFUNC(MAYBE_UNUSED(static bool insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes))); - -static bool -insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes) -{ - switch (insn) { -% RubyVM::Instructions.each do |insn| -% # handles_sp?: If true, it requires to move sp in JIT -% # always_leaf?: If false, it may call an arbitrary method. pc should be moved -% # before the call, and the method may refer to caller's pc (lineno). -% unless !insn.is_a?(RubyVM::TraceInstructions) && !insn.handles_sp? && insn.always_leaf? - case <%= insn.bin %>: -% end -% end - return true; - default: - return false; - } -} diff --git a/tool/ruby_vm/views/_insn_stack_increase.erb b/tool/ruby_vm/views/_insn_stack_increase.erb deleted file mode 100644 index 315d695cf1..0000000000 --- a/tool/ruby_vm/views/_insn_stack_increase.erb +++ /dev/null @@ -1,52 +0,0 @@ -%# -*- C -*- -%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -%# -PUREFUNC(MAYBE_UNUSED(static int insn_stack_increase(int depth, int insn, const VALUE *opes))); -PUREFUNC(static rb_snum_t insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); - -rb_snum_t -insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) -{ - static const signed char t[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map { |i| - if i.has_attribute?('sp_inc') - '-127' - else - sprintf("%4d", i.rets.size - i.pops.size) - end - }.join(', ') -%>, -% end - }; - signed char c = t[insn]; - - ASSERT_VM_INSTRUCTION_SIZE(t); - if (c != -127) { - return c; - } - else switch(insn) { - default: - UNREACHABLE; -% RubyVM::Instructions.each do |i| -% next unless i.has_attribute?('sp_inc') - case <%= i.bin %>: - return attr_sp_inc_<%= i.name %>(<%= - i.opes.map.with_index do |v, j| - i.cast_from_VALUE v, "opes[#{j}]" - end.join(", ") - %>); -% end - } -} - -int -insn_stack_increase(int depth, int insn, const VALUE *opes) -{ - enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn; - return depth + (int)insn_stack_increase_dispatch(itype, opes); -} diff --git a/tool/ruby_vm/views/_insn_type_chars.erb b/tool/ruby_vm/views/_insn_type_chars.erb index 4e1f63e660..27daec6c6d 100644 --- a/tool/ruby_vm/views/_insn_type_chars.erb +++ b/tool/ruby_vm/views/_insn_type_chars.erb @@ -11,3 +11,22 @@ enum ruby_insn_type_chars { <%= t %> = '<%= c %>', % end }; + +static inline union iseq_inline_storage_entry * +ISEQ_IS_ENTRY_START(const struct rb_iseq_constant_body *body, char op_type) +{ + unsigned int relative_ic_offset = 0; + switch (op_type) { + case TS_IC: + relative_ic_offset += body->ise_size; + case TS_ISE: + relative_ic_offset += body->icvarc_size; + case TS_ICVARC: + relative_ic_offset += body->ivc_size; + case TS_IVC: + break; + default: + rb_bug("Wrong op type"); + } + return &body->is_entries[relative_ic_offset]; +} diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb index 12defa5bf6..2756fa2dec 100644 --- a/tool/ruby_vm/views/_leaf_helpers.erb +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -8,63 +8,6 @@ %; #line <%= __LINE__ + 1 %> <%=cstr __FILE__ %> -static bool -leafness_of_getglobal(VALUE gentry) -{ - const struct rb_global_entry *e = (void *)gentry; - - if (UNLIKELY(rb_gvar_is_traced(e))) { - return false; - } - else { - /* We cannot write this function using a switch() because a - * case label cannot be a function pointer. */ - static rb_gvar_getter_t *const allowlist[] = { - rb_gvar_val_getter, - rb_gvar_var_getter, - /* rb_gvar_undef_getter issues rb_warning() */ - }; - rb_gvar_getter_t *f = rb_gvar_getter_function_of(e); - int i; - - for (i = 0; i < numberof(allowlist); i++) { - if (f == allowlist[i]) { - return true; - } - } - return false; - } -} - -static bool -leafness_of_setglobal(VALUE gentry) -{ - const struct rb_global_entry *e = (void *)gentry; - - if (UNLIKELY(rb_gvar_is_traced(e))) { - return false; - } - else { - /* We cannot write this function using a switch() because a - * case label cannot be a function pointer. */ - static rb_gvar_setter_t *const allowlist[] = { - rb_gvar_val_setter, - /* rb_gvar_readonly_setter issues rb_name_error() */ - rb_gvar_var_setter, - rb_gvar_undef_setter, - }; - rb_gvar_setter_t *f = rb_gvar_setter_function_of(e); - int i; - - for (i = 0; i < numberof(allowlist); i++) { - if (f == allowlist[i]) { - return true; - } - } - return false; - } -} - #include "iseq.h" static bool @@ -73,14 +16,14 @@ leafness_of_defined(rb_num_t op_type) /* see also: vm_insnhelper.c:vm_defined() */ switch (op_type) { case DEFINED_IVAR: - case DEFINED_IVAR2: case DEFINED_GVAR: case DEFINED_CVAR: case DEFINED_YIELD: case DEFINED_REF: case DEFINED_ZSUPER: - return false; + return true; case DEFINED_CONST: + case DEFINED_CONST_FROM: /* has rb_autoload_load(); */ return false; case DEFINED_FUNC: diff --git a/tool/ruby_vm/views/_mjit_compile_insn.erb b/tool/ruby_vm/views/_mjit_compile_insn.erb deleted file mode 100644 index 4488876da3..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_insn.erb +++ /dev/null @@ -1,87 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. - fprintf(f, "{\n"); - { - MAYBE_UNUSED(int pc_moved_p) = FALSE; -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% -% # JIT: Declare stack_size to be used in some macro of _mjit_compile_insn_body.erb - if (status->local_stack_p) { - fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); - } -% -% # JIT: Declare variables for operands, popped values and return values -% insn.declarations.each do |decl| - fprintf(f, " <%= decl %>;\n"); -% end - -% # JIT: Set const expressions for `RubyVM::OperandsUnifications` insn -% insn.preamble.each do |amble| - fprintf(f, "<%= amble.expr.sub(/const \S+\s+/, '') %>\n"); -% end -% -% # JIT: Initialize operands -% insn.opes.each_with_index do |ope, i| - fprintf(f, " <%= ope.fetch(:name) %> = (<%= ope.fetch(:type) %>)0x%"PRIxVALUE";", operands[<%= i %>]); -% case ope.fetch(:type) -% when 'ID' - comment_id(f, (ID)operands[<%= i %>]); -% when 'CALL_INFO' - comment_id(f, ((CALL_INFO)operands[<%= i %>])->mid); -% when 'VALUE' - if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>])); -% end - fprintf(f, "\n"); -% end -% -% # JIT: Initialize popped values -% insn.pops.reverse_each.with_index.reverse_each do |pop, i| - fprintf(f, " <%= pop.fetch(:name) %> = stack[%d];\n", b->stack_size - <%= i + 1 %>); -% end -% -% # JIT: move sp and pc if necessary -<%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> -% -% # JIT: Print insn body in insns.def -<%= render 'mjit_compile_insn_body', locals: { insn: insn } -%> -% -% # JIT: Set return values -% insn.rets.reverse_each.with_index do |ret, i| -% # TOPN(n) = ... - fprintf(f, " stack[%d] = <%= ret.fetch(:name) %>;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %> - <%= i + 1 %>); -% end -% -% # JIT: We should evaluate ISeq modified for TracePoint if it's enabled. Note: This is slow. -% unless insn.always_leaf? - fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %>); - if (!pc_moved_p) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); - } - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); - fprintf(f, " goto cancel;\n"); - fprintf(f, " }\n"); -% end -% -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - } - fprintf(f, "}\n"); -% -% # compiler: If insn has conditional JUMP, the branch which is not targeted by JUMP should be compiled too. -% if insn.expr.expr =~ /if\s+\([^{}]+\)\s+\{[^{}]+JUMP\([^)]+\);[^{}]+\}/ - compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); -% end -% -% # compiler: If insn returns (leave) or does longjmp (throw), the branch should no longer be compiled. TODO: create attr for it? -% if insn.expr.expr =~ /\sTHROW_EXCEPTION\([^)]+\);/ || insn.expr.expr =~ /\bvm_pop_frame\(/ - b->finish_p = TRUE; -% end diff --git a/tool/ruby_vm/views/_mjit_compile_insn_body.erb b/tool/ruby_vm/views/_mjit_compile_insn_body.erb deleted file mode 100644 index a2a750fbdc..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_insn_body.erb +++ /dev/null @@ -1,115 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% to_cstr = lambda do |line| -% normalized = line.gsub(/\t/, ' ' * 8) -% indented = normalized.sub(/\A(?!#)/, ' ') # avoid indenting preprocessor -% rstring2cstr(indented.rstrip).sub(/"\z/, '\\n"') -% end -% -% # -% # Expand simple macro, which doesn't require dynamic C code. -% # -% expand_simple_macros = lambda do |arg_expr| -% arg_expr.dup.tap do |expr| -% # For `leave`. We can't proceed next ISeq in the same JIT function. -% expr.gsub!(/^(?<indent>\s*)RESTORE_REGS\(\);\n/) do -% indent = Regexp.last_match[:indent] -% <<-RESTORE_REGS.gsub(/^ +/, '') -% #if OPT_CALL_THREADED_CODE -% #{indent}rb_ec_thread_ptr(ec)->retval = val; -% #{indent}return 0; -% #else -% #{indent}return val; -% #endif -% RESTORE_REGS -% end -% expr.gsub!(/^(?<indent>\s*)NEXT_INSN\(\);\n/) do -% indent = Regexp.last_match[:indent] -% <<-end.gsub(/^ +/, '') -% #{indent}UNREACHABLE_RETURN(Qundef); -% end -% end -% end -% end -% -% # -% # Print a body of insn, but with macro expansion. -% # -% expand_simple_macros.call(insn.expr.expr).each_line do |line| -% # -% # Expand dynamic macro here (only JUMP for now) -% # -% # TODO: support combination of following macros in the same line -% case line -% when /\A\s+JUMP\((?<dest>[^)]+)\);\s+\z/ -% dest = Regexp.last_match[:dest] -% -% if insn.name == 'opt_case_dispatch' # special case... TODO: use another macro to avoid checking name - { - struct case_dispatch_var arg; - arg.f = f; - arg.base_pos = pos + insn_len(insn); - arg.last_value = Qundef; - - fprintf(f, " switch (<%= dest %>) {\n"); - st_foreach(RHASH_TBL_RAW(hash), compile_case_dispatch_each, (VALUE)&arg); - fprintf(f, " case %lu:\n", else_offset); - fprintf(f, " goto label_%lu;\n", arg.base_pos + else_offset); - fprintf(f, " }\n"); - } -% else -% # Before we `goto` next insn, we need to set return values, especially for getinlinecache -% insn.rets.reverse_each.with_index do |ret, i| -% # TOPN(n) = ... - fprintf(f, " stack[%d] = <%= ret.fetch(:name) %>;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %> - <%= i + 1 %>); -% end -% - next_pos = pos + insn_len(insn) + (unsigned int)<%= dest %>; - fprintf(f, " goto label_%d;\n", next_pos); -% end -% when /\A\s+CALL_SIMPLE_METHOD\(\);\s+\z/ -% # For `opt_xxx`'s fallbacks. - if (status->local_stack_p) { - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - } - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); - fprintf(f, " goto cancel;\n"); -% when /\A(?<prefix>.+\b)INSN_LABEL\((?<name>[^)]+)\)(?<suffix>.+)\z/m -% prefix, name, suffix = Regexp.last_match[:prefix], Regexp.last_match[:name], Regexp.last_match[:suffix] - fprintf(f, " <%= prefix.gsub(/\t/, ' ' * 8) %>INSN_LABEL(<%= name %>_%d)<%= suffix.sub(/\n/, '\n') %>", pos); -% else -% if insn.handles_sp? -% # If insn.handles_sp? is true, cfp->sp might be changed inside insns (like vm_caller_setup_arg_block) -% # and thus we need to use cfp->sp, even when local_stack_p is TRUE. When insn.handles_sp? is true, -% # cfp->sp should be available too because _mjit_compile_pc_and_sp.erb sets it. - fprintf(f, <%= to_cstr.call(line) %>); -% else -% # If local_stack_p is TRUE and insn.handles_sp? is false, stack values are only available in local variables -% # for stack. So we need to replace those macros if local_stack_p is TRUE here. -% case line -% when /\bGET_SP\(\)/ -% # reg_cfp->sp - fprintf(f, <%= to_cstr.call(line.sub(/\bGET_SP\(\)/, '%s')) %>, (status->local_stack_p ? "(stack + stack_size)" : "GET_SP()")); -% when /\bSTACK_ADDR_FROM_TOP\((?<num>[^)]+)\)/ -% # #define STACK_ADDR_FROM_TOP(n) (GET_SP()-(n)) -% num = Regexp.last_match[:num] - fprintf(f, <%= to_cstr.call(line.sub(/\bSTACK_ADDR_FROM_TOP\(([^)]+)\)/, '%s')) %>, - (status->local_stack_p ? "stack + (stack_size - (<%= num %>))" : "STACK_ADDR_FROM_TOP(<%= num %>)")); -% when /\bTOPN\((?<num>[^)]+)\)/ -% # #define TOPN(n) (*(GET_SP()-(n)-1)) -% num = Regexp.last_match[:num] - fprintf(f, <%= to_cstr.call(line.sub(/\bTOPN\(([^)]+)\)/, '%s')) %>, - (status->local_stack_p ? "*(stack + (stack_size - (<%= num %>) - 1))" : "TOPN(<%= num %>)")); -% else - fprintf(f, <%= to_cstr.call(line) %>); -% end -% end -% end -% end diff --git a/tool/ruby_vm/views/_mjit_compile_ivar.erb b/tool/ruby_vm/views/_mjit_compile_ivar.erb deleted file mode 100644 index 3d5e0e8f54..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_ivar.erb +++ /dev/null @@ -1,54 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # Optimized case of get_instancevariable instruction. -#if OPT_IC_FOR_IVAR -{ -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% # compiler: Use copied IC to avoid race condition - IC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->cache; -% -% # compiler: Consider cfp->self as T_OBJECT if ic_copy->ic_serial is set - if (!status->compile_info->disable_ivar_cache && ic_copy->ic_serial) { -% # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. -% # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> -% -% # JIT: prepare vm_getivar's arguments and variables - fprintf(f, "{\n"); - fprintf(f, " VALUE obj = GET_SELF();\n"); - fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->ic_serial); - fprintf(f, " const st_index_t index = %"PRIuSIZE";\n", ic_copy->ic_value.index); -% # JIT: cache hit path of vm_getivar, or cancel JIT. -% if insn.name == 'setinstancevariable' - fprintf(f, " VALUE val = stack[%d];\n", b->stack_size - 1); - fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj))) {\n"); - fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); - fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], val);\n"); - fprintf(f, " }\n"); -% else - fprintf(f, " VALUE val;\n"); - fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); - fprintf(f, " stack[%d] = val;\n", b->stack_size); - fprintf(f, " }\n"); -% end - fprintf(f, " else {\n"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " goto ivar_cancel;\n"); - fprintf(f, " }\n"); - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - fprintf(f, "}\n"); - break; - } -} -#endif // OPT_IC_FOR_IVAR diff --git a/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb b/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb deleted file mode 100644 index 545b9f72f5..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb +++ /dev/null @@ -1,36 +0,0 @@ -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # JIT: Move pc for catch table on catch_except_p, and for #caller_locations and rb_profile_frames on !insn.always_leaf? - if (body->catch_except_p || <%= insn.always_leaf? ? 'FALSE' : 'TRUE' %>) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ - pc_moved_p = TRUE; - } -% -% # JIT: move sp to use or preserve stack variables - if (status->local_stack_p) { -% # sp motion is optimized away for `handles_sp? #=> false` case. -% # Thus sp should be set properly before `goto cancel`. -% if insn.handles_sp? -% # JIT-only behavior (pushing JIT's local variables to VM's stack): - { - rb_snum_t i, push_size; - push_size = -<%= insn.call_attribute('sp_inc') %> + <%= insn.rets.size %> - <%= insn.pops.size %>; - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ - for (i = 0; i < push_size; i++) { - fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); - } - } -% end - } - else { -% if insn.handles_sp? - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - <%= insn.pops.size %>); /* POPN(INSN_ATTR(popn)); */ -% else - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); -% end - } diff --git a/tool/ruby_vm/views/_mjit_compile_send.erb b/tool/ruby_vm/views/_mjit_compile_send.erb deleted file mode 100644 index da7e96581b..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_send.erb +++ /dev/null @@ -1,106 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # Optimized case of send / opt_send_without_block instructions. -{ - MAYBE_UNUSED(int pc_moved_p) = FALSE; -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% # compiler: Use copied cc to avoid race condition - CALL_CACHE cc_copy = status->cc_entries + (cc - body->cc_entries); -% - if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { - const rb_iseq_t *iseq; - unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. -% if insn.name == 'send' - argc += ((ci->flag & VM_CALL_ARGS_BLOCKARG) ? 1 : 0); // simulate `vm_caller_setup_arg_block`'s `--reg_cfp->sp` -% end - - if (!(ci->flag & VM_CALL_TAILCALL) // inlining non-tailcall path - && cc_copy->me->def->type == VM_METHOD_TYPE_ISEQ && fastpath_applied_iseq_p(ci, cc_copy, iseq = def_iseq_ptr(cc_copy->me->def))) { // CC_SET_FASTPATH in vm_callee_setup_arg - int param_size = iseq->body->param.size; - - fprintf(f, "{\n"); -% # JIT: Declare stack_size to be used in some macro of _mjit_compile_insn_body.erb - if (status->local_stack_p) { - fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); - } - -% # JIT: Invalidate call cache if it requires vm_search_method. This allows to inline some of following things. - fprintf(f, " if (UNLIKELY(GET_GLOBAL_METHOD_STATE() != %"PRI_SERIALT_PREFIX"u ||\n", cc_copy->method_state); - fprintf(f, " RCLASS_SERIAL(CLASS_OF(stack[%d])) != %"PRI_SERIALT_PREFIX"u)) {\n", b->stack_size - 1 - argc, cc_copy->class_serial); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " goto send_cancel;\n"); - fprintf(f, " }\n"); - -% # JIT: move sp and pc if necessary -<%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> - -% # JIT: If ISeq is inlinable, call the inlined method without pushing a frame. - if (status->inlined_iseqs != NULL && status->inlined_iseqs[pos] == iseq->body) { - fprintf(f, " {\n"); - fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); - fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size - argc - 1); - fprintf(f, " stack[%d] = _mjit_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size - argc - 1, pos); - fprintf(f, " reg_cfp->self = orig_self;\n"); - fprintf(f, " }\n"); - } - else { -% # JIT: Print insn body in insns.def - fprintf(f, " {\n"); - fprintf(f, " struct rb_calling_info calling;\n"); -% if insn.name == 'send' - fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", operands[0], operands[2]); -% else - fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); -% end - fprintf(f, " calling.argc = %d;\n", ci->orig_argc); - fprintf(f, " calling.recv = stack[%d];\n", b->stack_size - 1 - argc); - -% # JIT: Special CALL_METHOD. Bypass cc_copy->call and inline vm_call_iseq_setup_normal for vm_call_iseq_setup_func FASTPATH. - fprintf(f, " {\n"); - fprintf(f, " VALUE v;\n"); - fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n", - (VALUE)cc_copy->me, param_size, iseq->body->local_table_size); // fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE - if (iseq->body->catch_except_p) { - fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); - fprintf(f, " v = vm_exec(ec, TRUE);\n"); - } - else { - fprintf(f, " if ((v = mjit_exec(ec)) == Qundef) {\n"); - fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup - fprintf(f, " v = vm_exec(ec, FALSE);\n"); - fprintf(f, " }\n"); - } - fprintf(f, " stack[%d] = v;\n", b->stack_size - argc - 1); - fprintf(f, " }\n"); - - fprintf(f, " }\n"); - -% # JIT: We should evaluate ISeq modified for TracePoint if it's enabled. Note: This is slow. - fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %>); - if (!pc_moved_p) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); - } - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); - fprintf(f, " goto cancel;\n"); - fprintf(f, " }\n"); - } - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - - fprintf(f, "}\n"); - break; - } - } -} diff --git a/tool/ruby_vm/views/_sp_inc_helpers.erb b/tool/ruby_vm/views/_sp_inc_helpers.erb index f4cf50b8c8..740fe10142 100644 --- a/tool/ruby_vm/views/_sp_inc_helpers.erb +++ b/tool/ruby_vm/views/_sp_inc_helpers.erb @@ -9,7 +9,7 @@ #line <%= __LINE__ + 1 %> <%=cstr __FILE__ %> static rb_snum_t -sp_inc_of_sendish(const struct rb_call_info *ci) +sp_inc_of_sendish(const struct rb_callinfo *ci) { /* Send-ish instructions will: * @@ -18,8 +18,8 @@ sp_inc_of_sendish(const struct rb_call_info *ci) * 3. Pop receiver. * 4. Push return value. */ - const int argb = (ci->flag & VM_CALL_ARGS_BLOCKARG) ? 1 : 0; - const int argc = ci->orig_argc; + const int argb = (vm_ci_flag(ci) & (VM_CALL_ARGS_BLOCKARG | VM_CALL_FORWARDING)) ? 1 : 0; + const int argc = vm_ci_argc(ci); const int recv = 1; const int retn = 1; @@ -28,7 +28,7 @@ sp_inc_of_sendish(const struct rb_call_info *ci) } static rb_snum_t -sp_inc_of_invokeblock(const struct rb_call_info *ci) +sp_inc_of_invokeblock(const struct rb_callinfo *ci) { /* sp_inc of invokeblock is almost identical to that of sendish * instructions, except that it does not pop receiver. */ diff --git a/tool/ruby_vm/views/_trace_instruction.erb b/tool/ruby_vm/views/_trace_instruction.erb index be2b091d48..3588207d39 100644 --- a/tool/ruby_vm/views/_trace_instruction.erb +++ b/tool/ruby_vm/views/_trace_instruction.erb @@ -10,7 +10,12 @@ /* insn <%= insn.pretty_name %> */ INSN_ENTRY(<%= insn.name %>) { - vm_trace(ec, GET_CFP(), GET_PC()); - DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>); + vm_trace(ec, GET_CFP()); +% if insn.name =~ +% /\Atrace_opt_(plus|minus|mult|div|mod|eq|neq|lt|le|gt|ge|ltlt|and|or|aref|aset|length|size|empty_p|nil_p|succ|not|regexpmatch2)\z/ +% jump_dest = "opt_send_without_block" +% end + <%= 'ADD_PC(1);' if insn.name == 'trace_opt_neq' %> + DISPATCH_ORIGINAL_INSN(<%= jump_dest || insn.jump_destination %>); END_INSN(<%= insn.name %>); } diff --git a/tool/ruby_vm/views/_zjit_helpers.erb b/tool/ruby_vm/views/_zjit_helpers.erb new file mode 100644 index 0000000000..1185dbd9d8 --- /dev/null +++ b/tool/ruby_vm/views/_zjit_helpers.erb @@ -0,0 +1,31 @@ +#if USE_ZJIT + +MAYBE_UNUSED(static int vm_bare_insn_to_zjit_insn(int insn)); +static int +vm_bare_insn_to_zjit_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case BIN(<%= insn.jump_destination %>): + return <%= insn.bin %>; +% end + default: + return insn; + } +} + +MAYBE_UNUSED(static int vm_zjit_insn_to_bare_insn(int insn)); +static int +vm_zjit_insn_to_bare_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case <%= insn.bin %>: + return BIN(<%= insn.jump_destination %>); +% end + default: + return insn; + } +} + +#endif diff --git a/tool/ruby_vm/views/_zjit_instruction.erb b/tool/ruby_vm/views/_zjit_instruction.erb new file mode 100644 index 0000000000..7fd657697c --- /dev/null +++ b/tool/ruby_vm/views/_zjit_instruction.erb @@ -0,0 +1,12 @@ +#if USE_ZJIT + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + rb_zjit_profile_insn(BIN(<%= insn.jump_destination %>), ec); + DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} + +#endif diff --git a/tool/ruby_vm/views/insns.inc.erb b/tool/ruby_vm/views/insns.inc.erb index 29981a8a2d..6521a89b8a 100644 --- a/tool/ruby_vm/views/insns.inc.erb +++ b/tool/ruby_vm/views/insns.inc.erb @@ -6,21 +6,36 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'contains YARV instruction list', edit: __FILE__, } -%> +#ifndef INSNS_INC +#define INSNS_INC 1 + /* BIN : Basic Instruction Name */ #define BIN(n) YARVINSN_##n enum ruby_vminsn_type { -% RubyVM::Instructions.each do |i| +% insns.each do |i| + <%= i.bin %>, +% end +#if USE_ZJIT +% zjit_insns.each do |i| <%= i.bin %>, % end +#endif VM_INSTRUCTION_SIZE }; +#define VM_BARE_INSTRUCTION_SIZE <%= RubyVM::Instructions.count { |i| i.name !~ /\A(trace|zjit)_/ } %> + #define ASSERT_VM_INSTRUCTION_SIZE(array) \ STATIC_ASSERT(numberof_##array, numberof(array) == VM_INSTRUCTION_SIZE) + +#endif diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb index e5793a2a70..48dd0e8832 100644 --- a/tool/ruby_vm/views/insns_info.inc.erb +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -11,12 +11,16 @@ this_file: 'contains instruction information for yarv instruction sequence.', edit: __FILE__, } %> +#ifndef INSNS_INFO_INC +#define INSNS_INFO_INC 1 <%= render 'insn_type_chars' %> <%= render 'insn_name_info' %> <%= render 'insn_len_info' %> <%= render 'insn_operand_info' %> <%= render 'leaf_helpers' %> <%= render 'sp_inc_helpers' %> +<%= render 'zjit_helpers' %> <%= render 'attributes' %> -<%= render 'insn_stack_increase' %> -<%= render 'insn_sp_pc_dependency' %> +<%= render 'insn_leaf_info' %> +<%= render 'comptime_insn_stack_increase' %> +#endif diff --git a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb new file mode 100644 index 0000000000..793528af5d --- /dev/null +++ b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb @@ -0,0 +1,14 @@ +module RubyVM::RJIT # :nodoc: all + Instruction = Data.define(:name, :bin, :len, :operands) + + INSNS = { +% RubyVM::Instructions.each_with_index do |insn, i| + <%= i %> => Instruction.new( + name: :<%= insn.name %>, + bin: <%= i %>, # BIN(<%= insn.name %>) + len: <%= insn.width %>, # insn_len + operands: <%= (insn.operands unless insn.name.start_with?(/trace_|zjit_/)).inspect %>, + ), +% end + } +end diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb deleted file mode 100644 index d9092a756d..0000000000 --- a/tool/ruby_vm/views/mjit_compile.inc.erb +++ /dev/null @@ -1,93 +0,0 @@ -/* -*- C -*- */ - -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -<%= render 'copyright' %> -% -% # This is an ERB template that generates Ruby code that generates C code that -% # generates JIT-ed C code. -<%= render 'notice', locals: { - this_file: 'is the main part of compile_insn() in mjit_compile.c', - edit: __FILE__, -} -%> -% -% unsupported_insns = [ -% 'defineclass', # low priority -% 'opt_call_c_function', # low priority -% ] -% -% opt_send_without_block = RubyVM::Instructions.find { |i| i.name == 'opt_send_without_block' } -% if opt_send_without_block.nil? -% raise 'opt_send_without_block not found' -% end -% -% send_compatible_opt_insns = RubyVM::BareInstructions.to_a.select do |insn| -% insn.name.start_with?('opt_') && opt_send_without_block.opes == insn.opes && -% insn.expr.expr.lines.any? { |l| l.match(/\A\s+CALL_SIMPLE_METHOD\(\);\s+\z/) } -% end.map(&:name) -% -% # Available variables and macros in JIT-ed function: -% # ec: the first argument of _mjitXXX -% # reg_cfp: the second argument of _mjitXXX -% # GET_CFP(): refers to `reg_cfp` -% # GET_EP(): refers to `reg_cfp->ep` -% # GET_SP(): refers to `reg_cfp->sp`, or `(stack + stack_size)` if local_stack_p -% # GET_SELF(): refers to `reg_cfp->self` -% # GET_LEP(): refers to `VM_EP_LEP(reg_cfp->ep)` -% # EXEC_EC_CFP(): refers to `val = vm_exec(ec, TRUE)` with frame setup -% # CALL_METHOD(): using `GET_CFP()` and `EXEC_EC_CFP()` -% # TOPN(): refers to `reg_cfp->sp`, or `*(stack + (stack_size - num - 1))` if local_stack_p -% # STACK_ADDR_FROM_TOP(): refers to `reg_cfp->sp`, or `stack + (stack_size - num)` if local_stack_p -% # DISPATCH_ORIGINAL_INSN(): expanded in _mjit_compile_insn.erb -% # THROW_EXCEPTION(): specially defined for JIT -% # RESTORE_REGS(): specially defined for `leave` - -switch (insn) { -% (RubyVM::BareInstructions.to_a + RubyVM::OperandsUnifications.to_a).each do |insn| -% next if unsupported_insns.include?(insn.name) - case BIN(<%= insn.name %>): -% # Instruction-specific behavior in JIT -% case insn.name -% when 'opt_send_without_block', 'send' -<%= render 'mjit_compile_send', locals: { insn: insn } -%> -% when *send_compatible_opt_insns -% # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated. -% cc_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_CACHE' } - if (has_valid_method_type(status->cc_entries + ((CALL_CACHE)operands[<%= cc_index %>] - body->cc_entries))) { -<%= render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%> -<%= render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%> - break; - } -% when 'getinstancevariable', 'setinstancevariable' -<%= render 'mjit_compile_ivar', locals: { insn: insn } -%> -% when 'leave' - if (b->stack_size != 1) { - if (mjit_opts.warnings || mjit_opts.verbose) - fprintf(stderr, "MJIT warning: Unexpected JIT stack_size on leave: %d\n", b->stack_size); - status->success = false; - } -% # Special leave for an inlined call. - if (status->inlined_iseqs == NULL) { // the current ISeq is being inlined - fprintf(f, " return stack[0];\n"); - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - break; - } -% end -% -% # Main insn implementation generated by insns.def -<%= render 'mjit_compile_insn', locals: { insn: insn } -%> - break; -% end -% -% # We don't support InstructionsUnifications yet because it's not used for now. -% # We don't support TraceInstructions yet. There is no blocker for it but it's just not implemented. - default: - if (mjit_opts.warnings || mjit_opts.verbose) - fprintf(stderr, "MJIT warning: Skipped to compile unsupported instruction: %s\n", insn_name(insn)); - status->success = false; - break; -} diff --git a/tool/ruby_vm/views/opt_sc.inc.erb b/tool/ruby_vm/views/opt_sc.inc.erb deleted file mode 100644 index e58c81989f..0000000000 --- a/tool/ruby_vm/views/opt_sc.inc.erb +++ /dev/null @@ -1,40 +0,0 @@ -/* -*- C -*- */ - -%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -% raise ':FIXME:TBW' if RubyVM::VmOptsH['STACK_CACHING'] -<%= render 'copyright' %> -<%= render 'notice', locals: { - this_file: 'is for threaded code', - edit: __FILE__, -} -%> - -#define SC_STATE_SIZE 6 - -#define SCS_XX 1 -#define SCS_AX 2 -#define SCS_BX 3 -#define SCS_AB 4 -#define SCS_BA 5 - -#define SC_ERROR 0xffffffff - -static const VALUE sc_insn_info[][SC_STATE_SIZE] = { -#define NO_SC { SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR } -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map{|i| 'NO_SC' }.join(', ') %>, -% end -#undef NO_SC -}; - -static const VALUE sc_insn_next[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map{|i| 'SCS_XX' }.join(', ') %>, -% end -}; - -ASSERT_VM_INSTRUCTION_SIZE(sc_insn_next); diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb index 676f1edaba..9d9cf0a43a 100644 --- a/tool/ruby_vm/views/optinsn.inc.erb +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -23,20 +23,20 @@ insn_operands_unification(INSN *iobj) /* do nothing */; break; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| case <%= orig.bin %>: % unifs.each do |insn| /* <%= insn.pretty_name %> */ if ( <%= insn.condition('op') %> ) { -% insn.opes.each_with_index do |o, x| +% insn.operands.each_with_index do |o, x| % n = insn.operand_shift_of(o) % if n != 0 then op[<%= x %>] = op[<%= x + n %>]; % end % end iobj->insn_id = <%= insn.bin %>; - iobj->operand_size = <%= insn.opes.size %>; + iobj->operand_size = <%= insn.operands.size %>; break; } % end @@ -55,12 +55,12 @@ rb_insn_unified_local_var_level(VALUE insn) /* optimize rule */ switch (insn) { default: - return -1; /* do nothing */; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| + return -1; /* do nothing */; +% RubyVM::OperandsUnification.each_group do |orig, unifs| % unifs.each do|insn| case <%= insn.bin %>: % insn.spec.map{|(var,val)|val}.reject{|i| i == '*' }.each do |val| - return <%= val %>; + return <%= val %>; % break % end % end diff --git a/tool/ruby_vm/views/optunifs.inc.erb b/tool/ruby_vm/views/optunifs.inc.erb index e92a95beff..c096712936 100644 --- a/tool/ruby_vm/views/optunifs.inc.erb +++ b/tool/ruby_vm/views/optunifs.inc.erb @@ -7,7 +7,6 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % raise ':FIXME:TBW' if RubyVM::VmOptsH['INSTRUCTIONS_UNIFICATION'] -% n = RubyVM::Instructions.size <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -16,6 +15,4 @@ /* Let .bss section automatically initialize this variable */ /* cf. Section 6.7.8 of ISO/IEC 9899:1999 */ -static const int *const *const unified_insns_data[<%= n %>]; - -ASSERT_VM_INSTRUCTION_SIZE(unified_insns_data); +static const int *const *const unified_insns_data[VM_INSTRUCTION_SIZE]; diff --git a/tool/ruby_vm/views/vm.inc.erb b/tool/ruby_vm/views/vm.inc.erb index c1a3faf60a..38bf5f05ae 100644 --- a/tool/ruby_vm/views/vm.inc.erb +++ b/tool/ruby_vm/views/vm.inc.erb @@ -13,18 +13,22 @@ } -%> #include "vm_insnhelper.h" -% RubyVM::BareInstructions.to_a.each do |insn| +% RubyVM::BareInstruction.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::OperandsUnifications.to_a.each do |insn| +% RubyVM::OperandsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::InstructionsUnifications.to_a.each do |insn| +% RubyVM::InstructionsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::TraceInstructions.to_a.each do |insn| +% RubyVM::ZJITInstruction.all.each do |insn| +<%= render 'zjit_instruction', locals: { insn: insn } -%> +% end +% +% RubyVM::TraceInstruction.all.each do |insn| <%= render 'trace_instruction', locals: { insn: insn } -%> % end diff --git a/tool/ruby_vm/views/vmtc.inc.erb b/tool/ruby_vm/views/vmtc.inc.erb index 99cbd92614..39dc8bfa6b 100644 --- a/tool/ruby_vm/views/vmtc.inc.erb +++ b/tool/ruby_vm/views/vmtc.inc.erb @@ -6,6 +6,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' -%> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -13,9 +16,14 @@ } -%> static const void *const insns_address_table[] = { -% RubyVM::Instructions.each do |i| +% insns.each do |i| LABEL_PTR(<%= i.name %>), % end +#if USE_ZJIT +% zjit_insns.each do |i| + LABEL_PTR(<%= i.name %>), +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(insns_address_table); diff --git a/tool/run-gcov.rb b/tool/run-gcov.rb index 5df7622aa3..46626e4703 100644 --- a/tool/run-gcov.rb +++ b/tool/run-gcov.rb @@ -47,7 +47,8 @@ Pathname.glob("**/*.gcda").sort.each do |gcda| )? Creating\ .*\n \n - )+\z + )+ + (Lines\ executed:.*\n)?\z )x raise "Unexpected gcov output" end diff --git a/tool/run-lcov.rb b/tool/run-lcov.rb index f27578200a..bdccc29a11 100644 --- a/tool/run-lcov.rb +++ b/tool/run-lcov.rb @@ -20,7 +20,7 @@ def backup_gcda_files(gcda_files) end def run_lcov(*args) - system("lcov", "--rc", "lcov_branch_coverage=1", *args) + system("lcov", "--rc", "geninfo_unexecuted_blocks=1", "--rc", "lcov_branch_coverage=1", *args, exception: true) end $info_files = [] @@ -41,11 +41,19 @@ def run_lcov_remove(info_src, info_out) ext/-test-/* ext/nkf/nkf-utf8/nkf.c ).each {|f| dirs << File.join(File.dirname(__dir__), f) } - run_lcov("--remove", info_src, *dirs, "-o", info_out) + run_lcov("--ignore-errors", "unused", "--remove", info_src, *dirs, "-o", info_out) end def run_genhtml(info, out) - system("genhtml", "--branch-coverage", "--ignore-errors", "source", info, "-o", out) + base_dir = File.dirname(File.dirname(__dir__)) + ignore_errors = %w(source unmapped category).reject do |a| + Open3.capture3("genhtml", "--ignore-errors", a)[1].include?("unknown argument for --ignore-errors") + end + system("genhtml", + "--branch-coverage", + "--prefix", base_dir, + *ignore_errors.flat_map {|a| ["--ignore-errors", a] }, + info, "-o", out, exception: true) end def gen_rb_lcov(file) diff --git a/tool/runruby.rb b/tool/runruby.rb index d9fc1ac09d..ec63d1008a 100755 --- a/tool/runruby.rb +++ b/tool/runruby.rb @@ -11,6 +11,10 @@ when ENV['RUNRUBY_USE_GDB'] == 'true' debugger = :gdb when ENV['RUNRUBY_USE_LLDB'] == 'true' debugger = :lldb +when ENV['RUNRUBY_USE_RR'] == 'true' + debugger = :rr +when ENV['RUNRUBY_YJIT_STATS'] + use_yjit_stat = true end while arg = ARGV[0] break ARGV.shift if arg == '--' @@ -85,7 +89,8 @@ config = File.read(conffile) config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)\n.*\n/, '') config = Module.new {module_eval(config, conffile)}::RbConfig::CONFIG -ruby = File.join(archdir, config["RUBY_INSTALL_NAME"]+config['EXEEXT']) +install_name = config["RUBY_INSTALL_NAME"]+config['EXEEXT'] +ruby = File.join(archdir, install_name) unless File.exist?(ruby) abort "#{ruby} is not found.\nTry `make' first, then `make test', please.\n" end @@ -106,13 +111,12 @@ env = { 'RUBY_FIBER_MACHINE_STACK_SIZE' => '1', } -runner = File.join(abs_archdir, "exe/ruby#{config['EXEEXT']}") +runner = File.join(abs_archdir, "exe/#{install_name}") runner = nil unless File.exist?(runner) abs_ruby = runner || File.expand_path(ruby) env["RUBY"] = abs_ruby env["GEM_PATH"] = env["GEM_HOME"] = File.expand_path(".bundle", srcdir) -env["BUNDLE_RUBY"] = abs_ruby -env["BUNDLE_GEM"] = "#{abs_ruby} -rrubygems #{srcdir}/bin/gem --backtrace" +env["GEM_COMMAND"] = "#{abs_ruby} -rrubygems #{srcdir}/bin/gem --backtrace" env["PATH"] = [File.dirname(abs_ruby), abs_archdir, ENV["PATH"]].compact.join(File::PATH_SEPARATOR) if e = ENV["RUBYLIB"] @@ -120,27 +124,27 @@ if e = ENV["RUBYLIB"] end env["RUBYLIB"] = $:.replace(libs).join(File::PATH_SEPARATOR) +gem_path = [abs_archdir, srcdir].map {|d| File.realdirpath(".bundle", d)} +if e = ENV["GEM_PATH"] + gem_path |= e.split(File::PATH_SEPARATOR) +end +env["GEM_PATH"] = gem_path.join(File::PATH_SEPARATOR) + libruby_so = File.join(abs_archdir, config['LIBRUBY_SO']) if File.file?(libruby_so) if e = config['LIBPATHENV'] and !e.empty? env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) end - unless runner - if e = config['PRELOADENV'] - e = nil if e.empty? - e ||= "LD_PRELOAD" if /linux/ =~ RUBY_PLATFORM - end - if e - env[e] = [libruby_so, ENV[e]].compact.join(File::PATH_SEPARATOR) - end - end end +# Work around a bug in FreeBSD 13.2 which can cause fork(2) to hang +# See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271490 +env['LD_BIND_NOW'] = 'yes' if /freebsd/ =~ RUBY_PLATFORM ENV.update env if debugger case debugger - when :gdb, nil + when :gdb debugger = %W'gdb -x #{srcdir}/.gdbinit' if File.exist?(gdb = 'run.gdb') or File.exist?(gdb = File.join(abs_archdir, 'run.gdb')) @@ -154,6 +158,8 @@ if debugger debugger.push('-s', lldb) end debugger << '--' + when :rr + debugger = ['rr', 'record'] end if idx = precommand.index(:debugger) @@ -164,6 +170,9 @@ if debugger end cmd = [runner || ruby] +if use_yjit_stat + cmd << '--yjit-stats' +end cmd.concat(ARGV) cmd.unshift(*precommand) unless precommand.empty? diff --git a/tool/search-cgvars.rb b/tool/search-cgvars.rb new file mode 100644 index 0000000000..c62641a3ff --- /dev/null +++ b/tool/search-cgvars.rb @@ -0,0 +1,55 @@ +# +# Listing C's global variables in .so or .o, or .bundle on Mac OS using "objdump -t" (elf64-x86-64) +# to check ractor-safety. +# +# Usage: ruby search-cgvars.rb foo.so bar.o .ext/x86_64-darwin18/psych.bundle +# +MAC_OS = RbConfig::CONFIG['host_os'].match? /darwin|mac os/ + +def gvars file + # '0000000000031ac8 g O .bss 0000000000000008 rb_cSockIfaddr' + # On mac, with .bundle files: + # '0000000000004258 l O __DATA,__bss _passwd_blocking' + + strs = `objdump -t #{file}` + found = {} + strs.each_line{|line| + if /[\da-f]{16} / =~ line + addr = line[0...16] + flags = line[17...24].tr(' ', '').split(//).sort.uniq + rest = line[25..] + if MAC_OS + seg, name = rest.split(/\s+/) + else + seg, size, name = rest.split(/\s+/) + end + if flags.include?('O') + # p [addr, flags, seg, size, name] + found[name] = [flags, seg, *size] + end + end + } + puts "## #{file}:" + found.sort_by{|name, (flags, *)| + [flags, name] + }.each{|name, rest| + flags, seg, size = *rest + next if (size.to_i == 0 && !MAC_OS) && seg != '*UND*' + case seg + when ".rodata", ".data.rel.ro", ".got.plt", ".eh_frame", ".fini_array" + next + when /,__const$/ # Mac OS + next + end + case name + when /^id_/, /^rbimpl_id/, /^sym_/, /^rb_[cme]/, /\Acompleted\.\d+\z/ + next + when /^_id_/, /\.rbimpl_id(\.\d+)?$/ # Mac OS + next + end + puts " %40s %s" % [name, rest.inspect] + } +end +ARGV.each{|file| + gvars file +} diff --git a/tool/strip-rdoc.rb b/tool/strip-rdoc.rb index 0ac9c39323..d8e311cdbf 100755 --- a/tool/strip-rdoc.rb +++ b/tool/strip-rdoc.rb @@ -1,26 +1,14 @@ #!ruby +# frozen_string_literal: true # Filter for preventing Doxygen from processing RDoc comments. # Used by the Doxygen template. -ARGF.binmode -source = ARGF.read -source = source.gsub(%r{/\*([!*])((?!\*/).+?)\*/}m) do |comment| - marker, comment = $1, $2 - next "/**#{comment}*/" unless /^\s*\*\s?\-\-\s*$/ =~ comment - doxybody = nil - comment.each_line do |line| - if doxybody - if /^\s*\*\s?\+\+\s*$/ =~ line - break - end - doxybody << line - else - if /^\s*\*\s?--\s*$/ =~ line - doxybody = "\n" - end - end - end - "/*#{marker}#{doxybody}*/" -end -print source +print ARGF.binmode.read.tap {|src| + src.gsub!(%r|(/\*[!*])(?:(?!\*/).)+?^\s*\*\s?\-\-\s*$(.+?\*/)|m) { + marker = $1 + comment = $2 + comment.sub!(%r|^\s*\*\s?\+\+\s*$.+?(\s*\*/)\z|m, '\\1') + marker + comment + } +} diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 93031ac849..14d7a3893d 100644..100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -1,322 +1,943 @@ -# sync following repositories to ruby repository -# -# * https://github.com/rubygems/rubygems -# * https://github.com/bundler/bundler -# * https://github.com/ruby/rdoc -# * https://github.com/ruby/reline -# * https://github.com/flori/json -# * https://github.com/ruby/psych -# * https://github.com/ruby/fileutils -# * https://github.com/ruby/fiddle -# * https://github.com/ruby/stringio -# * https://github.com/ruby/io-console -# * https://github.com/ruby/csv -# * https://github.com/ruby/webrick -# * https://github.com/ruby/dbm -# * https://github.com/ruby/gdbm -# * https://github.com/ruby/sdbm -# * https://github.com/ruby/etc -# * https://github.com/ruby/date -# * https://github.com/ruby/zlib -# * https://github.com/ruby/fcntl -# * https://github.com/ruby/scanf -# * https://github.com/ruby/cmath -# * https://github.com/ruby/strscan -# * https://github.com/ruby/ipaddr -# * https://github.com/ruby/logger -# * https://github.com/ruby/prime -# * https://github.com/ruby/matrix -# * https://github.com/ruby/ostruct -# * https://github.com/ruby/rexml -# * https://github.com/ruby/rss -# * https://github.com/ruby/irb -# * https://github.com/ruby/sync -# * https://github.com/ruby/tracer -# * https://github.com/ruby/shell -# * https://github.com/ruby/forwardable -# * https://github.com/ruby/thwait -# * https://github.com/ruby/e2mmap -# * https://github.com/ruby/mutex_m -# * https://github.com/ruby/racc -# +#!/usr/bin/env ruby +# Sync upstream github repositories to ruby repository. +# See `tool/sync_default_gems.rb --help` for how to use this. require 'fileutils' -include FileUtils - -$repositories = { - rubygems: 'rubygems/rubygems', - bundler: 'bundler/bundler', - rdoc: 'ruby/rdoc', - reline: 'ruby/reline', - json: 'flori/json', - psych: 'ruby/psych', - fileutils: 'ruby/fileutils', - fiddle: 'ruby/fiddle', - stringio: 'ruby/stringio', - ioconsole: 'ruby/io-console', - csv: 'ruby/csv', - webrick: 'ruby/webrick', - dbm: 'ruby/dbm', - gdbm: 'ruby/gdbm', - sdbm: 'ruby/sdbm', - etc: 'ruby/etc', - date: 'ruby/date', - zlib: 'ruby/zlib', - fcntl: 'ruby/fcntl', - scanf: 'ruby/scanf', - cmath: 'ruby/cmath', - strscan: 'ruby/strscan', - ipaddr: 'ruby/ipaddr', - logger: 'ruby/logger', - prime: 'ruby/prime', - matrix: 'ruby/matrix', - ostruct: 'ruby/ostruct', - rexml: 'ruby/rexml', - rss: 'ruby/rss', - irb: 'ruby/irb', - sync: 'ruby/sync', - tracer: 'ruby/tracer', - shell: 'ruby/shell', - forwardable: "ruby/forwardable", - thwait: "ruby/thwait", - e2mmap: "ruby/e2mmap", - mutex_m: "ruby/mutex_m", - racc: "ruby/racc" -} - -def sync_default_gems(gem) - puts "Sync #{$repositories[gem.to_sym]}" - - upstream = File.join("..", "..", $repositories[gem.to_sym]) - - case gem - when "rubygems" - rm_rf(%w[lib/rubygems* test/rubygems]) - cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") - cp_r("#{upstream}/test/rubygems", "test") - when "bundler" - rm_rf(%w[lib/bundler* libexec/bundler libexec/bundle spec/bundler man/bundle* man/gemfile*]) - cp_r(Dir.glob("#{upstream}/lib/bundler*"), "lib") - cp_r(Dir.glob("#{upstream}/exe/bundle*"), "libexec") - cp_r("#{upstream}/bundler.gemspec", "lib/bundler") - cp_r("#{upstream}/spec", "spec/bundler") - cp_r(Dir.glob("#{upstream}/man/*.{1,5,1\.txt,5\.txt,ronn}"), "man") - rm_rf(%w[spec/bundler/support/artifice/vcr_cassettes]) - when "rdoc" - rm_rf(%w[lib/rdoc* test/rdoc libexec/rdoc libexec/ri]) - cp_r(Dir.glob("#{upstream}/lib/rdoc*"), "lib") - cp_r("#{upstream}/test", "test/rdoc") - cp_r("#{upstream}/rdoc.gemspec", "lib/rdoc") - cp_r("#{upstream}/exe/rdoc", "libexec") - cp_r("#{upstream}/exe/ri", "libexec") - rm_rf(%w[lib/rdoc/markdown.kpeg lib/rdoc/markdown/literals.kpeg lib/rdoc/rd/block_parser.ry lib/rdoc/rd/inline_parser.ry]) - `git checkout lib/rdoc/.document` - when "reline" - rm_rf(%w[lib/reline* test/reline]) - cp_r(Dir.glob("#{upstream}/lib/reline*"), "lib") - cp_r("#{upstream}/test", "test/reline") - cp_r("#{upstream}/reline.gemspec", "lib/reline") - when "json" - rm_rf(%w[ext/json test/json]) - cp_r("#{upstream}/ext/json/ext", "ext/json") - cp_r("#{upstream}/tests", "test/json") - cp_r("#{upstream}/lib", "ext/json") - rm_rf(%[ext/json/lib/json/pure*]) - cp_r("#{upstream}/json.gemspec", "ext/json") - rm_rf(%w[ext/json/lib/json/ext]) - `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend` - when "psych" - rm_rf(%w[ext/psych test/psych]) - cp_r("#{upstream}/ext/psych", "ext") - cp_r("#{upstream}/lib", "ext/psych") - cp_r("#{upstream}/test/psych", "test") - rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) - rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) - rm_rf(["ext/psych/yaml/LICENSE"]) - cp_r("#{upstream}/psych.gemspec", "ext/psych") - `git checkout ext/psych/depend` - when "fiddle" - rm_rf(%w[ext/fiddle test/fiddle]) - cp_r("#{upstream}/ext/fiddle", "ext") - cp_r("#{upstream}/lib", "ext/fiddle") - cp_r("#{upstream}/test/fiddle", "test") - cp_r("#{upstream}/fiddle.gemspec", "ext/fiddle") - `git checkout ext/fiddle/depend` - rm_rf(%w[ext/fiddle/lib/fiddle.{bundle,so}]) - when "stringio" - rm_rf(%w[ext/stringio test/stringio]) - cp_r("#{upstream}/ext/stringio", "ext") - cp_r("#{upstream}/test/stringio", "test") - cp_r("#{upstream}/stringio.gemspec", "ext/stringio") - `git checkout ext/stringio/depend ext/stringio/README.md` - when "ioconsole" - rm_rf(%w[ext/io/console test/io/console]) - cp_r("#{upstream}/ext/io/console", "ext/io") - cp_r("#{upstream}/test/io/console", "test/io") - mkdir_p("ext/io/console/lib") - cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") - cp_r("#{upstream}/io-console.gemspec", "ext/io/console") - `git checkout ext/io/console/depend` - when "dbm" - rm_rf(%w[ext/dbm test/dbm]) - cp_r("#{upstream}/ext/dbm", "ext") - cp_r("#{upstream}/test/dbm", "test") - cp_r("#{upstream}/dbm.gemspec", "ext/dbm") - `git checkout ext/dbm/depend` - when "gdbm" - rm_rf(%w[ext/gdbm test/gdbm]) - cp_r("#{upstream}/ext/gdbm", "ext") - cp_r("#{upstream}/test/gdbm", "test") - cp_r("#{upstream}/gdbm.gemspec", "ext/gdbm") - `git checkout ext/gdbm/depend ext/gdbm/README` - when "sdbm" - rm_rf(%w[ext/sdbm test/sdbm]) - cp_r("#{upstream}/ext/sdbm", "ext") - cp_r("#{upstream}/test/sdbm", "test") - cp_r("#{upstream}/sdbm.gemspec", "ext/sdbm") - `git checkout ext/sdbm/depend` - when "etc" - rm_rf(%w[ext/etc test/etc]) - cp_r("#{upstream}/ext/etc", "ext") - cp_r("#{upstream}/test/etc", "test") - cp_r("#{upstream}/etc.gemspec", "ext/etc") - `git checkout ext/etc/depend` - when "date" - rm_rf(%w[ext/date test/date]) - cp_r("#{upstream}/ext/date", "ext") - cp_r("#{upstream}/lib", "ext/date") - cp_r("#{upstream}/test/date", "test") - cp_r("#{upstream}/date.gemspec", "ext/date") - `git checkout ext/date/depend` - rm_rf(["ext/date/lib/date_core.bundle"]) - when "zlib" - rm_rf(%w[ext/zlib test/zlib]) - cp_r("#{upstream}/ext/zlib", "ext") - cp_r("#{upstream}/test/zlib", "test") - cp_r("#{upstream}/zlib.gemspec", "ext/zlib") - `git checkout ext/zlib/depend` - when "fcntl" - rm_rf(%w[ext/fcntl]) - cp_r("#{upstream}/ext/fcntl", "ext") - cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") - `git checkout ext/fcntl/depend` - when "thwait" - rm_rf(%w[lib/thwait*]) - cp_r(Dir.glob("#{upstream}/lib/*"), "lib") - cp_r("#{upstream}/thwait.gemspec", "lib/thwait") - when "e2mmap" - rm_rf(%w[lib/e2mmap*]) - cp_r(Dir.glob("#{upstream}/lib/*"), "lib") - cp_r("#{upstream}/e2mmap.gemspec", "lib") - when "strscan" - rm_rf(%w[ext/strscan test/strscan]) - cp_r("#{upstream}/ext/strscan", "ext") - cp_r("#{upstream}/test/strscan", "test") - cp_r("#{upstream}/strscan.gemspec", "ext/strscan") - rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) - `git checkout ext/strscan/depend` - when "racc" - rm_rf(%w[lib/racc* ext/racc test/racc]) - cp_r(Dir.glob("#{upstream}/lib/racc*"), "lib") - mkdir_p("ext/racc/cparse") - cp_r(Dir.glob("#{upstream}/ext/racc/cparse/*"), "ext/racc/cparse") - cp_r("#{upstream}/test", "test/racc") - `git checkout ext/racc/cparse/README` - when "rexml", "rss", "matrix", "irb", "csv", "shell", "logger", "ostruct", "scanf", "webrick", "fileutils", "forwardable", "prime", "tracer", "ipaddr", "cmath", "mutex_m", "sync" - sync_lib gem - else +require "rbconfig" +require "find" +require "tempfile" + +module SyncDefaultGems + include FileUtils + extend FileUtils + + module_function + + # upstream: "owner/repo" + # branch: "branch_name" + # mappings: [ ["path_in_upstream", "path_in_ruby"], ... ] + # NOTE: path_in_ruby is assumed to be "owned" by this gem, and the contents + # will be removed before sync + # exclude: [ "fnmatch_pattern_after_mapping", ... ] + Repository = Data.define(:upstream, :branch, :mappings, :exclude) do + def excluded?(newpath) + p = newpath + until p == "." + return true if exclude.any? {|pat| File.fnmatch?(pat, p, File::FNM_PATHNAME|File::FNM_EXTGLOB)} + p = File.dirname(p) + end + false + end + + def rewrite_for_ruby(path) + newpath = mappings.find do |src, dst| + if path == src || path.start_with?(src + "/") + break path.sub(src, dst) + end + end + return nil unless newpath + return nil if excluded?(newpath) + newpath + end + end + + CLASSICAL_DEFAULT_BRANCH = "master" + + def repo((upstream, branch), mappings, exclude: []) + branch ||= CLASSICAL_DEFAULT_BRANCH + exclude += ["ext/**/depend"] + Repository.new(upstream:, branch:, mappings:, exclude:) + end + + def lib((upstream, branch), gemspec_in_subdir: false) + _org, name = upstream.split("/") + gemspec_dst = gemspec_in_subdir ? "lib/#{name}/#{name}.gemspec" : "lib/#{name}.gemspec" + repo([upstream, branch], [ + ["lib/#{name}.rb", "lib/#{name}.rb"], + ["lib/#{name}", "lib/#{name}"], + ["test/test_#{name}.rb", "test/test_#{name}.rb"], + ["test/#{name}", "test/#{name}"], + ["#{name}.gemspec", gemspec_dst], + ]) end -end -def sync_default_gems_with_commits(gem, range) - puts "Sync #{$repositories[gem.to_sym]} with commit history." + # Note: tool/auto_review_pr.rb also depends on these constants. + NO_UPSTREAM = [ + "lib/unicode_normalize", # not to match with "lib/un" + ] + REPOSITORIES = { + Onigmo: repo("k-takata/Onigmo", [ + ["regcomp.c", "regcomp.c"], + ["regenc.c", "regenc.c"], + ["regenc.h", "regenc.h"], + ["regerror.c", "regerror.c"], + ["regexec.c", "regexec.c"], + ["regint.h", "regint.h"], + ["regparse.c", "regparse.c"], + ["regparse.h", "regparse.h"], + ["regsyntax.c", "regsyntax.c"], + ["onigmo.h", "include/ruby/onigmo.h"], + ["enc", "enc"], + ]), + "io-console": repo("ruby/io-console", [ + ["ext/io/console", "ext/io/console"], + ["test/io/console", "test/io/console"], + ["lib/io/console", "ext/io/console/lib/console"], + ["io-console.gemspec", "ext/io/console/io-console.gemspec"], + ]), + "io-nonblock": repo("ruby/io-nonblock", [ + ["ext/io/nonblock", "ext/io/nonblock"], + ["test/io/nonblock", "test/io/nonblock"], + ["io-nonblock.gemspec", "ext/io/nonblock/io-nonblock.gemspec"], + ]), + "io-wait": repo("ruby/io-wait", [ + ["ext/io/wait", "ext/io/wait"], + ["test/io/wait", "test/io/wait"], + ["io-wait.gemspec", "ext/io/wait/io-wait.gemspec"], + ]), + "net-http": repo("ruby/net-http", [ + ["lib/net/http.rb", "lib/net/http.rb"], + ["lib/net/http", "lib/net/http"], + ["test/net/http", "test/net/http"], + ["net-http.gemspec", "lib/net/http/net-http.gemspec"], + ]), + "net-protocol": repo("ruby/net-protocol", [ + ["lib/net/protocol.rb", "lib/net/protocol.rb"], + ["test/net/protocol", "test/net/protocol"], + ["net-protocol.gemspec", "lib/net/net-protocol.gemspec"], + ]), + "open-uri": lib("ruby/open-uri"), + "win32-registry": repo("ruby/win32-registry", [ + ["lib/win32/registry.rb", "ext/win32/lib/win32/registry.rb"], + ["test/win32/test_registry.rb", "test/win32/test_registry.rb"], + ["win32-registry.gemspec", "ext/win32/win32-registry.gemspec"], + ]), + English: lib("ruby/English"), + cgi: repo("ruby/cgi", [ + ["ext/cgi", "ext/cgi"], + ["lib/cgi/escape.rb", "lib/cgi/escape.rb"], + ["test/cgi/test_cgi_escape.rb", "test/cgi/test_cgi_escape.rb"], + ["test/cgi/update_env.rb", "test/cgi/update_env.rb"], + ]), + date: repo("ruby/date", [ + ["doc/date", "doc/date"], + ["ext/date", "ext/date"], + ["lib", "ext/date/lib"], + ["test/date", "test/date"], + ["date.gemspec", "ext/date/date.gemspec"], + ], exclude: [ + "ext/date/lib/date_core.bundle", + ]), + delegate: lib("ruby/delegate"), + did_you_mean: repo("ruby/did_you_mean", [ + ["lib/did_you_mean.rb", "lib/did_you_mean.rb"], + ["lib/did_you_mean", "lib/did_you_mean"], + ["test", "test/did_you_mean"], + ["did_you_mean.gemspec", "lib/did_you_mean/did_you_mean.gemspec"], + ], exclude: [ + "test/did_you_mean/lib", + "test/did_you_mean/tree_spell/test_explore.rb", + ]), + digest: repo("ruby/digest", [ + ["ext/digest/lib/digest/sha2", "ext/digest/sha2/lib/sha2"], + ["ext/digest", "ext/digest"], + ["lib/digest.rb", "ext/digest/lib/digest.rb"], + ["lib/digest/version.rb", "ext/digest/lib/digest/version.rb"], + ["lib/digest/sha2.rb", "ext/digest/sha2/lib/sha2.rb"], + ["test/digest", "test/digest"], + ["digest.gemspec", "ext/digest/digest.gemspec"], + ]), + erb: repo("ruby/erb", [ + ["ext/erb", "ext/erb"], + ["lib/erb", "lib/erb"], + ["lib/erb.rb", "lib/erb.rb"], + ["test/erb", "test/erb"], + ["erb.gemspec", "lib/erb/erb.gemspec"], + ["libexec/erb", "libexec/erb"], + ]), + error_highlight: repo("ruby/error_highlight", [ + ["lib/error_highlight.rb", "lib/error_highlight.rb"], + ["lib/error_highlight", "lib/error_highlight"], + ["test", "test/error_highlight"], + ["error_highlight.gemspec", "lib/error_highlight/error_highlight.gemspec"], + ]), + etc: repo("ruby/etc", [ + ["ext/etc", "ext/etc"], + ["test/etc", "test/etc"], + ["etc.gemspec", "ext/etc/etc.gemspec"], + ]), + fcntl: repo("ruby/fcntl", [ + ["ext/fcntl", "ext/fcntl"], + ["fcntl.gemspec", "ext/fcntl/fcntl.gemspec"], + ]), + fileutils: lib("ruby/fileutils"), + find: lib("ruby/find"), + forwardable: lib("ruby/forwardable", gemspec_in_subdir: true), + ipaddr: lib("ruby/ipaddr"), + json: repo("ruby/json", [ + ["ext/json/ext", "ext/json"], + ["test/json", "test/json"], + ["lib", "ext/json/lib"], + ["json.gemspec", "ext/json/json.gemspec"], + ], exclude: [ + "ext/json/lib/json/ext/.keep", + "ext/json/lib/json/pure.rb", + "ext/json/lib/json/pure", + "ext/json/lib/json/truffle_ruby", + "test/json/lib", + "ext/json/extconf.rb", + ]), + mmtk: repo(["ruby/mmtk", "main"], [ + ["gc/mmtk", "gc/mmtk"], + ]), + open3: lib("ruby/open3", gemspec_in_subdir: true).tap { + it.exclude << "lib/open3/jruby_windows.rb" + }, + openssl: repo("ruby/openssl", [ + ["ext/openssl", "ext/openssl"], + ["lib", "ext/openssl/lib"], + ["test/openssl", "test/openssl"], + ["sample", "sample/openssl"], + ["openssl.gemspec", "ext/openssl/openssl.gemspec"], + ["History.md", "ext/openssl/History.md"], + ], exclude: [ + "test/openssl/envutil.rb", + "ext/openssl/depend", + ]), + optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap { + it.mappings << ["doc/optparse", "doc/optparse"] + }, + pathname: repo("ruby/pathname", [ + ["ext/pathname/pathname.c", "pathname.c"], + ["lib/pathname_builtin.rb", "pathname_builtin.rb"], + ["lib/pathname.rb", "lib/pathname.rb"], + ["test/pathname", "test/pathname"], + ]), + pp: lib("ruby/pp"), + prettyprint: lib("ruby/prettyprint"), + prism: repo(["ruby/prism", "main"], [ + ["ext/prism", "prism"], + ["lib/prism.rb", "lib/prism.rb"], + ["lib/prism", "lib/prism"], + ["test/prism", "test/prism"], + ["src", "prism"], + ["prism.gemspec", "lib/prism/prism.gemspec"], + ["include/prism", "prism"], + ["include/prism.h", "prism/prism.h"], + ["config.yml", "prism/config.yml"], + ["templates", "prism/templates"], + ], exclude: [ + "prism/templates/{javascript,java,rbi,sig}", + "test/prism/snapshots_test.rb", + "test/prism/snapshots", + "prism/extconf.rb", + "prism/srcs.mk*", + ]), + psych: repo("ruby/psych", [ + ["ext/psych", "ext/psych"], + ["lib", "ext/psych/lib"], + ["test/psych", "test/psych"], + ["psych.gemspec", "ext/psych/psych.gemspec"], + ], exclude: [ + "ext/psych/lib/org", + "ext/psych/lib/psych.jar", + "ext/psych/lib/psych_jars.rb", + "ext/psych/lib/psych.{bundle,so}", + "ext/psych/lib/2.*", + "ext/psych/yaml/LICENSE", + "ext/psych/.gitignore", + ]), + resolv: repo("ruby/resolv", [ + ["lib/resolv.rb", "lib/resolv.rb"], + ["test/resolv", "test/resolv"], + ["resolv.gemspec", "lib/resolv.gemspec"], + ["ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32/resolv.rb"], + ["ext/win32/resolv", "ext/win32/resolv"], + ]), + rubygems: repo("ruby/rubygems", [ + ["lib/rubygems.rb", "lib/rubygems.rb"], + ["lib/rubygems", "lib/rubygems"], + ["test/rubygems", "test/rubygems"], + ["bundler/lib/bundler.rb", "lib/bundler.rb"], + ["bundler/lib/bundler", "lib/bundler"], + ["bundler/exe/bundle", "libexec/bundle"], + ["bundler/exe/bundler", "libexec/bundler"], + ["bundler/bundler.gemspec", "lib/bundler/bundler.gemspec"], + ["bundler/spec", "spec/bundler"], + *["bundle", "parallel_rspec", "rspec"].map {|binstub| + ["bundler/bin/#{binstub}", "spec/bin/#{binstub}"] + }, + *%w[dev_gems test_gems rubocop_gems standard_gems].flat_map {|gemfile| + ["rb.lock", "rb"].map do |ext| + ["tool/bundler/#{gemfile}.#{ext}", "tool/bundler/#{gemfile}.#{ext}"] + end + }, + ], exclude: [ + "spec/bundler/bin", + "spec/bundler/support/artifice/vcr_cassettes", + "spec/bundler/support/artifice/used_cassettes.txt", + "lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}", + ]), + securerandom: lib("ruby/securerandom"), + shellwords: lib("ruby/shellwords"), + singleton: lib("ruby/singleton"), + stringio: repo("ruby/stringio", [ + ["ext/stringio", "ext/stringio"], + ["test/stringio", "test/stringio"], + ["stringio.gemspec", "ext/stringio/stringio.gemspec"], + ["doc/stringio", "doc/stringio"], + ], exclude: [ + "ext/stringio/README.md", + ]), + strscan: repo("ruby/strscan", [ + ["ext/strscan", "ext/strscan"], + ["lib", "ext/strscan/lib"], + ["test/strscan", "test/strscan"], + ["strscan.gemspec", "ext/strscan/strscan.gemspec"], + ["doc/strscan", "doc/strscan"], + ], exclude: [ + "ext/strscan/regenc.h", + "ext/strscan/regint.h", + ]), + syntax_suggest: lib(["ruby/syntax_suggest", "main"], gemspec_in_subdir: true), + tempfile: lib("ruby/tempfile"), + time: lib("ruby/time"), + timeout: lib("ruby/timeout"), + tmpdir: lib("ruby/tmpdir"), + un: lib("ruby/un"), + uri: lib("ruby/uri", gemspec_in_subdir: true), + weakref: lib("ruby/weakref"), + yaml: lib("ruby/yaml", gemspec_in_subdir: true), + zlib: repo("ruby/zlib", [ + ["ext/zlib", "ext/zlib"], + ["test/zlib", "test/zlib"], + ["zlib.gemspec", "ext/zlib/zlib.gemspec"], + ]), + }.transform_keys(&:to_s) - IO.popen(%W"git remote") do |f| - unless f.read.split.include?(gem) - `git remote add #{gem} git@github.com:#{$repositories[gem.to_sym]}.git` - `git fetch #{gem}` + class << Repository + def find_upstream(file) + return if NO_UPSTREAM.any? {|dst| file.start_with?(dst) } + REPOSITORIES.find do |repo_name, repository| + if repository.mappings.any? {|_src, dst| file.start_with?(dst) } + break repo_name + end + end + end + + def group(files) + files.group_by {|file| find_upstream(file)} end end - IO.popen(%W"git log --format=%H,%s #{range}") do |f| - commits = f.read.split("\n").reverse.map{|commit| commit.split(',')} - commits.each do |sha, subject| - puts "Pick #{sha} from #{$repositories[gem.to_sym]}." - if subject =~ /^Merge/ - puts "Skip #{sha}. Because It was merge commit" - next + # Allow synchronizing commits up to this FETCH_DEPTH. We've historically merged PRs + # with about 250 commits to ruby/ruby, so we use this depth for ruby/ruby in general. + FETCH_DEPTH = 500 + + def pipe_readlines(args, rs: "\0", chomp: true) + IO.popen(args) do |f| + f.readlines(rs, chomp: chomp) + end + end + + def porcelain_status(*pattern) + pipe_readlines(%W"git status --porcelain --no-renames -z --" + pattern) + end + + def replace_rdoc_ref(file) + src = File.binread(file) + changed = false + changed |= src.gsub!(%r[\[\Khttps://docs\.ruby-lang\.org/en/master(?:/doc)?/(([A-Z]\w+(?:/[A-Z]\w+)*)|\w+_rdoc)\.html(\#\S+)?(?=\])]) do + name, mod, label = $1, $2, $3 + mod &&= mod.gsub('/', '::') + if label && (m = label.match(/\A\#(?:method-([ci])|(?:(?:class|module)-#{mod}-)?label)-([-+\w]+)\z/)) + scope, label = m[1], m[2] + scope = scope ? scope.tr('ci', '.#') : '@' end + "rdoc-ref:#{mod || name.chomp("_rdoc") + ".rdoc"}#{scope}#{label}" + end + changed or return false + File.binwrite(file, src) + return true + end + + def replace_rdoc_ref_all + result = porcelain_status("*.c", "*.rb", "*.rdoc") + result.map! {|line| line[/\A.M (.*)/, 1]} + result.compact! + return if result.empty? + result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + result) + result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def replace_rdoc_ref_all_full + Dir.glob("**/*.{c,rb,rdoc}").inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end - `git cherry-pick #{sha}` - unless $?.success? - puts "Failed to pick #{sha}" - break + def rubygems_do_fixup + gemspec_content = File.readlines("lib/bundler/bundler.gemspec").map do |line| + next if line =~ /LICENSE\.md/ + + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") + end.compact.join + File.write("lib/bundler/bundler.gemspec", gemspec_content) + + ["bundle", "parallel_rspec", "rspec"].each do |binstub| + path = "spec/bin/#{binstub}" + next unless File.exist?(path) + content = File.read(path).gsub("../spec", "../bundler") + File.write(path, content) + chmod("+x", path) + end + end + + # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. + def sync_default_gems(gem) + config = REPOSITORIES[gem] + puts "Sync #{config.upstream}" + + upstream = File.join("..", "..", config.upstream) + + config.mappings.each do |src, dst| + rm_rf(dst) + end + + copied = Set.new + config.mappings.each do |src, dst| + prefix = File.join(upstream, src) + # Maybe mapping needs to be updated? + next unless File.exist?(prefix) + Find.find(prefix) do |path| + next if File.directory?(path) + if copied.add?(path) + newpath = config.rewrite_for_ruby(path.sub(%r{\A#{Regexp.escape(upstream)}/}, "")) + next unless newpath + mkdir_p(File.dirname(newpath)) + cp(path, newpath) + end end + end - prefix = "[#{($repositories[gem.to_sym])}]".gsub(/\//, '\/') - suffix = "https://github.com/#{($repositories[gem.to_sym])}/commit/#{sha[0,10]}" - `git filter-branch -f --msg-filter 'sed "1s/^/#{prefix} /" && echo && echo #{suffix}' -- HEAD~1..HEAD` - unless $?.success? - puts "Failed to modify commit message of #{sha}" - break + porcelain_status().each do |line| + /\A(?:.)(?:.) (?<path>.*)\z/ =~ line or raise + if config.excluded?(path) + puts "Restoring excluded file: #{path}" + IO.popen(%W"git checkout --" + [path], "rb", &:read) end end + + # RubyGems/Bundler needs special care + if gem == "rubygems" + rubygems_do_fixup + end + + check_prerelease_version(gem) + + # Architecture-dependent files must not pollute libdir. + rm_rf(Dir["lib/**/*.#{RbConfig::CONFIG['DLEXT']}"]) + replace_rdoc_ref_all end -end -def sync_lib(repo) - unless File.directory?("../#{repo}") - abort "Expected '../#{repo}' (#{File.expand_path("../#{repo}")}) to be a directory, but it wasn't." + def check_prerelease_version(gem) + return if ["rubygems", "mmtk", "cgi", "pathname", "Onigmo"].include?(gem) + + require "net/https" + require "json" + require "uri" + + uri = URI("https://rubygems.org/api/v1/versions/#{gem.downcase}/latest.json") + response = Net::HTTP.get(uri) + latest_version = JSON.parse(response)["version"] + + gemspec = [ + "lib/#{gem}/#{gem}.gemspec", + "lib/#{gem}.gemspec", + "ext/#{gem}/#{gem}.gemspec", + "ext/#{gem.split("-").join("/")}/#{gem}.gemspec", + "lib/#{gem.split("-").first}/#{gem}.gemspec", + "ext/#{gem.split("-").first}/#{gem}.gemspec", + "lib/#{gem.split("-").join("/")}/#{gem}.gemspec", + ].find{|gemspec| File.exist?(gemspec)} + spec = Gem::Specification.load(gemspec) + puts "#{gem}-#{spec.version} is not latest version of rubygems.org" if spec.version.to_s != latest_version + end + + def message_filter(repo, sha, log, context: nil) + unless repo.count("/") == 1 and /\A\S+\z/ =~ repo + raise ArgumentError, "invalid repository: #{repo}" + end + unless /\A\h{10,40}\z/ =~ sha + raise ArgumentError, "invalid commit-hash: #{sha}" + end + repo_url = "https://github.com/#{repo}" + + # Log messages generated by GitHub web UI have inconsistent line endings + log = log.delete("\r") + log << "\n" if !log.end_with?("\n") + + # Split the subject from the log message according to git conventions. + # SPECIAL TREAT: when the first line ends with a dot `.` (which is not + # obeying the conventions too), takes only that line. + subject, log = log.split(/\A.+\.\K\n(?=\S)|\n(?:[ \t]*(?:\n|\z))/, 2) + conv = proc do |s| + mod = true if s.gsub!(/\b(?:(?i:fix(?:e[sd])?|close[sd]?|resolve[sd]?) +)\K#(?=\d+\b)|\bGH-#?(?=\d+\b)|\(\K#(?=\d+\))/) { + "#{repo_url}/pull/" + } + mod |= true if s.gsub!(%r{(?<![-\[\](){}\w@/])(?:(\w+(?:-\w+)*/\w+(?:-\w+)*)@)?(\h{10,40})\b}) {|c| + "https://github.com/#{$1 || repo}/commit/#{$2[0,12]}" + } + mod + end + subject = "[#{repo}] #{subject}" + subject.gsub!(/\s*\n\s*/, " ") + if conv[subject] + if subject.size > 68 + subject.gsub!(/\G.{,67}[^\s.,][.,]*\K\s+/, "\n") + end + end + commit_url = "#{repo_url}/commit/#{sha[0,10]}\n" + sync_note = context ? "#{commit_url}\n#{context}" : commit_url + if log and !log.empty? + log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last + conv[log] + log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) { + ($~.begin(1) ? "" : "\n\n") + sync_note + ($~.begin(2) ? "\n" : "") + } + else + log = sync_note + end + "#{subject}\n\n#{log}" + end + + def log_format(format, args, &block) + IO.popen(%W[git -c core.autocrlf=false -c core.eol=lf + log --no-show-signature --format=#{format}] + args, "rb", &block) + end + + def commits_in_range(upto, exclude, toplevel:) + args = [upto, *exclude.map {|s|"^#{s}"}] + log_format('%H,%P,%s', %W"--first-parent" + args) do |f| + f.read.split("\n").reverse.flat_map {|commit| + hash, parents, subject = commit.split(',', 3) + parents = parents.split + + # Non-merge commit + if parents.size <= 1 + puts "#{hash} #{subject}" + next [[hash, subject]] + end + + # Clean 2-parent merge commit: follow the other parent as long as it + # contains no potentially-non-clean merges + if parents.size == 2 && + IO.popen(%W"git diff-tree --remerge-diff #{hash}", "rb", &:read).empty? + puts "\e[2mChecking the other parent of #{hash} #{subject}\e[0m" + ret = catch(:quit) { + commits_in_range(parents[1], exclude + [parents[0]], toplevel: false) + } + next ret if ret + end + + unless toplevel + puts "\e[1mMerge commit with possible conflict resolution #{hash} #{subject}\e[0m" + throw :quit + end + + puts "#{hash} #{subject} " \ + "\e[1m[merge commit with possible conflicts, will do a squash merge]\e[0m" + [[hash, subject]] + } + end end - rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) - cp_r(Dir.glob("../#{repo}/lib/*"), "lib") - tests = if File.directory?("test/#{repo}") - "test/#{repo}" - else - "test/test_#{repo}.rb" - end - cp_r("../#{repo}/#{tests}", "test") - gemspec = if File.directory?("lib/#{repo}") - "lib/#{repo}/#{repo}.gemspec" + + # Returns commit list as array of [commit_hash, subject, sync_note]. + def commits_in_ranges(ranges) + ranges.flat_map do |range| + exclude, upto = range.include?("..") ? range.split("..", 2) : ["#{range}~1", range] + puts "Looking for commits in range #{exclude}..#{upto}" + commits_in_range(upto, exclude.empty? ? [] : [exclude], toplevel: true) + end.uniq + end + + #-- + # Following methods used by sync_default_gems_with_commits return + # true: success + # false: skipped + # nil: failed + #++ + + def resolve_conflicts(gem, sha, edit) + # Discover unmerged files: any unstaged changes + changes = porcelain_status() + conflict = changes.grep(/\A(?:.[^ ?]) /) {$'} + # If -e option is given, open each conflicted file with an editor + unless conflict.empty? + if edit + case + when (editor = ENV["GIT_EDITOR"] and !editor.empty?) + when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) + end + if editor + system([editor, conflict].join(' ')) + conflict.delete_if {|f| !File.exist?(f)} + return true if conflict.empty? + return system(*%w"git add --", *conflict) + end + end + return false + end + + return true + end + + def collect_cacheinfo(tree) + pipe_readlines(%W"git ls-tree -r -t -z #{tree}").filter_map do |line| + fields, path = line.split("\t", 2) + mode, type, object = fields.split(" ", 3) + next unless type == "blob" + [mode, type, object, path] + end + end + + def rewrite_cacheinfo(gem, blobs) + config = REPOSITORIES[gem] + rewritten = [] + ignored = blobs.dup + ignored.delete_if do |mode, type, object, path| + newpath = config.rewrite_for_ruby(path) + next unless newpath + rewritten << [mode, type, object, newpath] + end + [rewritten, ignored] + end + + def make_commit_info(gem, sha) + config = REPOSITORIES[gem] + headers, orig = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + /^author (?<author_name>.+?) <(?<author_email>.*?)> (?<author_date>.+?)$/ =~ headers or + raise "unable to parse author info for commit #{sha}" + author = { + "GIT_AUTHOR_NAME" => author_name, + "GIT_AUTHOR_EMAIL" => author_email, + "GIT_AUTHOR_DATE" => author_date, + } + context = nil + if /^parent (?<first_parent>.{40})\nparent .{40}$/ =~ headers + # Squashing a merge commit: keep authorship information + context = IO.popen(%W"git shortlog #{first_parent}..#{sha} --", "rb", &:read) + end + message = message_filter(config.upstream, sha, orig, context: context) + [author, message] + end + + def fixup_commit(gem, commit) + wt = File.join("tmp", "sync_default_gems-fixup-worktree") + if File.directory?(wt) + IO.popen(%W"git -C #{wt} clean -xdf", "rb", &:read) + IO.popen(%W"git -C #{wt} reset --hard #{commit}", "rb", &:read) + else + IO.popen(%W"git worktree remove --force #{wt}", "rb", err: File::NULL, &:read) + IO.popen(%W"git worktree add --detach #{wt} #{commit}", "rb", &:read) + end + raise "git worktree prepare failed for commit #{commit}" unless $?.success? + + Dir.chdir(wt) do + if gem == "rubygems" + rubygems_do_fixup + end + replace_rdoc_ref_all_full + end + + IO.popen(%W"git -C #{wt} add -u", "rb", &:read) + IO.popen(%W"git -C #{wt} commit --amend --no-edit", "rb", &:read) + IO.popen(%W"git -C #{wt} rev-parse HEAD", "rb", &:read).chomp + end + + def make_and_fixup_commit(gem, original_commit, cacheinfo, parent: nil, message: nil, author: nil) + tree = Tempfile.create("sync_default_gems-#{gem}-index") do |f| + File.unlink(f.path) + IO.popen({"GIT_INDEX_FILE" => f.path}, + %W"git update-index --index-info", "wb", out: IO::NULL) do |io| + cacheinfo.each do |mode, type, object, path| + io.puts("#{mode} #{type} #{object}\t#{path}") + end + end + raise "git update-index failed" unless $?.success? + + IO.popen({"GIT_INDEX_FILE" => f.path}, %W"git write-tree --missing-ok", "rb", &:read).chomp + end + + args = ["-m", message || "Rewriten commit for #{original_commit}"] + args += ["-p", parent] if parent + commit = IO.popen({**author}, %W"git commit-tree #{tree}" + args, "rb", &:read).chomp + + # Apply changes that require a working tree + commit = fixup_commit(gem, commit) + + commit + end + + def rewrite_commit(gem, sha) + author, message = make_commit_info(gem, sha) + new_blobs = collect_cacheinfo("#{sha}") + new_rewritten, new_ignored = rewrite_cacheinfo(gem, new_blobs) + + headers, _ = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + first_parent = headers[/^parent (.{40})$/, 1] + unless first_parent + # Root commit, first time to sync this repo + return make_and_fixup_commit(gem, sha, new_rewritten, message: message, author: author) + end + + old_blobs = collect_cacheinfo(first_parent) + old_rewritten, old_ignored = rewrite_cacheinfo(gem, old_blobs) + if old_ignored != new_ignored + paths = (old_ignored + new_ignored - (old_ignored & new_ignored)) + .map {|*_, path| path}.uniq + puts "\e\[1mIgnoring file changes not in mappings: #{paths.join(" ")}\e\[0m" + end + changed_paths = (old_rewritten + new_rewritten - (old_rewritten & new_rewritten)) + .map {|*_, path| path}.uniq + if changed_paths.empty? + puts "Skip commit only for tools or toplevel" + return false + end + + # Build commit objects from "cacheinfo" + new_parent = make_and_fixup_commit(gem, first_parent, old_rewritten) + new_commit = make_and_fixup_commit(gem, sha, new_rewritten, parent: new_parent, message: message, author: author) + puts "Created a temporary commit for cherry-pick: #{new_commit}" + new_commit + end + + def pickup_commit(gem, sha, edit) + rewritten = rewrite_commit(gem, sha) + + # No changes remaining after rewriting + return false unless rewritten + + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{rewritten}", "rb", err: [:child, :out], &:read) + unless $?.success? + if result =~ /The previous cherry-pick is now empty/ + system(*%w"git cherry-pick --skip") + puts "Skip empty commit #{sha}" + return false + end + + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless resolve_conflicts(gem, sha, edit) + system(*%w"git --no-pager diff") if !edit # If failed, show `git diff` unless editing + `git reset` && `git checkout .` && `git clean -fd` # Clean up un-committed diffs + return nil # Fail unless cherry-picked + end + + # Commit cherry-picked commit + if porcelain_status().empty? + system(*%w"git cherry-pick --skip") + return false + else + system(*%w"git cherry-pick --continue --no-edit") + return nil unless $?.success? + end + end + + new_head = IO.popen(%W"git rev-parse HEAD", "rb", &:read).chomp + puts "Committed cherry-pick as #{new_head}" + return true + end + + # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. + # @param ranges [Array<String>, true] "commit", "before..after", or true. Note that it will NOT sync "before" (but commits after that). + # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. + def sync_default_gems_with_commits(gem, ranges, edit: nil) + config = REPOSITORIES[gem] + repo, default_branch = config.upstream, config.branch + puts "Sync #{repo} with commit history." + + # Fetch the repository to be synchronized + IO.popen(%W"git remote") do |f| + unless f.read.split.include?(gem) + `git remote add #{gem} https://github.com/#{repo}.git` + end + end + system(*%W"git fetch --no-tags --depth=#{FETCH_DEPTH} #{gem} #{default_branch}") + + # If -a is given, discover all commits since the last picked commit + if ranges == true + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" + log = log_format('%B', %W"-E --grep=#{pattern} -n1 --", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] + end + commits = commits_in_ranges(ranges) + if commits.empty? + puts "No commits to pick" + return true + end + + failed_commits = [] + commits.each do |sha, subject| + puts "----" + puts "Pick #{sha} #{subject}" + case pickup_commit(gem, sha, edit) + when false + # skipped + when nil + failed_commits << [sha, subject] + end + end + + unless failed_commits.empty? + puts "---- failed commits ----" + failed_commits.each do |sha, subject| + puts "#{sha} #{subject}" + end + return false + end + return true + end + + def sync_lib(repo, upstream = nil) + unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") + abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] + end + rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) + cp_r(Dir.glob("#{upstream}/lib/*"), "lib") + tests = if File.directory?("test/#{repo}") + "test/#{repo}" else - "lib/#{repo}.gemspec" + "test/test_#{repo}.rb" end - cp_r("../#{repo}/#{repo}.gemspec", "#{gemspec}") -end + cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") + gemspec = if File.directory?("lib/#{repo}") + "lib/#{repo}/#{repo}.gemspec" + else + "lib/#{repo}.gemspec" + end + cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") + end -def update_default_gems(gem) - author, repository = $repositories[gem.to_sym].split('/') + def update_default_gems(gem, release: false) + config = REPOSITORIES[gem] + author, repository = config.upstream.split('/') + default_branch = config.branch - unless File.exist?("../../#{author}/#{repository}") - mkdir_p("../../#{author}") - `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` - end + puts "Update #{author}/#{repository}" + + unless File.exist?("../../#{author}/#{repository}") + mkdir_p("../../#{author}") + `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` + end - Dir.chdir("../../#{author}/#{repository}") do - unless `git remote`.match(/ruby\-core/) - `git remote add ruby-core git@github.com:ruby/ruby.git` - `git fetch ruby-core` - `git co ruby-core/trunk` - `git branch ruby-core` - end - `git co ruby-core` - `git fetch ruby-core trunk` - `git rebase ruby-core/trunk` - `git co master` - `git stash` - `git pull --rebase` + Dir.chdir("../../#{author}/#{repository}") do + unless `git remote`.match(/ruby\-core/) + `git remote add ruby-core git@github.com:ruby/ruby.git` + end + `git fetch ruby-core master --no-tags` + unless `git branch`.match(/ruby\-core/) + `git checkout ruby-core/master` + `git branch ruby-core` + end + `git checkout ruby-core` + `git rebase ruby-core/master` + `git fetch origin --tags` + + if release + last_release = `git tag | sort -V`.chomp.split.delete_if{|v| v =~ /pre|beta/ }.last + `git checkout #{last_release}` + else + `git checkout #{default_branch}` + `git rebase origin/#{default_branch}` + end + end end -end -case ARGV[0] -when "up" - $repositories.keys.each{|gem| update_default_gems(gem.to_s)} -when "all" - $repositories.keys.each{|gem| sync_default_gems(gem.to_s)} -else - if ARGV[1] - sync_default_gems_with_commits(ARGV[0], ARGV[1]) + case ARGV[0] + when "up" + if ARGV[1] + update_default_gems(ARGV[1]) + else + REPOSITORIES.each_key {|gem| update_default_gems(gem)} + end + when "all" + REPOSITORIES.each_key do |gem| + next if ["Onigmo"].include?(gem) + update_default_gems(gem, release: true) if ARGV[1] == "release" + sync_default_gems(gem) + end + when "list" + ARGV.shift + pattern = Regexp.new(ARGV.join('|')) + REPOSITORIES.each do |gem, config| + next unless pattern =~ gem or pattern =~ config.upstream + printf "%-15s https://github.com/%s\n", gem, config.upstream + end + when "rdoc-ref" + ARGV.shift + pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV + result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + pattern) + result.inject(false) do |changed, file| + if replace_rdoc_ref(file) + puts "replaced rdoc-ref in #{file}" + changed = true + end + changed + end + when nil, "-h", "--help" + puts <<-HELP +\e[1mSync with upstream code of default libraries\e[0m + +\e[1mImport all default gems through `git clone` and `cp -rf` (git commits are lost)\e[0m + ruby #$0 all + +\e[1mImport all released version of default gems\e[0m + ruby #$0 all release + +\e[1mImport a default gem with specific gem same as all command\e[0m + ruby #$0 rubygems + +\e[1mPick a single commit from the upstream repository\e[0m + ruby #$0 rubygems 97e9768612 + +\e[1mPick a commit range from the upstream repository\e[0m + ruby #$0 rubygems 97e9768612..9e53702832 + +\e[1mPick all commits since the last picked commit\e[0m + ruby #$0 -a rubygems + +\e[1mUpdate repositories of default gems\e[0m + ruby #$0 up + +\e[1mList known libraries\e[0m + ruby #$0 list + +\e[1mList known libraries matching with patterns\e[0m + ruby #$0 list read + HELP + + exit else - sync_default_gems(ARGV[0]) - end + while /\A-/ =~ ARGV[0] + case ARGV[0] + when "-e" + edit = true + ARGV.shift + when "-a" + auto = true + ARGV.shift + else + $stderr.puts "Unknown command line option: #{ARGV[0]}" + exit 1 + end + end + gem = ARGV.shift + if ARGV[0] + exit sync_default_gems_with_commits(gem, ARGV, edit: edit) + elsif auto + exit sync_default_gems_with_commits(gem, true, edit: edit) + else + sync_default_gems(gem) + end + end if $0 == __FILE__ end diff --git a/tool/test-annocheck.sh b/tool/test-annocheck.sh new file mode 100755 index 0000000000..6896869e07 --- /dev/null +++ b/tool/test-annocheck.sh @@ -0,0 +1,40 @@ +#!/bin/sh -eu +# Run the `tool/test-annocheck.sh [binary files]` to check security issues +# by annocheck <https://sourceware.org/annobin/>. +# +# E.g. `tool/test-annocheck.sh ruby libruby.so.3.2.0`. +# +# Note that as the annocheck binary package is not available on Ubuntu, and it +# is working in progress in Debian, this script uses Fedora container for now. +# It requires docker or podman. +# https://www.debian.org/devel/wnpp/itp.en.html +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=926470 + +set -x + +DOCKER="$(command -v docker || command -v podman)" +TAG=ruby-fedora-annocheck +TOOL_DIR="$(dirname "${0}")" +TMP_DIR="tmp/annocheck" +DOCKER_RUN_VOLUME_OPTS= + +if [ -z "${CI-}" ]; then + # Use a volume option on local (non-CI). + DOCKER_RUN_VOLUME_OPTS="-v $(pwd):/work" + "${DOCKER}" build --rm -t "${TAG}" ${TOOL_DIR}/annocheck/ +else + # TODO: A temporary workaround on CI to build by copying binary files from + # host to container without volume option, as I couldn't find a way to use + # volume in container in container on GitHub Actions + # <.github/workflows/compilers.yml>. + TAG="${TAG}-copy" + rm -rf "${TMP_DIR}" + mkdir -p "${TMP_DIR}" + for file in "${@}"; do + cp -p "${file}" "${TMP_DIR}" + done + "${DOCKER}" build --rm -t "${TAG}" --build-arg=IN_DIR="${TMP_DIR}" -f ${TOOL_DIR}/annocheck/Dockerfile-copy . + rm -rf "${TMP_DIR}" +fi + +"${DOCKER}" run --rm -t ${DOCKER_RUN_VOLUME_OPTS} "${TAG}" annocheck --verbose ${TEST_ANNOCHECK_OPTS-} "${@}" diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb new file mode 100644 index 0000000000..778fe3311a --- /dev/null +++ b/tool/test-bundled-gems.rb @@ -0,0 +1,140 @@ +require 'rbconfig' +require 'timeout' +require 'fileutils' +require 'shellwords' +require_relative 'lib/colorize' +require_relative 'lib/gem_env' + +ENV.delete("GNUMAKEFLAGS") + +github_actions = ENV["GITHUB_ACTIONS"] == "true" + +DEFAULT_ALLOWED_FAILURES = RUBY_PLATFORM =~ /mswin|mingw/ ? [ + 'debug', + 'irb', + 'csv', +] : [] +allowed_failures = ENV['TEST_BUNDLED_GEMS_ALLOW_FAILURES'] || '' +allowed_failures = allowed_failures.split(',').concat(DEFAULT_ALLOWED_FAILURES).uniq.reject(&:empty?) + +# make test-bundled-gems BUNDLED_GEMS=gem1,gem2,gem3 +bundled_gems = nil if (bundled_gems = ARGV.first&.split(","))&.empty? + +colorize = Colorize.new +rake = File.realpath("../../.bundle/bin/rake", __FILE__) +gem_dir = File.realpath('../../gems', __FILE__) +rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +run_opts = ENV["RUN_OPTS"]&.shellsplit +exit_code = 0 +ruby = ENV['RUBY'] || RbConfig.ruby +failed = [] +File.foreach("#{gem_dir}/bundled_gems") do |line| + next unless gem = line[/^[^\s\#]+/] + next if bundled_gems&.none? {|pat| File.fnmatch?(pat, gem)} + next unless File.directory?("#{gem_dir}/src/#{gem}/test") + + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", rake, "test"] + first_timeout = 600 # 10min + + toplib = gem + unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + toplib = gem.tr("-", "/") + next unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + end + + case gem + when "rbs" + # TODO: We should skip test file instead of test class/methods + skip_test_files = %w[ + ] + + skip_test_files.each do |file| + path = "#{gem_dir}/src/#{gem}/#{file}" + File.unlink(path) if File.exist?(path) + end + + rbs_skip_tests = [ + File.join(__dir__, "/rbs_skip_tests") + ] + + if /mswin|mingw/ =~ RUBY_PLATFORM + rbs_skip_tests << File.join(__dir__, "/rbs_skip_tests_windows") + end + + test_command.concat %W[stdlib_test validate RBS_SKIP_TESTS=#{rbs_skip_tests.join(File::PATH_SEPARATOR)} SKIP_RBS_VALIDATION=true] + first_timeout *= 3 + + when "debug" + # Since debug gem requires debug.so in child processes without + # activating the gem, we preset necessary paths in RUBYLIB + # environment variable. + load_path = true + + when "test-unit" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", "test/run.rb"] + + when "csv" + first_timeout = 30 + + when "win32ole" + next unless /mswin|mingw/ =~ RUBY_PLATFORM + + end + + if load_path + libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) + next unless $?.success? + ENV["RUBYLIB"] = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) + else + ENV["RUBYLIB"] = rubylib + end + + # 93(bright yellow) is copied from .github/workflows/mingw.yml + puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" + print "[command]" if github_actions + p test_command + timeouts = {nil => first_timeout, INT: 30, TERM: 10, KILL: nil} + if /mingw|mswin/ =~ RUBY_PLATFORM + timeouts.delete(:TERM) # Inner process signal on Windows + group = :new_pgroup + pg = "" + else + group = :pgroup + pg = "-" + end + pid = Process.spawn(*test_command, group => true) + timeouts.each do |sig, sec| + if sig + puts "Sending #{sig} signal" + Process.kill("#{pg}#{sig}", pid) + end + begin + break Timeout.timeout(sec) {Process.wait(pid)} + rescue Timeout::Error + end + rescue Interrupt + exit_code = Signal.list["INT"] + Process.kill("#{pg}KILL", pid) + Process.wait(pid) + break + end + + print "::endgroup::\n" if github_actions + unless $?.success? + + mesg = "Tests failed " + + ($?.signaled? ? "by SIG#{Signal.signame($?.termsig)}" : + "with exit code #{$?.exitstatus}") + puts colorize.decorate(mesg, "fail") + if allowed_failures.include?(gem) + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" + puts colorize.decorate(mesg, "skip") + else + failed << gem + exit_code = 1 + end + end +end + +puts "Failed gems: #{failed.join(', ')}" unless failed.empty? +exit exit_code diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb index e0d7dd06e1..28ef0bf7f8 100644 --- a/tool/test-coverage.rb +++ b/tool/test-coverage.rb @@ -4,6 +4,16 @@ Coverage.start(lines: true, branches: true, methods: true) TEST_COVERAGE_DATA_FILE = "test-coverage.dat" +FILTER_PATHS = %w[ + lib/bundler/vendor + lib/rubygems/resolver/molinillo + lib/rubygems/tsort + lib/rubygems/optparse + tool + test + spec +] + def merge_coverage_data(res1, res2) res1.each do |path, cov1| cov2 = res2[path] @@ -44,7 +54,7 @@ def save_coverage_data(res1) if cov[:methods] h = {} cov[:methods].each do |(klass, *key), count| - h[[klass.inspect, *key]] = count + h[[klass.name, *key]] = count end cov[:methods].replace h end @@ -62,8 +72,11 @@ def save_coverage_data(res1) end def invoke_simplecov_formatter - %w[doclie simplecov-html simplecov].each do |f| - $LOAD_PATH.unshift "#{__dir__}/../coverage/#{f}/lib" + # XXX docile-x.y.z and simplecov-x.y.z, simplecov-html-x.y.z, simplecov_json_formatter-x.y.z + %w[simplecov simplecov-html simplecov_json_formatter docile].each do |f| + Dir.glob("#{__dir__}/../.bundle/gems/#{f}-*/lib").each do |d| + $LOAD_PATH.unshift d + end end require "simplecov" @@ -74,8 +87,8 @@ def invoke_simplecov_formatter res.each do |path, cov| next unless path.start_with?(base_dir) || path.start_with?(cur_dir) - next if path.start_with?(File.join(base_dir, "test")) - simplecov_result[path] = cov[:lines] + next if FILTER_PATHS.any? {|dir| path.start_with?(File.join(base_dir, dir))} + simplecov_result[path] = cov end a, b = base_dir, cur_dir @@ -101,6 +114,10 @@ pid = $$ pwd = Dir.pwd at_exit do + # Some tests leave GC.stress enabled, causing slow coverage processing. + # Reset it here to avoid performance issues. + GC.stress = false + exit_exc = $! Dir.chdir(pwd) do diff --git a/tool/test/init.rb b/tool/test/init.rb new file mode 100644 index 0000000000..3fd1419a9c --- /dev/null +++ b/tool/test/init.rb @@ -0,0 +1,26 @@ +# This file includes the settings for "make test-all" and "make test-tool". +# Note that this file is loaded not only by test/runner.rb but also by tool/lib/test/unit/parallel.rb. + +# Prevent test-all from using bundled gems +["GEM_HOME", "GEM_PATH"].each do |gem_env| + # Preserve the gem environment prepared by tool/runruby.rb for test-tool, which uses bundled gems. + ENV["BUNDLED_#{gem_env}"] = ENV[gem_env] + + ENV[gem_env] = "".freeze +end +ENV["GEM_SKIP"] = "".freeze + +ENV.delete("RUBY_CODESIGN") + +Warning[:experimental] = false + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) + +require 'test/unit' + +require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') +require "tracepointchecker" +require "zombie_hunter" +require "iseq_loader_checker" +require "gc_checker" +require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') diff --git a/tool/test/minitest/test_minitest_benchmark.rb b/tool/test/minitest/test_minitest_benchmark.rb deleted file mode 100644 index a783e684c2..0000000000 --- a/tool/test/minitest/test_minitest_benchmark.rb +++ /dev/null @@ -1,131 +0,0 @@ -# encoding: utf-8 -# frozen_string_literal: false - -require 'minitest/autorun' -require 'minitest/benchmark' - -## -# Used to verify data: -# http://www.wolframalpha.com/examples/RegressionAnalysis.html - -class TestMiniTestBenchmark < MiniTest::Unit::TestCase - def test_cls_bench_exp - assert_equal [2, 4, 8, 16, 32], self.class.bench_exp(2, 32, 2) - end - - def test_cls_bench_linear - assert_equal [2, 4, 6, 8, 10], self.class.bench_linear(2, 10, 2) - end - - def test_cls_benchmark_methods - assert_equal [], self.class.benchmark_methods - - c = Class.new(MiniTest::Unit::TestCase) do - def bench_blah - end - end - - assert_equal ["bench_blah"], c.benchmark_methods - end - - def test_cls_bench_range - assert_equal [1, 10, 100, 1_000, 10_000], self.class.bench_range - end - - def test_fit_exponential_clean - x = [1.0, 2.0, 3.0, 4.0, 5.0] - y = x.map { |n| 1.1 * Math.exp(2.1 * n) } - - assert_fit :exponential, x, y, 1.0, 1.1, 2.1 - end - - def test_fit_exponential_noisy - x = [1.0, 1.9, 2.6, 3.4, 5.0] - y = [12, 10, 8.2, 6.9, 5.9] - - # verified with Numbers and R - assert_fit :exponential, x, y, 0.95, 13.81148, -0.1820 - end - - def test_fit_logarithmic_clean - x = [1.0, 2.0, 3.0, 4.0, 5.0] - y = x.map { |n| 1.1 + 2.1 * Math.log(n) } - - assert_fit :logarithmic, x, y, 1.0, 1.1, 2.1 - end - - def test_fit_logarithmic_noisy - x = [1.0, 2.0, 3.0, 4.0, 5.0] - # Generated with - # y = x.map { |n| jitter = 0.999 + 0.002 * rand; (Math.log(n) ) * jitter } - y = [0.0, 0.6935, 1.0995, 1.3873, 1.6097] - - assert_fit :logarithmic, x, y, 0.95, 0, 1 - end - - def test_fit_constant_clean - x = (1..5).to_a - y = [5.0, 5.0, 5.0, 5.0, 5.0] - - assert_fit :linear, x, y, nil, 5.0, 0 - end - - def test_fit_constant_noisy - x = (1..5).to_a - y = [1.0, 1.2, 1.0, 0.8, 1.0] - - # verified in numbers and R - assert_fit :linear, x, y, nil, 1.12, -0.04 - end - - def test_fit_linear_clean - # y = m * x + b where m = 2.2, b = 3.1 - x = (1..5).to_a - y = x.map { |n| 2.2 * n + 3.1 } - - assert_fit :linear, x, y, 1.0, 3.1, 2.2 - end - - def test_fit_linear_noisy - x = [ 60, 61, 62, 63, 65] - y = [3.1, 3.6, 3.8, 4.0, 4.1] - - # verified in numbers and R - assert_fit :linear, x, y, 0.8315, -7.9635, 0.1878 - end - - def test_fit_power_clean - # y = A x ** B, where B = b and A = e ** a - # if, A = 1, B = 2, then - - x = [1.0, 2.0, 3.0, 4.0, 5.0] - y = [1.0, 4.0, 9.0, 16.0, 25.0] - - assert_fit :power, x, y, 1.0, 1.0, 2.0 - end - - def test_fit_power_noisy - # from www.engr.uidaho.edu/thompson/courses/ME330/lecture/least_squares.html - x = [10, 12, 15, 17, 20, 22, 25, 27, 30, 32, 35] - y = [95, 105, 125, 141, 173, 200, 253, 298, 385, 459, 602] - - # verified in numbers - assert_fit :power, x, y, 0.90, 2.6217, 1.4556 - - # income to % of households below income amount - # http://library.wolfram.com/infocenter/Conferences/6461/PowerLaws.nb - x = [15000, 25000, 35000, 50000, 75000, 100000] - y = [0.154, 0.283, 0.402, 0.55, 0.733, 0.843] - - # verified in numbers - assert_fit :power, x, y, 0.96, 3.119e-5, 0.8959 - end - - def assert_fit msg, x, y, fit, exp_a, exp_b - a, b, rr = send "fit_#{msg}", x, y - - assert_operator rr, :>=, fit if fit - assert_in_delta exp_a, a - assert_in_delta exp_b, b - end -end diff --git a/tool/test/minitest/test_minitest_mock.rb b/tool/test/minitest/test_minitest_mock.rb deleted file mode 100644 index da222c5aa8..0000000000 --- a/tool/test/minitest/test_minitest_mock.rb +++ /dev/null @@ -1,404 +0,0 @@ -# encoding: utf-8 -# frozen_string_literal: false - -require 'minitest/autorun' - -class TestMiniTestMock < MiniTest::Unit::TestCase - def setup - @mock = MiniTest::Mock.new.expect(:foo, nil) - @mock.expect(:meaning_of_life, 42) - end - - def test_create_stub_method - assert_nil @mock.foo - end - - def test_allow_return_value_specification - assert_equal 42, @mock.meaning_of_life - end - - def test_blow_up_if_not_called - @mock.foo - - util_verify_bad "expected meaning_of_life() => 42, got []" - end - - def test_not_blow_up_if_everything_called - @mock.foo - @mock.meaning_of_life - - assert @mock.verify - end - - def test_allow_expectations_to_be_added_after_creation - @mock.expect(:bar, true) - assert @mock.bar - end - - def test_not_verify_if_new_expected_method_is_not_called - @mock.foo - @mock.meaning_of_life - @mock.expect(:bar, true) - - util_verify_bad "expected bar() => true, got []" - end - - def test_blow_up_on_wrong_number_of_arguments - @mock.foo - @mock.meaning_of_life - @mock.expect(:sum, 3, [1, 2]) - - e = assert_raises ArgumentError do - @mock.sum - end - - assert_equal "mocked method :sum expects 2 arguments, got 0", e.message - end - - def test_return_mock_does_not_raise - retval = MiniTest::Mock.new - mock = MiniTest::Mock.new - mock.expect(:foo, retval) - mock.foo - - assert mock.verify - end - - def test_mock_args_does_not_raise - skip "non-opaque use of ==" if maglev? - - arg = MiniTest::Mock.new - mock = MiniTest::Mock.new - mock.expect(:foo, nil, [arg]) - mock.foo(arg) - - assert mock.verify - end - - def test_blow_up_on_wrong_arguments - @mock.foo - @mock.meaning_of_life - @mock.expect(:sum, 3, [1, 2]) - - e = assert_raises MockExpectationError do - @mock.sum(2, 4) - end - - exp = "mocked method :sum called with unexpected arguments [2, 4]" - assert_equal exp, e.message - end - - def test_expect_with_non_array_args - e = assert_raises ArgumentError do - @mock.expect :blah, 3, false - end - - assert_equal "args must be an array", e.message - end - - def test_respond_appropriately - assert @mock.respond_to?(:foo) - assert @mock.respond_to?(:foo, true) - assert @mock.respond_to?('foo') - assert !@mock.respond_to?(:bar) - end - - def test_no_method_error_on_unexpected_methods - e = assert_raises NoMethodError do - @mock.bar - end - - expected = "unmocked method :bar, expected one of [:foo, :meaning_of_life]" - - assert_equal expected, e.message - end - - def test_assign_per_mock_return_values - a = MiniTest::Mock.new - b = MiniTest::Mock.new - - a.expect(:foo, :a) - b.expect(:foo, :b) - - assert_equal :a, a.foo - assert_equal :b, b.foo - end - - def test_do_not_create_stub_method_on_new_mocks - a = MiniTest::Mock.new - a.expect(:foo, :a) - - assert !MiniTest::Mock.new.respond_to?(:foo) - end - - def test_mock_is_a_blank_slate - @mock.expect :kind_of?, true, [Integer] - @mock.expect :==, true, [1] - - assert @mock.kind_of?(Integer), "didn't mock :kind_of\?" - assert @mock == 1, "didn't mock :==" - end - - def test_verify_allows_called_args_to_be_loosely_specified - mock = MiniTest::Mock.new - mock.expect :loose_expectation, true, [Integer] - mock.loose_expectation 1 - - assert mock.verify - end - - def test_verify_raises_with_strict_args - mock = MiniTest::Mock.new - mock.expect :strict_expectation, true, [2] - - e = assert_raises MockExpectationError do - mock.strict_expectation 1 - end - - exp = "mocked method :strict_expectation called with unexpected arguments [1]" - assert_equal exp, e.message - end - - def test_method_missing_empty - mock = MiniTest::Mock.new - - mock.expect :a, nil - - mock.a - - e = assert_raises MockExpectationError do - mock.a - end - - assert_equal "No more expects available for :a: []", e.message - end - - def test_same_method_expects_are_verified_when_all_called - mock = MiniTest::Mock.new - mock.expect :foo, nil, [:bar] - mock.expect :foo, nil, [:baz] - - mock.foo :bar - mock.foo :baz - - assert mock.verify - end - - def test_same_method_expects_blow_up_when_not_all_called - mock = MiniTest::Mock.new - mock.expect :foo, nil, [:bar] - mock.expect :foo, nil, [:baz] - - mock.foo :bar - - e = assert_raises(MockExpectationError) { mock.verify } - - exp = "expected foo(:baz) => nil, got [foo(:bar) => nil]" - - assert_equal exp, e.message - end - - def test_verify_passes_when_mock_block_returns_true - mock = MiniTest::Mock.new - mock.expect :foo, nil do - true - end - - mock.foo - - assert mock.verify - end - - def test_mock_block_is_passed_function_params - arg1, arg2, arg3 = :bar, [1,2,3], {:a => 'a'} - mock = MiniTest::Mock.new - mock.expect :foo, nil do |a1, a2, a3| - a1 == arg1 && - a2 == arg2 && - a3 == arg3 - end - - mock.foo arg1, arg2, arg3 - - assert mock.verify - end - - def test_verify_fails_when_mock_block_returns_false - mock = MiniTest::Mock.new - mock.expect :foo, nil do - false - end - - e = assert_raises(MockExpectationError) { mock.foo } - exp = "mocked method :foo failed block w/ []" - - assert_equal exp, e.message - end - - def test_mock_block_throws_if_args_passed - mock = MiniTest::Mock.new - - e = assert_raises(ArgumentError) do - mock.expect :foo, nil, [:a, :b, :c] do - true - end - end - - exp = "args ignored when block given" - - assert_equal exp, e.message - end - - def test_mock_returns_retval_when_called_with_block - mock = MiniTest::Mock.new - mock.expect(:foo, 32) do - true - end - - rs = mock.foo - - assert_equal rs, 32 - end - - def util_verify_bad exp - e = assert_raises MockExpectationError do - @mock.verify - end - - assert_equal exp, e.message - end -end - -require_relative "metametameta" - -class TestMiniTestStub < MiniTest::Unit::TestCase - def setup - super - MiniTest::Unit::TestCase.reset - - @tc = MiniTest::Unit::TestCase.new 'fake tc' - @assertion_count = 1 - end - - def teardown - super - assert_equal @assertion_count, @tc._assertions - end - - class Time - def self.now - 24 - end - end - - def assert_stub val_or_callable - @assertion_count += 1 - - t = Time.now.to_i - - Time.stub :now, val_or_callable do - @tc.assert_equal 42, Time.now - end - - @tc.assert_operator Time.now.to_i, :>=, t - end - - def test_stub_private_module_method - @assertion_count += 1 - - t0 = Time.now - - self.stub :sleep, nil do - @tc.assert_nil sleep(10) - end - - @tc.assert_operator Time.now - t0, :<=, 1 - end - - def test_stub_private_module_method_indirect - @assertion_count += 1 - - slow_clapper = Class.new do - def slow_clap - sleep 3 - :clap - end - end.new - - slow_clapper.stub :sleep, nil do |fast_clapper| - @tc.assert_equal :clap, fast_clapper.slow_clap # either form works - @tc.assert_equal :clap, slow_clapper.slow_clap # yay closures - end - end - - def test_stub_public_module_method - Math.stub(:log10, 42.0) do - @tc.assert_in_delta 42.0, Math.log10(1000) - end - end - - def test_stub_value - assert_stub 42 - end - - def test_stub_block - assert_stub lambda { 42 } - end - - def test_stub_block_args - @assertion_count += 1 - - t = Time.now.to_i - - Time.stub :now, lambda { |n| n * 2 } do - @tc.assert_equal 42, Time.now(21) - end - - @tc.assert_operator Time.now.to_i, :>=, t - end - - def test_stub_callable - obj = Object.new - - def obj.call - 42 - end - - assert_stub obj - end - - def test_stub_yield_self - obj = "foo" - - val = obj.stub :to_s, "bar" do |s| - s.to_s - end - - @tc.assert_equal "bar", val - end - - def test_dynamic_method - @assertion_count = 2 - - dynamic = Class.new do - def self.respond_to?(meth) - meth == :found - end - - def self.method_missing(meth, *args, &block) - if meth == :found - false - else - super - end - end - end - - val = dynamic.stub(:found, true) do |s| - s.found - end - - @tc.assert_equal true, val - @tc.assert_equal false, dynamic.found - end -end diff --git a/tool/test/runner.rb b/tool/test/runner.rb index 64f6df167e..9001fc2d06 100644 --- a/tool/test/runner.rb +++ b/tool/test/runner.rb @@ -1,39 +1,14 @@ -# frozen_string_literal: false +# frozen_string_literal: true require 'rbconfig' -src_testdir = File.dirname(File.realpath(__FILE__)) -$LOAD_PATH << src_testdir -tool_dir = File.join src_testdir, ".." -$LOAD_PATH.unshift "#{tool_dir}/lib" +require_relative "init" -# Get bundled gems on load path -Dir.glob("#{src_testdir}/../gems/*/*.gemspec") - .reject {|f| f =~ /minitest|test-unit|power_assert/ } - .map {|f| $LOAD_PATH.unshift File.join(File.dirname(f), "lib") } - -require 'test/unit' - -module Gem -end -class Gem::TestCase < MiniTest::Unit::TestCase - @@project_dir = File.dirname($LOAD_PATH.last) -end - -ENV["GEM_SKIP"] = ENV["GEM_HOME"] = ENV["GEM_PATH"] = "".freeze - -require_relative "#{tool_dir}/lib/profile_test_all" if ENV.has_key?('RUBY_TEST_ALL_PROFILE') -require_relative "#{tool_dir}/lib/tracepointchecker" -require_relative "#{tool_dir}/lib/zombie_hunter" -require_relative "#{tool_dir}/lib/iseq_loader_checker" - -if ENV['COVERAGE'] - require_relative "#{tool_dir}/test-coverage.rb" -end - -begin - exit Test::Unit::AutoRunner.run(true, src_testdir) -rescue NoMemoryError - system("cat /proc/meminfo") if File.exist?("/proc/meminfo") - system("ps x -opid,args,%cpu,%mem,nlwp,rss,vsz,wchan,stat,start,time,etime,blocked,caught,ignored,pending,f") if File.exist?("/bin/ps") - raise +case $0 +when __FILE__ + dir = __dir__ +when "-e" + # No default directory +else + dir = File.realdirpath("..", $0) end +exit Test::Unit::AutoRunner.run(true, dir) diff --git a/tool/test/test_commit_email.rb b/tool/test/test_commit_email.rb new file mode 100644 index 0000000000..db441584fd --- /dev/null +++ b/tool/test/test_commit_email.rb @@ -0,0 +1,102 @@ +require 'test/unit' +require 'shellwords' +require 'tmpdir' +require 'fileutils' +require 'open3' + +class TestCommitEmail < Test::Unit::TestCase + STDIN_DELIMITER = "---\n" + + def setup + omit 'git command is not available' unless system('git', '--version', out: File::NULL, err: File::NULL) + + @ruby = Dir.mktmpdir + Dir.chdir(@ruby) do + git('init', '--initial-branch=master') + git('config', 'user.name', 'Jóhän Grübél') + git('config', 'user.email', 'johan@example.com') + env = { + 'GIT_AUTHOR_DATE' => '2025-10-08T12:00:00Z', + 'GIT_CONFIG_GLOBAL' => @ruby + "/gitconfig", + 'TZ' => 'UTC', + } + git('commit', '--allow-empty', '-m', 'New repository initialized by cvs2svn.', env:) + git('commit', '--allow-empty', '-m', 'Initial revision', env:) + git('commit', '--allow-empty', '-m', 'version 1.0.0', env:) + end + + @sendmail = File.join(Dir.mktmpdir, 'sendmail') + File.write(@sendmail, <<~SENDMAIL, mode: "wx", perm: 0755) + #!/bin/sh + echo #{STDIN_DELIMITER.chomp.dump} + exec cat + SENDMAIL + + @commit_email = File.expand_path('../../tool/commit-email.rb', __dir__) + end + + def teardown + # Clean up temporary files if #setup was not omitted + if @sendmail + File.unlink(@sendmail) + Dir.rmdir(File.dirname(@sendmail)) + end + if @ruby + FileUtils.rm_rf(@ruby) + end + end + + def test_sendmail_encoding + omit 'the sendmail script does not work on windows' if windows? + + Dir.chdir(@ruby) do + before_rev = git('rev-parse', 'HEAD^').chomp + after_rev = git('rev-parse', 'HEAD').chomp + short_rev = after_rev[0...10] + + out, _, status = EnvUtil.invoke_ruby([ + { 'SENDMAIL' => @sendmail, 'TZ' => 'UTC' }.merge!(gem_env), + @commit_email, './', 'cvs-admin@ruby-lang.org', + before_rev, after_rev, 'refs/heads/master', + '--viewer-uri', 'https://github.com/ruby/ruby/commit/', + '--error-to', 'cvs-admin@ruby-lang.org', + ], '', true) + stdin = out.b.split(STDIN_DELIMITER.b, 2).last.force_encoding('UTF-8') + + assert_true(status.success?) + assert_equal(stdin, <<~EOS) + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: =?UTF-8?B?SsOzaMOkbiBHcsO8YsOpbA==?= <noreply@ruby-lang.org> + To: cvs-admin@ruby-lang.org + Subject: #{short_rev} (master): =?UTF-8?B?dmVyc2lvbuOAgDEuMC4w?= + J=C3=B3h=C3=A4n Gr=C3=BCb=C3=A9l\t2025-10-08 12:00:00 +0000 (Wed, 08 Oct 2= + 025) + + New Revision: #{short_rev} + + https://github.com/ruby/ruby/commit/#{short_rev} + + Log: + version=E3=80=801.0.0= + EOS + end + end + + private + + # Resurrect the gem environment preserved by tool/test/init.rb. + # This should work as long as you have run `make up` or `make install`. + def gem_env + { 'GEM_PATH' => ENV['BUNDLED_GEM_PATH'], 'GEM_HOME' => ENV['BUNDLED_GEM_HOME'] } + end + + def git(*cmd, env: {}) + out, status = Open3.capture2(env, 'git', *cmd) + unless status.success? + raise "git #{cmd.shelljoin}\n#{out}" + end + out + end +end diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb new file mode 100755 index 0000000000..252687f3f3 --- /dev/null +++ b/tool/test/test_sync_default_gems.rb @@ -0,0 +1,373 @@ +#!/usr/bin/ruby +require 'test/unit' +require 'stringio' +require 'tmpdir' +require 'rubygems/version' +require_relative '../sync_default_gems' + +module Test_SyncDefaultGems + class TestMessageFilter < Test::Unit::TestCase + def assert_message_filter(expected, trailers, input, repo = "ruby/test", sha = "0123456789") + subject, *expected = expected + expected = [ + "[#{repo}] #{subject}\n", + *expected.map {_1+"\n"}, + "\n", + "https://github.com/#{repo}/commit/#{sha[0, 10]}\n", + ] + if trailers + expected << "\n" + expected.concat(trailers.map {_1+"\n"}) + end + + out = SyncDefaultGems.message_filter(repo, sha, input) + assert_pattern_list(expected, out) + end + + def test_subject_only + expected = [ + "initial commit", + ] + assert_message_filter(expected, nil, "initial commit") + end + + def test_link_in_parenthesis + expected = [ + "fix (https://github.com/ruby/test/pull/1)", + ] + assert_message_filter(expected, nil, "fix (#1)") + end + + def test_co_authored_by + expected = [ + "commit something", + ] + trailers = [ + "Co-Authored-By: git <git@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers, ""].join("\n")) + end + + def test_multiple_co_authored_by + expected = [ + "many commits", + ] + trailers = [ + "Co-authored-by: git <git@ruby-lang.org>", + "Co-authored-by: svn <svn@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers, ""].join("\n")) + end + + def test_co_authored_by_no_newline + expected = [ + "commit something", + ] + trailers = [ + "Co-Authored-By: git <git@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers].join("\n")) + end + + def test_dot_ending_subject + expected = [ + "subject with a dot.", + "", + "- next body line", + ] + assert_message_filter(expected, nil, [expected[0], expected[2], ""].join("\n")) + end + end + + class TestSyncWithCommits < Test::Unit::TestCase + def setup + super + @target = nil + pend "No git" unless system("git --version", out: IO::NULL) + @testdir = Dir.mktmpdir("sync") + user, email = "Ruby", "test@ruby-lang.org" + @git_config = %W"HOME USER GIT_CONFIG_GLOBAL GNUPGHOME".each_with_object({}) {|k, c| c[k] = ENV[k]} + ENV["HOME"] = @testdir + ENV["USER"] = user + ENV["GNUPGHOME"] = @testdir + '/.gnupg' + expire = EnvUtil.apply_timeout_scale(30).to_i + # Generate a new unprotected key with default parameters that + # expires after 30 seconds. + if @gpgsign = system(*%w"gpg --quiet --batch --passphrase", "", + "--quick-generate-key", email, *%W"default default seconds=#{expire}", + err: IO::NULL) + # Fetch the generated public key. + signingkey = IO.popen(%W"gpg --quiet --list-public-key #{email}", &:read)[/^pub .*\n +\K\h+/] + end + ENV["GIT_CONFIG_GLOBAL"] = @testdir + "/gitconfig" + git(*%W"config --global user.email", email) + git(*%W"config --global user.name", user) + git(*%W"config --global init.defaultBranch default") + if signingkey + git(*%W"config --global user.signingkey", signingkey) + git(*%W"config --global commit.gpgsign true") + git(*%W"config --global gpg.program gpg") + git(*%W"config --global log.showSignature true") + end + @target = "sync-test" + SyncDefaultGems::REPOSITORIES[@target] = SyncDefaultGems.repo( + ["ruby/#{@target}", "default"], + [ + ["lib", "lib"], + ["test", "test"], + ], + exclude: [ + "test/fixtures/*", + ], + ) + @sha = {} + @origdir = Dir.pwd + Dir.chdir(@testdir) + ["src", @target].each do |dir| + git(*%W"init -q #{dir}") + File.write("#{dir}/.gitignore", "*~\n") + Dir.mkdir("#{dir}/lib") + File.write("#{dir}/lib/common.rb", ":ok\n") + Dir.mkdir("#{dir}/.github") + Dir.mkdir("#{dir}/.github/workflows") + File.write("#{dir}/.github/workflows/default.yml", "default:\n") + git(*%W"add .gitignore lib/common.rb .github", chdir: dir) + git(*%W"commit -q -m", "Initialize", chdir: dir) + if dir == "src" + File.write("#{dir}/lib/fine.rb", "return\n") + Dir.mkdir("#{dir}/test") + File.write("#{dir}/test/test_fine.rb", "return\n") + git(*%W"add lib/fine.rb test/test_fine.rb", chdir: dir) + git(*%W"commit -q -m", "Looks fine", chdir: dir) + end + Dir.mkdir("#{dir}/tool") + File.write("#{dir}/tool/ok", "#!/bin/sh\n""echo ok\n") + git(*%W"add tool/ok", chdir: dir) + git(*%W"commit -q -m", "Add tool #{dir}", chdir: dir) + @sha[dir] = top_commit(dir) + end + git(*%W"remote add #{@target} ../#{@target}", chdir: "src") + end + + def teardown + if @target + if @gpgsign + system(*%W"gpgconf --kill all") + end + Dir.chdir(@origdir) + SyncDefaultGems::REPOSITORIES.delete(@target) + ENV.update(@git_config) + FileUtils.rm_rf(@testdir) + end + super + end + + def capture_process_output_to(outputs) + return yield unless outputs&.empty? == false + IO.pipe do |r, w| + orig = outputs.map {|out| out.dup} + outputs.each {|out| out.reopen(w)} + w.close + reader = Thread.start {r.read} + yield + ensure + outputs.each {|out| o = orig.shift; out.reopen(o); o.close} + return reader.value + end + end + + def capture_process_outputs + out = err = nil + synchronize do + out = capture_process_output_to(STDOUT) do + err = capture_process_output_to(STDERR) do + yield + end + end + end + return out, err + end + + def git(*commands, **opts) + system("git", *commands, exception: true, **opts) + end + + def top_commit(dir, format: "%H") + IO.popen(%W[git log --no-show-signature --format=#{format} -1], chdir: dir, &:read)&.chomp + end + + def assert_sync(commits = true, success: true, editor: nil) + result = nil + out = capture_process_output_to([STDOUT, STDERR]) do + Dir.chdir("src") do + orig_editor = ENV["GIT_EDITOR"] + ENV["GIT_EDITOR"] = editor || 'false' + edit = true if editor + + result = SyncDefaultGems.sync_default_gems_with_commits(@target, commits, edit: edit) + ensure + ENV["GIT_EDITOR"] = orig_editor + end + end + assert_equal(success, result, out) + out + end + + def test_sync + File.write("#@target/lib/common.rb", "# OK!\n") + git(*%W"commit -q -m", "OK", "lib/common.rb", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# OK!\n", File.read("src/lib/common.rb")) + log = top_commit("src", format: "%B").lines + assert_equal("[ruby/#@target] OK\n", log.first, out) + assert_match(%r[/ruby/#{@target}/commit/\h+$], log.last, out) + assert_operator(top_commit(@target), :start_with?, log.last[/\h+$/], out) + end + + def test_skip_tool + git(*%W"rm -q tool/ok", chdir: @target) + git(*%W"commit -q -m", "Remove tool", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_test_fixtures + Dir.mkdir("#@target/test") + Dir.mkdir("#@target/test/fixtures") + File.write("#@target/test/fixtures/fixme.rb", "") + git(*%W"add test/fixtures/fixme.rb", chdir: @target) + git(*%W"commit -q -m", "Add fixtures", chdir: @target) + out = assert_sync(["#{@sha[@target]}..#{@target}/default"]) + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= NEWS!!!\n") + git(*%W"add --", "docs/NEWS.md", chdir: @target) + File.write("#@target/docs/hello.md", "Hello\n") + git(*%W"add --", "docs/hello.md", chdir: @target) + git(*%W"commit -q -m", "It's a news", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_adding_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= New library\n") + File.write("#@target/lib/news.rb", "return\n") + git(*%W"add --", "docs/NEWS.md", "lib/news.rb", chdir: @target) + git(*%W"commit -q -m", "New lib", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal "return\n", File.read("src/lib/news.rb") + assert_include top_commit("src", format: "oneline"), "[ruby/#{@target}] New lib" + assert_not_operator File, :exist?, "src/docs" + end + + def test_gitignore + File.write("#@target/.gitignore", "*.bak\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add .github", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", + *%W".gitignore lib/common.rb .github", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("*~\n", File.read("src/.gitignore"), out) + assert_equal("#!/bin/sh\n""echo ok\n", File.read("src/tool/ok"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_gitignore_after_conflict + File.write("src/Gemfile", "# main\n") + git(*%W"add Gemfile", chdir: "src") + git(*%W"commit -q -m", "Add Gemfile", chdir: "src") + File.write("#@target/Gemfile", "# conflict\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add Gemfile .github lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# main\n", File.read("src/Gemfile"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_delete_after_conflict + File.write("#@target/lib/bad.rb", "raise\n") + git(*%W"add lib/bad.rb", chdir: @target) + git(*%W"commit -q -m", "Add bad.rb", chdir: @target) + out = assert_sync + assert_equal("raise\n", File.read("src/lib/bad.rb")) + + git(*%W"rm lib/bad.rb", chdir: "src", out: IO::NULL) + git(*%W"commit -q -m", "Remove bad.rb", chdir: "src") + + File.write("#@target/lib/bad.rb", "raise 'bar'\n") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + git(*%W"add lib/bad.rb lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict", chdir: @target) + + head = top_commit("src") + out = assert_sync(editor: "git rm -f lib/bad.rb") + assert_not_equal(head, top_commit("src")) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/lib/bad.rb", out) + end + + def test_squash_merge + # This test is known to fail with git 2.43.0, which is used by Ubuntu 24.04. + # We don't know which exact version fixed it, but we know git 2.52.0 works. + stdout, status = Open3.capture2('git', '--version', err: File::NULL) + omit 'git version check failed' unless status.success? + git_version = stdout[/\Agit version \K\S+/] + omit "git #{git_version} is too old" if Gem::Version.new(git_version) < Gem::Version.new('2.44.0') + + # 2---. <- branch + # / \ + # 1---3---3'<- merge commit with conflict resolution + File.write("#@target/lib/conflict.rb", "# 1\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict.rb", chdir: @target) + + git(*%W"checkout -q -b branch", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 2\n") + File.write("#@target/lib/new.rb", "# new\n") + git(*%W"add lib/conflict.rb lib/new.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in branch", chdir: @target) + + git(*%W"checkout -q default", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 3\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in default", chdir: @target) + + # How can I suppress "Auto-merging ..." message from git merge? + git(*%W"merge -X ours -m", "Merge commit", "branch", chdir: @target, out: IO::NULL) + + out = assert_sync() + assert_equal("# 3\n", File.read("src/lib/conflict.rb"), out) + subject, body = top_commit("src", format: "%B").split("\n\n", 2) + assert_equal("[ruby/#@target] Merge commit", subject, out) + assert_includes(body, "Commit in branch", out) + end + + def test_no_upstream_file + group = SyncDefaultGems::Repository.group(%w[ + lib/un.rb + lib/unicode_normalize/normalize.rb + lib/unicode_normalize/tables.rb + lib/net/https.rb + ]) + expected = { + "un" => %w[lib/un.rb], + "net-http" => %w[lib/net/https.rb], + nil => %w[lib/unicode_normalize/normalize.rb lib/unicode_normalize/tables.rb], + } + assert_equal(expected, group) + end + end if /darwin|linux/ =~ RUBY_PLATFORM +end diff --git a/tool/test/minitest/metametameta.rb b/tool/test/testunit/metametameta.rb index a12717c8b1..e494038939 100644 --- a/tool/test/minitest/metametameta.rb +++ b/tool/test/testunit/metametameta.rb @@ -3,15 +3,14 @@ require 'tempfile' require 'stringio' -require 'minitest/autorun' -class MiniTest::Unit::TestCase +class Test::Unit::TestCase def clean s s.gsub(/^ {6}/, '') end end -class MetaMetaMetaTestCase < MiniTest::Unit::TestCase +class MetaMetaMetaTestCase < Test::Unit::TestCase def assert_report expected, flags = %w[--seed 42] header = clean <<-EOM Run options: #{flags.map { |s| s =~ /\|/ ? s.inspect : s }.join " "} @@ -45,10 +44,10 @@ class MetaMetaMetaTestCase < MiniTest::Unit::TestCase def setup super srand 42 - MiniTest::Unit::TestCase.reset - @tu = MiniTest::Unit.new + Test::Unit::TestCase.reset + @tu = Test::Unit::Runner.new - MiniTest::Unit.runner = nil # protect the outer runner from the inner tests + Test::Unit::Runner.runner = nil # protect the outer runner from the inner tests end def teardown @@ -58,13 +57,13 @@ class MetaMetaMetaTestCase < MiniTest::Unit::TestCase def with_output synchronize do begin - save = MiniTest::Unit.output + save = Test::Unit::Runner.output @output = StringIO.new("") - MiniTest::Unit.output = @output + Test::Unit::Runner.output = @output yield ensure - MiniTest::Unit.output = save + Test::Unit::Runner.output = save end end end diff --git a/tool/test/testunit/test4test_hideskip.rb b/tool/test/testunit/test4test_hideskip.rb index 410bffc13c..14f79a5743 100644 --- a/tool/test/testunit/test4test_hideskip.rb +++ b/tool/test/testunit/test4test_hideskip.rb @@ -4,7 +4,11 @@ $LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" require 'test/unit' class TestForTestHideSkip < Test::Unit::TestCase - def test_skip - skip "do nothing" + def test_omit + omit "do nothing" + end + + def test_pend + pend "do nothing" end end diff --git a/tool/test/testunit/test4test_load_failure.rb b/tool/test/testunit/test4test_load_failure.rb new file mode 100644 index 0000000000..e1570c2542 --- /dev/null +++ b/tool/test/testunit/test4test_load_failure.rb @@ -0,0 +1 @@ +raise LoadError, "no-such-library" diff --git a/tool/test/testunit/test4test_sorting.rb b/tool/test/testunit/test4test_sorting.rb index 698c875b79..f5a6866425 100644 --- a/tool/test/testunit/test4test_sorting.rb +++ b/tool/test/testunit/test4test_sorting.rb @@ -5,7 +5,7 @@ require 'test/unit' class TestForTestHideSkip < Test::Unit::TestCase def test_c - skip "do nothing" + omit "do nothing" end def test_b diff --git a/tool/test/testunit/test4test_timeout.rb b/tool/test/testunit/test4test_timeout.rb new file mode 100644 index 0000000000..3225f66398 --- /dev/null +++ b/tool/test/testunit/test4test_timeout.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' +require 'timeout' + +class TestForTestTimeout < Test::Unit::TestCase + 10.times do |i| + define_method("test_timeout_#{i}") do + Timeout.timeout(0.001) do + sleep + end + end + end +end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index 0c7c4882b8..d9bdc8f3c5 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -8,6 +8,8 @@ class TestAssertion < Test::Unit::TestCase end def test_timeout_separately + pend "hang-up" if /mswin|mingw/ =~ RUBY_PLATFORM + assert_raise(Timeout::Error) do assert_separately([], <<~"end;", timeout: 0.1) sleep @@ -15,6 +17,29 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assertion_count_separately + beginning = self._assertions + + assert_separately([], "") + assertions_at_nothing = self._assertions - beginning + + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "assert true") + assert_equal(1, self._assertions - prev_assertions) + + omit unless Process.respond_to?(:fork) + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "Process.fork {assert true}; assert true") + assert_equal(2, self._assertions - prev_assertions) + + prev_assertions = self._assertions + assertions_at_nothing + # TODO: assertions before `fork` are counted twice; it is possible + # to reset `_assertions` at `Process._fork`, but the hook can + # interfere in other tests. + assert_separately([], "assert true; Process.fork {assert true}") + assert_equal(3, self._assertions - prev_assertions) + end + def return_in_assert_raise assert_raise(RuntimeError) do return @@ -22,8 +47,182 @@ class TestAssertion < Test::Unit::TestCase end def test_assert_raise - assert_raise(MiniTest::Assertion) do + assert_raise(Test::Unit::AssertionFailedError) do return_in_assert_raise end end + + def test_assert_raise_with_message + my_error = Class.new(StandardError) + + assert_raise_with_message(my_error, "with message") do + raise my_error, "with message" + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(RuntimeError, "with message") do + raise my_error, "with message" + end + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(my_error, "without message") do + raise my_error, "with message" + end + end + end + + def test_assert_raise_kind_of + my_error = Class.new(StandardError) + + assert_raise_kind_of(my_error) do + raise my_error + end + + assert_raise_kind_of(StandardError) do + raise my_error + end + end + + def test_assert_pattern_list + assert_pattern_list([/foo?/], "foo") + assert_not_pattern_list([/foo?/], "afoo") + assert_not_pattern_list([/foo?/], "foo?") + assert_pattern_list([:*, /foo?/, :*], "foo") + assert_pattern_list([:*, /foo?/], "afoo") + assert_not_pattern_list([:*, /foo?/], "afoo?") + assert_pattern_list([/foo?/, :*], "foo?") + + assert_not_pattern_list(["foo?"], "foo") + assert_not_pattern_list(["foo?"], "afoo") + assert_pattern_list(["foo?"], "foo?") + assert_not_pattern_list([:*, "foo?", :*], "foo") + assert_not_pattern_list([:*, "foo?"], "afoo") + assert_pattern_list([:*, "foo?"], "afoo?") + assert_pattern_list(["foo?", :*], "foo?") + end + + def assert_not_pattern_list(pattern_list, actual, message=nil) + assert_raise(Test::Unit::AssertionFailedError) do + assert_pattern_list(pattern_list, actual, message) + end + end + + def test_caller_bactrace_location + begin + line = __LINE__; assert_fail_for_backtrace_location + rescue Test::Unit::AssertionFailedError => e + end + location = Test::Unit::Runner.new.location(e) + assert_equal "#{__FILE__}:#{line}", location + end + + def assert_fail_for_backtrace_location + assert false + end + + VersionClass = Struct.new(:version) do + def version?(*ver) + Test::Unit::CoreAssertions.version_match?(ver, self.version) + end + end + + V14_6_0 = VersionClass.new([14, 6, 0]) + V15_0_0 = VersionClass.new([15, 0, 0]) + + def test_version_match_integer + assert_not_operator(V14_6_0, :version?, 13) + assert_operator(V14_6_0, :version?, 14) + assert_not_operator(V14_6_0, :version?, 15) + assert_not_operator(V15_0_0, :version?, 14) + assert_operator(V15_0_0, :version?, 15) + end + + def test_version_match_integer_range + assert_operator(V14_6_0, :version?, 13..14) + assert_not_operator(V15_0_0, :version?, 13..14) + assert_not_operator(V14_6_0, :version?, 13...14) + assert_not_operator(V15_0_0, :version?, 13...14) + end + + def test_version_match_array_range + assert_operator(V14_6_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V14_6_0, :version?, [14, 0]...[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[14, 6]) + assert_operator(V14_6_0, :version?, [14, 0]..[15]) + assert_operator(V15_0_0, :version?, [14, 0]..[15]) + assert_operator(V14_6_0, :version?, [14, 0]...[15]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[15]) + end + + def test_version_match_integer_endless_range + assert_operator(V14_6_0, :version?, 14..) + assert_operator(V15_0_0, :version?, 14..) + assert_not_operator(V14_6_0, :version?, 15..) + assert_operator(V15_0_0, :version?, 15..) + end + + def test_version_match_integer_endless_range_exclusive + assert_operator(V14_6_0, :version?, 14...) + assert_operator(V15_0_0, :version?, 14...) + assert_not_operator(V14_6_0, :version?, 15...) + assert_operator(V15_0_0, :version?, 15...) + end + + def test_version_match_array_endless_range + assert_operator(V14_6_0, :version?, [14, 5]..) + assert_operator(V15_0_0, :version?, [14, 5]..) + assert_not_operator(V14_6_0, :version?, [14, 7]..) + assert_operator(V15_0_0, :version?, [14, 7]..) + assert_not_operator(V14_6_0, :version?, [15]..) + assert_operator(V15_0_0, :version?, [15]..) + assert_not_operator(V14_6_0, :version?, [15, 0]..) + assert_operator(V15_0_0, :version?, [15, 0]..) + end + + def test_version_match_array_endless_range_exclude_end + assert_operator(V14_6_0, :version?, [14, 5]...) + assert_operator(V15_0_0, :version?, [14, 5]...) + assert_not_operator(V14_6_0, :version?, [14, 7]...) + assert_operator(V15_0_0, :version?, [14, 7]...) + assert_not_operator(V14_6_0, :version?, [15]...) + assert_operator(V15_0_0, :version?, [15]...) + assert_not_operator(V14_6_0, :version?, [15, 0]...) + assert_operator(V15_0_0, :version?, [15, 0]...) + end + + def test_version_match_integer_beginless_range + assert_operator(V14_6_0, :version?, ..14) + assert_not_operator(V15_0_0, :version?, ..14) + assert_operator(V14_6_0, :version?, ..15) + assert_operator(V15_0_0, :version?, ..15) + + assert_not_operator(V14_6_0, :version?, ...14) + assert_not_operator(V15_0_0, :version?, ...14) + assert_operator(V14_6_0, :version?, ...15) + assert_not_operator(V15_0_0, :version?, ...15) + end + + def test_version_match_array_beginless_range + assert_not_operator(V14_6_0, :version?, ..[14, 5]) + assert_not_operator(V15_0_0, :version?, ..[14, 5]) + assert_operator(V14_6_0, :version?, ..[14, 6]) + assert_not_operator(V15_0_0, :version?, ..[14, 6]) + assert_operator(V14_6_0, :version?, ..[15]) + assert_operator(V15_0_0, :version?, ..[15]) + assert_operator(V14_6_0, :version?, ..[15, 0]) + assert_operator(V15_0_0, :version?, ..[15, 0]) + end + + def test_version_match_array_beginless_range_exclude_end + assert_not_operator(V14_6_0, :version?, ...[14, 5]) + assert_not_operator(V15_0_0, :version?, ...[14, 5]) + assert_not_operator(V14_6_0, :version?, ...[14, 6]) + assert_not_operator(V15_0_0, :version?, ...[14, 6]) + assert_operator(V14_6_0, :version?, ...[15]) + assert_not_operator(V15_0_0, :version?, ...[15]) + assert_operator(V14_6_0, :version?, ...[15, 0]) + assert_not_operator(V15_0_0, :version?, ...[15, 0]) + end end diff --git a/tool/test/testunit/test_hideskip.rb b/tool/test/testunit/test_hideskip.rb index 0188316a2c..a470368bca 100644 --- a/tool/test/testunit/test_hideskip.rb +++ b/tool/test/testunit/test_hideskip.rb @@ -4,16 +4,15 @@ require 'test/unit' class TestHideSkip < Test::Unit::TestCase def test_hideskip assert_not_match(/^ *1\) Skipped/, hideskip) - assert_match(/^ *1\) Skipped/, hideskip("--show-skip")) + assert_match(/^ *1\) Skipped.*^ *2\) Skipped/m, hideskip("--show-skip")) output = hideskip("--hide-skip") - output.gsub!(/Successful MJIT finish\n/, '') if RubyVM::MJIT.enabled? - assert_match(/assertions\/s.\n+1 tests, 0 assertions, 0 failures, 0 errors, 1 skips/, output) + assert_match(/assertions\/s.\n+2 tests, 0 assertions, 0 failures, 0 errors, 2 skips/, output) end private def hideskip(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", "--verbose", *args], err: [:child, :out]) {|f| f.read } diff --git a/tool/test/testunit/test_launchable.rb b/tool/test/testunit/test_launchable.rb new file mode 100644 index 0000000000..76be876456 --- /dev/null +++ b/tool/test/testunit/test_launchable.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: false +require 'test/unit' +require 'tempfile' +require 'json' +require_relative '../../lib/launchable' + +class TestLaunchable < Test::Unit::TestCase + def test_json_stream_writer + Tempfile.create(['launchable-test-', '.json']) do |f| + json_stream_writer = Launchable::JsonStreamWriter.new(f.path) + json_stream_writer.write_array('testCases') + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 42, + status: "TEST_FAILED", + stdout: nil, + stderr: nil, + createdAt: "2021-10-05T12:34:00", + data: { + lineNumber: 1 + } + } + ) + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 45, + status: "TEST_PASSED", + stdout: "This is stdout", + stderr: "This is stderr", + createdAt: "2021-10-05T12:36:00", + data: { + lineNumber: 10 + } + } + ) + json_stream_writer.close() + expected = <<JSON +{ + "testCases": [ + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 42, + "status": "TEST_FAILED", + "stdout": null, + "stderr": null, + "createdAt": "2021-10-05T12:34:00", + "data": { + "lineNumber": 1 + } + }, + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 45, + "status": "TEST_PASSED", + "stdout": "This is stdout", + "stderr": "This is stderr", + "createdAt": "2021-10-05T12:36:00", + "data": { + "lineNumber": 10 + } + } + ] +} +JSON + assert_equal(expected, f.read) + end + end +end diff --git a/tool/test/testunit/test_load_failure.rb b/tool/test/testunit/test_load_failure.rb new file mode 100644 index 0000000000..8defa9e39a --- /dev/null +++ b/tool/test/testunit/test_load_failure.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +require 'test/unit' + +class TestLoadFailure < Test::Unit::TestCase + def test_load_failure + assert_not_predicate(load_failure, :success?) + end + + def test_load_failure_parallel + assert_not_predicate(load_failure("-j2"), :success?) + end + + private + + def load_failure(*args) + IO.popen([*@__runner_options__[:ruby], "#{__dir__}/../runner.rb", + "#{__dir__}/test4test_load_failure.rb", + "--verbose", *args], err: [:child, :out]) {|f| + assert_include(f.read, "test4test_load_failure.rb") + } + $? + end +end diff --git a/tool/test/minitest/test_minitest_unit.rb b/tool/test/testunit/test_minitest_unit.rb index 1286b15372..84b6cf688c 100644 --- a/tool/test/minitest/test_minitest_unit.rb +++ b/tool/test/testunit/test_minitest_unit.rb @@ -10,7 +10,7 @@ class ImmutableString < String; def inspect; super.freeze; end; end class TestMiniTestUnit < MetaMetaMetaTestCase pwd = Pathname.new File.expand_path Dir.pwd - basedir = Pathname.new(File.expand_path "lib/minitest") + 'mini' + basedir = Pathname.new(File.expand_path "lib/test") basedir = basedir.relative_path_from(pwd).to_s MINITEST_BASE_DIR = basedir[/\A\./] ? basedir : "./#{basedir}" BT_MIDDLE = ["#{MINITEST_BASE_DIR}/test.rb:161:in `each'", @@ -19,7 +19,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase "#{MINITEST_BASE_DIR}/test.rb:106:in `run'"] def test_class_puke_with_assertion_failed - exception = MiniTest::Assertion.new "Oh no!" + exception = Test::Unit::AssertionFailedError.new "Oh no!" exception.set_backtrace ["unhappy"] assert_equal 'F', @tu.puke('SomeClass', 'method_name', exception) assert_equal 1, @tu.failures @@ -29,7 +29,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase def test_class_puke_with_assertion_failed_and_long_backtrace bt = (["test/test_some_class.rb:615:in `method_name'", - "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raises'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", "test/test_some_class.rb:615:in `each'", "test/test_some_class.rb:614:in `test_method_name'", "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + @@ -39,7 +39,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first - exception = MiniTest::Assertion.new "Oh no!" + exception = Test::Unit::AssertionFailedError.new "Oh no!" exception.set_backtrace bt assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) assert_equal 1, @tu.failures @@ -49,7 +49,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase def test_class_puke_with_assertion_failed_and_user_defined_assertions bt = (["lib/test/my/util.rb:16:in `another_method_name'", - "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raises'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", "lib/test/my/util.rb:15:in `block in assert_something'", "lib/test/my/util.rb:14:in `each'", "lib/test/my/util.rb:14:in `assert_something'", @@ -62,7 +62,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first - exception = MiniTest::Assertion.new "Oh no!" + exception = Test::Unit::AssertionFailedError.new "Oh no!" exception.set_backtrace bt assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) assert_equal 1, @tu.failures @@ -72,8 +72,8 @@ class TestMiniTestUnit < MetaMetaMetaTestCase def test_class_puke_with_failure_and_flunk_in_backtrace exception = begin - MiniTest::Unit::TestCase.new('fake tc').flunk - rescue MiniTest::Assertion => failure + Test::Unit::TestCase.new('fake tc').flunk + rescue Test::Unit::AssertionFailedError => failure failure end assert_equal 'F', @tu.puke('SomeClass', 'method_name', exception) @@ -82,7 +82,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase def test_class_puke_with_flunk_and_user_defined_assertions bt = (["lib/test/my/util.rb:16:in `flunk'", - "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raises'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", "lib/test/my/util.rb:15:in `block in assert_something'", "lib/test/my/util.rb:14:in `each'", "lib/test/my/util.rb:14:in `assert_something'", @@ -95,7 +95,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first - exception = MiniTest::Assertion.new "Oh no!" + exception = Test::Unit::AssertionFailedError.new "Oh no!" exception.set_backtrace bt assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) assert_equal 1, @tu.failures @@ -125,7 +125,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase "test/test_autotest.rb:62:in `test_add_exception'"] ex = util_expand_bt ex - fu = MiniTest::filter_backtrace(bt) + fu = Test::filter_backtrace(bt) assert_equal ex, fu end @@ -135,7 +135,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase BT_MIDDLE + ["#{MINITEST_BASE_DIR}/test.rb:29"]) ex = bt.clone - fu = MiniTest::filter_backtrace(bt) + fu = Test::filter_backtrace(bt) assert_equal ex, fu end @@ -148,17 +148,17 @@ class TestMiniTestUnit < MetaMetaMetaTestCase bt = util_expand_bt bt ex = ["-e:1"] - fu = MiniTest::filter_backtrace bt + fu = Test::filter_backtrace bt assert_equal ex, fu end def test_default_runner_is_minitest_unit - assert_instance_of MiniTest::Unit, MiniTest::Unit.runner + assert_instance_of Test::Unit::Runner, Test::Unit::Runner.runner end def test_passed_eh_teardown_good - test_class = Class.new MiniTest::Unit::TestCase do + test_class = Class.new Test::Unit::TestCase do def teardown; assert true; end def test_omg; assert true; end end @@ -169,9 +169,9 @@ class TestMiniTestUnit < MetaMetaMetaTestCase end def test_passed_eh_teardown_skipped - test_class = Class.new MiniTest::Unit::TestCase do + test_class = Class.new Test::Unit::TestCase do def teardown; assert true; end - def test_omg; skip "bork"; end + def test_omg; omit "bork"; end end test = test_class.new :test_omg @@ -180,7 +180,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase end def test_passed_eh_teardown_flunked - test_class = Class.new MiniTest::Unit::TestCase do + test_class = Class.new Test::Unit::TestCase do def teardown; flunk; end def test_omg; assert true; end end @@ -223,7 +223,7 @@ class TestMiniTestUnitInherited < MetaMetaMetaTestCase def test_inherited_hook_plays_nice_with_others with_overridden_include do assert_throws :inherited_hook do - Class.new MiniTest::Unit::TestCase + Class.new Test::Unit::TestCase end end end @@ -235,151 +235,23 @@ class TestMiniTestRunner < MetaMetaMetaTestCase def test_class_test_suites @assertion_count = 0 - tc = Class.new(MiniTest::Unit::TestCase) + tc = Class.new(Test::Unit::TestCase) - assert_equal 1, MiniTest::Unit::TestCase.test_suites.size - assert_equal [tc], MiniTest::Unit::TestCase.test_suites - end - - def test_run_test - Class.new MiniTest::Unit::TestCase do - attr_reader :foo - - def run_test name - @foo = "hi mom!" - super - @foo = "okay" - end - - def test_something - assert_equal "hi mom!", foo - end - end - - expected = clean <<-EOM - . - - Finished tests in 0.00 - - 1 tests, 1 assertions, 0 failures, 0 errors, 0 skips - EOM - - assert_report expected - end - - def test_run_error - Class.new MiniTest::Unit::TestCase do - def test_something - assert true - end - - def test_error - raise "unhandled exception" - end - end - - expected = clean <<-EOM - E. - - Finished tests in 0.00 - - 1) Error: - #<Class:0xXXX>#test_error: - RuntimeError: unhandled exception - FILE:LINE:in \`test_error\' - - 2 tests, 1 assertions, 0 failures, 1 errors, 0 skips - EOM - - assert_report expected - end - - def test_run_error_teardown - Class.new MiniTest::Unit::TestCase do - def test_something - assert true - end - - def teardown - raise "unhandled exception" - end - end - - expected = clean <<-EOM - E - - Finished tests in 0.00 - - 1) Error: - #<Class:0xXXX>#test_something: - RuntimeError: unhandled exception - FILE:LINE:in \`teardown\' - - 1 tests, 1 assertions, 0 failures, 1 errors, 0 skips - EOM - - assert_report expected - end - - def test_run_failing - Class.new MiniTest::Unit::TestCase do - def test_something - assert true - end - - def test_failure - assert false - end - end - - expected = clean <<-EOM - F. - - Finished tests in 0.00 - - 1) Failure: - #<Class:0xXXX>#test_failure [FILE:LINE]: - Failed assertion, no message given. - - 2 tests, 2 assertions, 1 failures, 0 errors, 0 skips - EOM - - assert_report expected - end - - def test_run_failing_filtered - Class.new MiniTest::Unit::TestCase do - def test_something - assert true - end - - def test_failure - assert false - end - end - - expected = clean <<-EOM - . - - Finished tests in 0.00 - - 1 tests, 1 assertions, 0 failures, 0 errors, 0 skips - EOM - - assert_report expected, %w[--name /some|thing/ --seed 42] + assert_equal 2, Test::Unit::TestCase.test_suites.size + assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites.sort_by {|ts| ts.name.to_s} end def assert_filtering name, expected, a = false args = %W[--name #{name} --seed 42] - alpha = Class.new MiniTest::Unit::TestCase do + alpha = Class.new Test::Unit::TestCase do define_method :test_something do assert a end end Object.const_set(:Alpha, alpha) - beta = Class.new MiniTest::Unit::TestCase do + beta = Class.new Test::Unit::TestCase do define_method :test_something do assert true end @@ -392,123 +264,20 @@ class TestMiniTestRunner < MetaMetaMetaTestCase Object.send :remove_const, :Beta end - def test_run_filtered_including_suite_name - expected = clean <<-EOM - . - - Finished tests in 0.00 - - 1 tests, 1 assertions, 0 failures, 0 errors, 0 skips - EOM - - assert_filtering "/Beta#test_something/", expected - end - - def test_run_filtered_including_suite_name_string - expected = clean <<-EOM - . - - Finished tests in 0.00 - - 1 tests, 1 assertions, 0 failures, 0 errors, 0 skips - EOM - - assert_filtering "Beta#test_something", expected - end - - def test_run_filtered_string_method_only - expected = clean <<-EOM - .. - - Finished tests in 0.00 - - 2 tests, 2 assertions, 0 failures, 0 errors, 0 skips - EOM - - assert_filtering "test_something", expected, :pass - end - - def test_run_passing - Class.new MiniTest::Unit::TestCase do - def test_something - assert true - end - end - - expected = clean <<-EOM - . - - Finished tests in 0.00 - - 1 tests, 1 assertions, 0 failures, 0 errors, 0 skips - EOM - - assert_report expected - end - - def test_run_skip - Class.new MiniTest::Unit::TestCase do - def test_something - assert true - end - - def test_skip - skip "not yet" - end - end - - expected = clean <<-EOM - S. - - Finished tests in 0.00 - - 2 tests, 1 assertions, 0 failures, 0 errors, 1 skips - EOM - - assert_report expected - end - - def test_run_skip_verbose - Class.new MiniTest::Unit::TestCase do - def test_something - assert true - end - - def test_skip - skip "not yet" - end - end - - expected = clean <<-EOM - #<Class:0xXXX>#test_skip = 0.00 s = S - #<Class:0xXXX>#test_something = 0.00 s = . - - - Finished tests in 0.00 - - 1) Skipped: - #<Class:0xXXX>#test_skip [FILE:LINE]: - not yet - - 2 tests, 1 assertions, 0 failures, 0 errors, 1 skips - EOM - - assert_report expected, %w[--seed 42 --verbose] - end - def test_run_with_other_runner - MiniTest::Unit.runner = Class.new MiniTest::Unit do + pend "We don't imagine to replace the default runner with ruby/ruby test suite." + Test::Unit::Runner.runner = Class.new Test::Unit::Runner do def _run_suite suite, type suite.before_suite # Run once before each suite super suite, type end end.new - Class.new MiniTest::Unit::TestCase do + Class.new Test::Unit::TestCase do def self.name; "wacky!" end def self.before_suite - MiniTest::Unit.output.puts "Running #{self.name} tests" + Test::Unit::Runner.output.puts "Running #{self.name} tests" @@foo = 1 end @@ -559,8 +328,10 @@ class TestMiniTestUnitOrder < MetaMetaMetaTestCase # do not parallelize this suite... it just can't handle it. def test_before_setup + pend "Surpressing the raise message when running with tests" + call_order = [] - Class.new MiniTest::Unit::TestCase do + Class.new Test::Unit::TestCase do define_method :setup do super() call_order << :setup @@ -582,8 +353,10 @@ class TestMiniTestUnitOrder < MetaMetaMetaTestCase end def test_after_teardown + pend "Surpressing the result message of this tests" + call_order = [] - Class.new MiniTest::Unit::TestCase do + Class.new Test::Unit::TestCase do define_method :teardown do super() call_order << :teardown @@ -605,8 +378,10 @@ class TestMiniTestUnitOrder < MetaMetaMetaTestCase end def test_all_teardowns_are_guaranteed_to_run + pend "Surpressing the raise message when running with tests" + call_order = [] - Class.new MiniTest::Unit::TestCase do + Class.new Test::Unit::TestCase do define_method :after_teardown do super() call_order << :after_teardown @@ -637,9 +412,11 @@ class TestMiniTestUnitOrder < MetaMetaMetaTestCase end def test_setup_and_teardown_survive_inheritance + pend "Surpressing the result message of this tests" + call_order = [] - parent = Class.new MiniTest::Unit::TestCase do + parent = Class.new Test::Unit::TestCase do define_method :setup do call_order << :setup_method end @@ -666,19 +443,17 @@ class TestMiniTestUnitOrder < MetaMetaMetaTestCase end end -class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase +class TestMiniTestUnitTestCase < Test::Unit::TestCase # do not call parallelize_me! - teardown accesses @tc._assertions # which is not threadsafe. Nearly every method in here is an # assertion test so it isn't worth splitting it out further. - RUBY18 = ! defined? Encoding - def setup super - MiniTest::Unit::TestCase.reset + Test::Unit::TestCase.reset - @tc = MiniTest::Unit::TestCase.new 'fake tc' + @tc = Test::Unit::TestCase.new 'fake tc' @zomg = "zomg ponies!" @assertion_count = 1 end @@ -736,9 +511,8 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase def test_assert_equal_different_collection_array_hex_invisible object1 = Object.new object2 = Object.new - msg = "No visible difference in the Array#inspect output. - You should look at the implementation of #== on Array or its members. - [#<Object:0xXXXXXX>]".gsub(/^ +/, "") + msg = "<[#{object1.inspect}]> expected but was + <[#{object2.inspect}]>.".gsub(/^ +/, "") util_assert_triggered msg do @tc.assert_equal [object1], [object2] end @@ -748,9 +522,8 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase h1, h2 = {}, {} h1[1] = Object.new h2[1] = Object.new - msg = "No visible difference in the Hash#inspect output. - You should look at the implementation of #== on Hash or its members. - {1=>#<Object:0xXXXXXX>}".gsub(/^ +/, "") + msg = "<#{h1.inspect}> expected but was + <#{h2.inspect}>.".gsub(/^ +/, "") util_assert_triggered msg do @tc.assert_equal h1, h2 @@ -758,8 +531,6 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_assert_equal_different_diff_deactivated - skip "https://github.com/MagLev/maglev/issues/209" if maglev? - without_diff do util_assert_triggered util_msg("haha" * 10, "blah" * 10) do o1 = "haha" * 10 @@ -777,12 +548,8 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase o1 = c.new "a" o2 = c.new "b" - msg = "--- expected - +++ actual - @@ -1 +1 @@ - -#<#<Class:0xXXXXXX>:0xXXXXXX @name=\"a\"> - +#<#<Class:0xXXXXXX>:0xXXXXXX @name=\"b\"> - ".gsub(/^ +/, "") + msg = "<#{o1.inspect}> expected but was + <#{o2.inspect}>.".gsub(/^ +/, "") util_assert_triggered msg do @tc.assert_equal o1, o2 @@ -793,9 +560,8 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase o1 = Object.new o2 = Object.new - msg = "No visible difference in the Object#inspect output. - You should look at the implementation of #== on Object or its members. - #<Object:0xXXXXXX>".gsub(/^ +/, "") + msg = "<#{o1.inspect}> expected but was + <#{o2.inspect}>.".gsub(/^ +/, "") util_assert_triggered msg do @tc.assert_equal o1, o2 @@ -803,12 +569,8 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_assert_equal_different_long - msg = "--- expected - +++ actual - @@ -1 +1 @@ - -\"hahahahahahahahahahahahahahahahahahahaha\" - +\"blahblahblahblahblahblahblahblahblahblah\" - ".gsub(/^ +/, "") + msg = "<\"hahahahahahahahahahahahahahahahahahahaha\"> expected but was + <\"blahblahblahblahblahblahblahblahblahblah\">.".gsub(/^ +/, "") util_assert_triggered msg do o1 = "haha" * 10 @@ -819,9 +581,8 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_assert_equal_different_long_invisible - msg = "No visible difference in the String#inspect output. - You should look at the implementation of #== on String or its members. - \"blahblahblahblahblahblahblahblahblahblah\"".gsub(/^ +/, "") + msg = "<\"blahblahblahblahblahblahblahblahblahblah\"> (UTF-8) expected but was + <\"blahblahblahblahblahblahblahblahblahblah\"> (UTF-8).".gsub(/^ +/, "") util_assert_triggered msg do o1 = "blah" * 10 @@ -835,12 +596,8 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase def test_assert_equal_different_long_msg msg = "message. - --- expected - +++ actual - @@ -1 +1 @@ - -\"hahahahahahahahahahahahahahahahahahahaha\" - +\"blahblahblahblahblahblahblahblahblahblah\" - ".gsub(/^ +/, "") + <\"hahahahahahahahahahahahahahahahahahahaha\"> expected but was + <\"blahblahblahblahblahblahblahblahblahblah\">.".gsub(/^ +/, "") util_assert_triggered msg do o1 = "haha" * 10 @@ -862,14 +619,14 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_assert_equal_different_short_multiline - msg = "--- expected\n+++ actual\n@@ -1,2 +1,2 @@\n \"a\n-b\"\n+c\"\n" + msg = "<\"a\\n\" + \"b\"> expected but was\n<\"a\\n\" + \"c\">." util_assert_triggered msg do @tc.assert_equal "a\nb", "a\nc" end end def test_assert_equal_different_escaped_newline - msg = "--- expected\n+++ actual\n@@ -1,2 +1,2 @@\n \"xxx\n-a\\\\nb\"\n+a\\\\nc\"\n" + msg = "<\"xxx\\n\" + \"a\\\\nb\"> expected but was\n<\"xxx\\n\" + \"a\\\\nc\">." util_assert_triggered msg do @tc.assert_equal "xxx\na\\nb", "xxx\na\\nc" end @@ -888,7 +645,7 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_assert_in_delta_triggered - x = maglev? ? "9.999999xxxe-07" : "1.0e-06" + x = "1.0e-06" util_assert_triggered "Expected |0.0 - 0.001| (0.001) to be <= #{x}." do @tc.assert_in_delta 0.0, 1.0 / 1000, 0.000001 end @@ -927,8 +684,8 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_assert_in_epsilon_triggered_negative_case - x = (RUBY18 and not maglev?) ? "0.1" : "0.100000xxx" - y = maglev? ? "0.100000xxx" : "0.1" + x = "0.100000xxx" + y = "0.1" util_assert_triggered "Expected |-1.1 - -1| (#{x}) to be <= #{y}." do @tc.assert_in_epsilon(-1.1, -1, 0.1) end @@ -943,7 +700,7 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase def test_assert_includes_triggered @assertion_count = 3 - e = @tc.assert_raises MiniTest::Assertion do + e = @tc.assert_raise Test::Unit::AssertionFailedError do @tc.assert_includes [true], false end @@ -1113,14 +870,14 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end end - def test_assert_raises - @tc.assert_raises RuntimeError do + def test_assert_raise + @tc.assert_raise RuntimeError do raise "blah" end end - def test_assert_raises_module - @tc.assert_raises MyModule do + def test_assert_raise_module + @tc.assert_raise MyModule do raise AnError end end @@ -1131,34 +888,29 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase # # https://github.com/ruby/ruby/commit/6bab4ea9917dc05cd2c94aead2e96eb7df7d4be1 - def test_assert_raises_skip + def test_assert_raise_skip @assertion_count = 0 - util_assert_triggered "skipped", MiniTest::Skip do - @tc.assert_raises ArgumentError do + util_assert_triggered "skipped", Test::Unit::PendedError do + @tc.assert_raise ArgumentError do begin raise "blah" rescue - skip "skipped" + omit "skipped" end end end end - def test_assert_raises_triggered_different - e = assert_raises MiniTest::Assertion do - @tc.assert_raises RuntimeError do + def test_assert_raise_triggered_different + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise RuntimeError do raise SyntaxError, "icky" end end expected = clean <<-EOM.chomp - [RuntimeError] exception expected, not - Class: <SyntaxError> - Message: <\"icky\"> - ---Backtrace--- - FILE:LINE:in \`test_assert_raises_triggered_different\' - --------------- + [RuntimeError] exception expected, not #<SyntaxError: icky>. EOM actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') @@ -1167,21 +919,16 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase assert_equal expected, actual end - def test_assert_raises_triggered_different_msg - e = assert_raises MiniTest::Assertion do - @tc.assert_raises RuntimeError, "XXX" do + def test_assert_raise_triggered_different_msg + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise RuntimeError, "XXX" do raise SyntaxError, "icky" end end expected = clean <<-EOM XXX. - [RuntimeError] exception expected, not - Class: <SyntaxError> - Message: <\"icky\"> - ---Backtrace--- - FILE:LINE:in \`test_assert_raises_triggered_different_msg\' - --------------- + [RuntimeError] exception expected, not #<SyntaxError: icky>. EOM actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') @@ -1190,44 +937,39 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase assert_equal expected.chomp, actual end - def test_assert_raises_triggered_none - e = assert_raises MiniTest::Assertion do - @tc.assert_raises MiniTest::Assertion do + def test_assert_raise_triggered_none + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise Test::Unit::AssertionFailedError do # do nothing end end - expected = "MiniTest::Assertion expected but nothing was raised." + expected = "Test::Unit::AssertionFailedError expected but nothing was raised." assert_equal expected, e.message end - def test_assert_raises_triggered_none_msg - e = assert_raises MiniTest::Assertion do - @tc.assert_raises MiniTest::Assertion, "XXX" do + def test_assert_raise_triggered_none_msg + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise Test::Unit::AssertionFailedError, "XXX" do # do nothing end end - expected = "XXX.\nMiniTest::Assertion expected but nothing was raised." + expected = "XXX.\nTest::Unit::AssertionFailedError expected but nothing was raised." assert_equal expected, e.message end - def test_assert_raises_triggered_subclass - e = assert_raises MiniTest::Assertion do - @tc.assert_raises StandardError do + def test_assert_raise_triggered_subclass + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise StandardError do raise AnError end end expected = clean <<-EOM.chomp - [StandardError] exception expected, not - Class: <AnError> - Message: <\"AnError\"> - ---Backtrace--- - FILE:LINE:in \`test_assert_raises_triggered_subclass\' - --------------- + [StandardError] exception expected, not #<AnError: AnError>. EOM actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') @@ -1275,7 +1017,7 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_assert_send_bad - util_assert_triggered "Expected 1.>(*[2]) to return true." do + util_assert_triggered "Expected 1.>(2) to return true." do @tc.assert_send [1, :>, 2] end end @@ -1328,11 +1070,11 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end end - def test_capture_io + def test_capture_output @assertion_count = 0 non_verbose do - out, err = capture_io do + out, err = capture_output do puts 'hi' $stderr.puts 'bye!' end @@ -1342,43 +1084,6 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end end - def test_capture_subprocess_io - @assertion_count = 0 - - non_verbose do - out, err = capture_subprocess_io do - system("echo", "hi") - system("echo", "bye!", out: :err) - end - - assert_equal "hi\n", out - assert_equal "bye!\n", err - end - end - - def test_class_asserts_match_refutes - @assertion_count = 0 - - methods = MiniTest::Assertions.public_instance_methods - methods.map! { |m| m.to_s } if Symbol === methods.first - - # These don't have corresponding refutes _on purpose_. They're - # useless and will never be added, so don't bother. - ignores = %w[assert_output assert_raises assert_send - assert_silent assert_throws] - - # These are test/unit methods. I'm not actually sure why they're still here - ignores += %w[assert_no_match assert_not_equal assert_not_nil - assert_not_same assert_nothing_raised - assert_nothing_thrown assert_raise] - - asserts = methods.grep(/^assert/).sort - ignores - refutes = methods.grep(/^refute/).sort - ignores - - assert_empty refutes.map { |n| n.sub(/^refute/, 'assert') } - asserts - assert_empty asserts.map { |n| n.sub(/^assert/, 'refute') } - refutes - end - def test_flunk util_assert_triggered 'Epic Fail!' do @tc.flunk @@ -1407,13 +1112,13 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_message_message - util_assert_triggered "whoops.\nExpected: 1\n Actual: 2" do + util_assert_triggered "whoops.\n<1> expected but was\n<2>." do @tc.assert_equal 1, 2, message { "whoops" } end end def test_message_lambda - util_assert_triggered "whoops.\nExpected: 1\n Actual: 2" do + util_assert_triggered "whoops.\n<1> expected but was\n<2>." do @tc.assert_equal 1, 2, lambda { "whoops" } end end @@ -1435,7 +1140,7 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_prints - printer = Class.new { extend MiniTest::Assertions } + printer = Class.new { extend Test::Unit::CoreAssertions } @tc.assert_equal '"test"', printer.mu_pp(ImmutableString.new 'test') end @@ -1474,7 +1179,7 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end def test_refute_in_delta_triggered - x = maglev? ? "0.100000xxx" : "0.1" + x = "0.1" util_assert_triggered "Expected |0.0 - 0.001| (0.001) to not be <= #{x}." do @tc.refute_in_delta 0.0, 1.0 / 1000, 0.1 end @@ -1500,7 +1205,7 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase def test_refute_includes_triggered @assertion_count = 3 - e = @tc.assert_raises MiniTest::Assertion do + e = @tc.assert_raise Test::Unit::AssertionFailedError do @tc.refute_includes [true], true end @@ -1618,46 +1323,37 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase end end - def test_skip + def test_omit @assertion_count = 0 - util_assert_triggered "haha!", MiniTest::Skip do - @tc.skip "haha!" + util_assert_triggered "haha!", Test::Unit::PendedError do + @tc.omit "haha!" end end - def test_test_methods_random + def test_pend @assertion_count = 0 - sample_test_case = Class.new MiniTest::Unit::TestCase do - def self.test_order; :random; end - def test_test1; assert "does not matter" end - def test_test2; assert "does not matter" end - def test_test3; assert "does not matter" end - @test_order = [1, 0, 2] - def self.rand(n) @test_order.shift; end + util_assert_triggered "haha!", Test::Unit::PendedError do + @tc.pend "haha!" end - - expected = %w(test_test2 test_test1 test_test3) - assert_equal expected, sample_test_case.test_methods end - def test_test_methods_sorted + def test_test_methods @assertion_count = 0 - sample_test_case = Class.new MiniTest::Unit::TestCase do - def self.test_order; :sorted end - def test_test3; assert "does not matter" end - def test_test2; assert "does not matter" end + sample_test_case = Class.new Test::Unit::TestCase do def test_test1; assert "does not matter" end + def test_test2; assert "does not matter" end + def test_test3; assert "does not matter" end end - expected = %w(test_test1 test_test2 test_test3) - assert_equal expected, sample_test_case.test_methods + expected = %i(test_test1 test_test2 test_test3) + assert_equal expected, sample_test_case.test_methods.sort end - def assert_triggered expected, klass = MiniTest::Assertion - e = assert_raises klass do + def assert_triggered expected, klass = Test::Unit::AssertionFailedError + e = assert_raise klass do yield end @@ -1670,22 +1366,22 @@ class TestMiniTestUnitTestCase < MiniTest::Unit::TestCase alias util_assert_triggered assert_triggered def util_msg exp, act, msg = nil - s = "Expected: #{exp.inspect}\n Actual: #{act.inspect}" + s = "<#{exp.inspect}> expected but was\n<#{act.inspect}>." s = "#{msg}.\n#{s}" if msg s end def without_diff - old_diff = MiniTest::Assertions.diff - MiniTest::Assertions.diff = nil + old_diff = Test::Unit::Assertions.diff + Test::Unit::Assertions.diff = nil yield ensure - MiniTest::Assertions.diff = old_diff + Test::Unit::Assertions.diff = old_diff end end -class TestMiniTestGuard < MiniTest::Unit::TestCase +class TestMiniTestGuard < Test::Unit::TestCase def test_mri_eh assert self.class.mri? "ruby blah" assert self.mri? "ruby blah" @@ -1696,11 +1392,6 @@ class TestMiniTestGuard < MiniTest::Unit::TestCase assert self.jruby? "java" end - def test_rubinius_eh - assert self.class.rubinius? "rbx" - assert self.rubinius? "rbx" - end - def test_windows_eh assert self.class.windows? "mswin" assert self.windows? "mswin" @@ -1711,6 +1402,7 @@ class TestMiniTestUnitRecording < MetaMetaMetaTestCase # do not parallelize this suite... it just can't handle it. def assert_run_record(*expected, &block) + pend "Test::Unit::Runner#run was changed about recoding feature. We should fix it." def @tu.record suite, method, assertions, time, error recording[method] << error end @@ -1719,9 +1411,9 @@ class TestMiniTestUnitRecording < MetaMetaMetaTestCase @recording ||= Hash.new { |h,k| h[k] = [] } end - MiniTest::Unit.runner = @tu + Test::Unit::Runner.runner = @tu - Class.new MiniTest::Unit::TestCase, &block + Class.new Test::Unit::TestCase, &block with_output do @tu.run @@ -1741,7 +1433,7 @@ class TestMiniTestUnitRecording < MetaMetaMetaTestCase end def test_record_failing - assert_run_record MiniTest::Assertion do + assert_run_record Test::Unit::AssertionFailedError do def test_method assert false end @@ -1781,9 +1473,15 @@ class TestMiniTestUnitRecording < MetaMetaMetaTestCase end def test_record_skip - assert_run_record MiniTest::Skip do + assert_run_record Test::Unit::PendedError do + def test_method + omit "not yet" + end + end + + assert_run_record Test::Unit::PendedError do def test_method - skip "not yet" + pend "not yet" end end end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb index 2cb85ad3e0..adf7d62ecd 100644 --- a/tool/test/testunit/test_parallel.rb +++ b/tool/test/testunit/test_parallel.rb @@ -3,17 +3,25 @@ require 'test/unit' require 'timeout' module TestParallel - PARALLEL_RB = "#{File.dirname(__FILE__)}/../../lib/test/unit/parallel.rb" - TESTS = "#{File.dirname(__FILE__)}/tests_for_parallel" + PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" + TESTS = "#{__dir__}/tests_for_parallel" # use large timeout for --jit-wait - TIMEOUT = EnvUtil.apply_timeout_scale(RubyVM::MJIT.enabled? ? 100 : 30) + TIMEOUT = EnvUtil.apply_timeout_scale(100) + + def self.timeout(n, &blk) + start_time = Time.now + Timeout.timeout(n, &blk) + rescue Timeout::Error + end_time = Time.now + raise Timeout::Error, "execution expired (start: #{ start_time }, end: #{ end_time })" + end class TestParallelWorker < Test::Unit::TestCase def setup i, @worker_in = IO.pipe @worker_out, o = IO.pipe - @worker_pid = spawn(*@options[:ruby], PARALLEL_RB, - "--ruby", @options[:ruby].join(" "), + @worker_pid = spawn(*@__runner_options__[:ruby], PARALLEL_RB, + "--ruby", @__runner_options__[:ruby].join(" "), "-j", "t1", "-v", out: o, in: i) [i,o].each(&:close) end @@ -22,10 +30,10 @@ module TestParallel if @worker_pid && @worker_in begin begin - @worker_in.puts "quit" + @worker_in.puts "quit normal" rescue IOError, Errno::EPIPE end - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@worker_pid) end rescue Timeout::Error @@ -45,10 +53,11 @@ module TestParallel end def test_run - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) @@ -57,13 +66,15 @@ module TestParallel end def test_run_multiple_testcase_in_one_file - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_second.rb test" assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) @@ -72,19 +83,22 @@ module TestParallel end def test_accept_run_command_multiple_times - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_second.rb test" assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) @@ -93,59 +107,62 @@ module TestParallel end def test_p - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_first.rb test" while buf = @worker_out.gets break if /^p (.+?)$/ =~ buf end - assert_match(/TestA#test_nothing_test = \d+\.\d+ s = \.\n/, $1.chomp.unpack("m")[0]) + assert_not_nil($1, "'p' was not found") + assert_match(/TestA#test_nothing_test = \d+\.\d+ s = \.\n/, $1.chomp.unpack1("m")) end end def test_done - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_forth.rb test" while buf = @worker_out.gets break if /^done (.+?)$/ =~ buf end - - result = Marshal.load($1.chomp.unpack("m")[0]) - - assert_equal(5, result[0]) - assert_equal(2, result[1]) - assert_kind_of(Array,result[2]) - assert_kind_of(Array,result[3]) - assert_kind_of(Array,result[4]) - assert_kind_of(Array,result[2][1]) - assert_kind_of(MiniTest::Assertion,result[2][0][2]) - assert_kind_of(MiniTest::Skip,result[2][1][2]) - assert_kind_of(MiniTest::Skip,result[2][2][2]) - assert_kind_of(Exception, result[2][3][2]) - assert_equal(result[5], "TestE") + assert_not_nil($1, "'done' was not found") + + result = Marshal.load($1.chomp.unpack1("m")) + tests, asserts, reports, failures, loadpaths, suite = result + assert_equal(5, tests) + assert_equal(12, asserts) + assert_kind_of(Array, reports) + assert_kind_of(Array, failures) + assert_kind_of(Array, loadpaths) + reports.sort_by! {|_, t| t} + assert_kind_of(Array, reports[1]) + assert_kind_of(Test::Unit::AssertionFailedError, reports[0][2]) + assert_kind_of(Test::Unit::PendedError, reports[1][2]) + assert_kind_of(Test::Unit::PendedError, reports[2][2]) + assert_kind_of(Exception, reports[3][2]) + assert_equal("TestE", suite) end end def test_quit - Timeout.timeout(TIMEOUT) do - @worker_in.puts "quit" + ::TestParallel.timeout(TIMEOUT) do + @worker_in.puts "quit normal" assert_match(/^bye$/m,@worker_out.read) end end end class TestParallel < Test::Unit::TestCase - def spawn_runner(*opt_args) + def spawn_runner(*opt_args, jobs: "t1", env: {}) @test_out, o = IO.pipe - @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", - "--ruby", @options[:ruby].join(" "), - "-j","t1",*opt_args, out: o, err: o) + @test_pid = spawn(env, *@__runner_options__[:ruby], TESTS+"/runner.rb", + "--ruby", @__runner_options__[:ruby].join(" "), + "-j", jobs, *opt_args, out: o, err: o) o.close end def teardown begin if @test_pid - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@test_pid) end end @@ -157,47 +174,50 @@ module TestParallel end def test_ignore_jzero - @test_out, o = IO.pipe - @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", - "--ruby", @options[:ruby].join(" "), - "-j","0", out: File::NULL, err: o) - o.close - Timeout.timeout(TIMEOUT) { + spawn_runner(jobs: "0") + ::TestParallel.timeout(TIMEOUT) { assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) } end def test_should_run_all_without_any_leaks spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^9 tests/,buf) end def test_should_retry_failed_on_workers - spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner "--retry" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying\.+$/,buf) end def test_no_retry_option spawn_runner "--no-retry" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} refute_match(/^Retrying\.+$/,buf) assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf) end def test_jobs_status spawn_runner "--jobs-status" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/\d+=ptest_(first|second|third|forth) */,buf) end def test_separate # this test depends to --jobs-status spawn_runner "--jobs-status", "--separate" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1, message("retried tests should run in different processes") {buf}) end + + def test_hungup + spawn_runner("--worker-timeout=1", "--retry", "test4test_hungup.rb", env: {"RUBY_CRASH_REPORT"=>nil}) + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} + assert_match(/^Retrying hung up testcases\.+$/, buf) + assert_match(/^2 tests,.* 0 failures,/, buf) + end end end diff --git a/tool/test/testunit/test_redefinition.rb b/tool/test/testunit/test_redefinition.rb index 9129e55489..b4f5cabd4f 100644 --- a/tool/test/testunit/test_redefinition.rb +++ b/tool/test/testunit/test_redefinition.rb @@ -3,14 +3,9 @@ require 'test/unit' class TestRedefinition < Test::Unit::TestCase def test_redefinition - assert_match(/^test\/unit warning: method TestForTestRedefinition#test_redefinition is redefined$/, - redefinition) - end - - def redefinition(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_redefinition.rb", *args], - err: [:child, :out]) {|f| - f.read - } + message = %r[test/unit: method TestForTestRedefinition#test_redefinition is redefined$] + assert_raise_with_message(Test::Unit::AssertionFailedError, message) do + require_relative("test4test_redefinition.rb") + end end end diff --git a/tool/test/testunit/test_sorting.rb b/tool/test/testunit/test_sorting.rb index f9de3ec154..3e5d7bfdcc 100644 --- a/tool/test/testunit/test_sorting.rb +++ b/tool/test/testunit/test_sorting.rb @@ -10,9 +10,66 @@ class TestTestUnitSorting < Test::Unit::TestCase end def sorting(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", "--verbose", *args], err: [:child, :out]) {|f| f.read } end + + Item = Struct.new(:name) + SEED = 0x50975eed + + def make_test_list + (1..16).map {"test_%.3x" % rand(0x1000)}.freeze + end + + def test_sort_alpha + sorter = Test::Unit::Order::Types[:alpha].new(SEED) + assert_kind_of(Test::Unit::Order::Types[:sorted], sorter) + + list = make_test_list + sorted = list.sort + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + sorted = list.sort_by(&:name) + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + end + + def test_sort_nosort + sorter = Test::Unit::Order::Types[:nosort].new(SEED) + + list = make_test_list + 16.times do + assert_equal(list, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(list, sorter.sort_by_name(list)) + end + end + + def test_sort_random + type = Test::Unit::Order::Types[:random] + sorter = type.new(SEED) + + list = make_test_list + sorted = type.new(SEED).sort_by_string(list).freeze + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_string(list)) + + list = list.map {|s| Item.new(s)}.freeze + sorted = sorted.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_name(list)) + end end diff --git a/tool/test/testunit/test_timeout.rb b/tool/test/testunit/test_timeout.rb new file mode 100644 index 0000000000..452f5e1a7e --- /dev/null +++ b/tool/test/testunit/test_timeout.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestTiemout < Test::Unit::TestCase + def test_timeout + cmd = [*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_timeout.rb"] + result = IO.popen(cmd, err: [:child, :out], &:read) + assert_not_match(/^T{10}$/, result) + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb index 8831676e19..54474c828d 100644 --- a/tool/test/testunit/tests_for_parallel/ptest_forth.rb +++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb @@ -8,19 +8,19 @@ class TestE < Test::Unit::TestCase assert_equal(1,1) end - def test_always_skip - skip "always" + def test_always_omit + omit "always" end def test_always_fail assert_equal(0,1) end - def test_skip_after_unknown_error + def test_pend_after_unknown_error begin raise UnknownError, "unknown error" rescue - skip "after raise" + pend "after raise" end end diff --git a/tool/test/testunit/tests_for_parallel/slow_helper.rb b/tool/test/testunit/tests_for_parallel/slow_helper.rb new file mode 100644 index 0000000000..38067c1f47 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/slow_helper.rb @@ -0,0 +1,8 @@ +require 'test/unit' + +module TestSlowTimeout + def test_slow + sleep_for = EnvUtil.apply_timeout_scale((ENV['sec'] || 3).to_i) + sleep sleep_for if on_parallel_worker? + end +end diff --git a/tool/test/testunit/tests_for_parallel/test4test_hungup.rb b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb new file mode 100644 index 0000000000..49f503ba9e --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +require_relative '../../../lib/test/unit' + +class TestHung < Test::Unit::TestCase + def test_success_at_worker + assert true + end + + def test_hungup_at_worker + if on_parallel_worker? + sleep EnvUtil.apply_timeout_scale(10) + end + assert true + end +end diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb index 156b2de197..1257a92d38 100644 --- a/tool/transcode-tblgen.rb +++ b/tool/transcode-tblgen.rb @@ -724,8 +724,8 @@ def citrus_decode_mapsrc(ces, csid, mapsrcs) path = File.join(*path) path << ".src" path[path.rindex('/')] = '%' - STDERR.puts 'load mapsrc %s' % path if VERBOSE_MODE - open(path, 'rb') do |f| + STDOUT.puts 'load mapsrc %s' % path if VERBOSE_MODE > 1 + File.open(path, 'rb') do |f| f.each_line do |l| break if /^BEGIN_MAP/ =~ l end @@ -822,11 +822,11 @@ TRANSCODERS = [] TRANSCODE_GENERATED_TRANSCODER_CODE = ''.dup def transcode_tbl_only(from, to, map, valid_encoding=UnspecifiedValidEncoding) - if VERBOSE_MODE + if VERBOSE_MODE > 1 if from.empty? || to.empty? - STDERR.puts "converter for #{from.empty? ? to : from}" + STDOUT.puts "converter for #{from.empty? ? to : from}" else - STDERR.puts "converter from #{from} to #{to}" + STDOUT.puts "converter from #{from} to #{to}" end end id_from = from.tr('^0-9A-Za-z', '_') @@ -895,9 +895,9 @@ static const rb_transcoder #{max_input}, /* max_input */ #{max_output}, /* max_output */ #{ascii_compatibility}, /* asciicompat_type */ - 0, NULL, NULL, /* state_size, state_init, state_fini */ - NULL, NULL, NULL, NULL, - NULL, NULL, NULL + 0, 0, 0, /* state_size, state_init, state_fini */ + 0, 0, 0, 0, + 0, 0, 0 }; End TRANSCODE_GENERATED_TRANSCODER_CODE << transcoder_code @@ -905,7 +905,7 @@ End end def transcode_generate_node(am, name_hint=nil) - STDERR.puts "converter for #{name_hint}" if VERBOSE_MODE + STDOUT.puts "converter for #{name_hint}" if VERBOSE_MODE > 1 am.gennode(TRANSCODE_GENERATED_BYTES_CODE, TRANSCODE_GENERATED_WORDS_CODE, name_hint) '' end @@ -1018,12 +1018,12 @@ if __FILE__ == $0 start_time = Time.now output_filename = nil - verbose_mode = false + verbose_mode = 0 force_mode = false op = OptionParser.new op.def_option("--help", "show help message") { puts op; exit 0 } - op.def_option("--verbose", "verbose mode") { verbose_mode = true } + op.def_option("--verbose", "verbose mode, twice for more verbose") { verbose_mode += 1 } op.def_option("--force", "force table generation") { force_mode = true } op.def_option("--output=FILE", "specify output file") {|arg| output_filename = arg } op.parse! @@ -1066,23 +1066,19 @@ if __FILE__ == $0 if old_signature == chk_signature now = Time.now File.utime(now, now, output_filename) - STDERR.puts "already up-to-date: #{output_filename}" if VERBOSE_MODE + STDOUT.puts "already up-to-date: #{output_filename}" if VERBOSE_MODE > 0 exit end end - if VERBOSE_MODE + if VERBOSE_MODE > 0 if output_filename - STDERR.puts "generating #{output_filename} ..." + STDOUT.puts "generating #{output_filename} ..." end end libs1 = $".dup - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%') - else - erb = ERB.new(src, nil, '%') - end + erb = ERB.new(src, trim_mode: '%') erb.filename = arg erb_result = erb.result(binding) libs2 = $".dup @@ -1111,7 +1107,7 @@ if __FILE__ == $0 File.rename(new_filename, output_filename) tms = Process.times elapsed = Time.now - start_time - STDERR.puts "done. (#{'%.2f' % tms.utime}user #{'%.2f' % tms.stime}system #{'%.2f' % elapsed}elapsed)" if VERBOSE_MODE + STDOUT.puts "done. (#{'%.2f' % tms.utime}user #{'%.2f' % tms.stime}system #{'%.2f' % elapsed}elapsed)" if VERBOSE_MODE > 1 else print result end diff --git a/tool/transform_mjit_header.rb b/tool/transform_mjit_header.rb deleted file mode 100644 index 5f89446c81..0000000000 --- a/tool/transform_mjit_header.rb +++ /dev/null @@ -1,298 +0,0 @@ -# Copyright (C) 2017 Vladimir Makarov, <vmakarov@redhat.com> -# This is a script to transform functions to static inline. -# Usage: transform_mjit_header.rb <c-compiler> <header file> <out> - -require 'fileutils' -require 'tempfile' - -PROGRAM = File.basename($0, ".*") - -module MJITHeader - ATTR_VALUE_REGEXP = /[^()]|\([^()]*\)/ - ATTR_REGEXP = /__attribute__\s*\(\(#{ATTR_VALUE_REGEXP}*\)\)/ - # Example: - # VALUE foo(int bar) - # VALUE __attribute__ ((foo)) bar(int baz) - # __attribute__ ((foo)) VALUE bar(int baz) - FUNC_HEADER_REGEXP = /\A[^\[{(]*(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/ - TARGET_NAME_REGEXP = /\A(rb|ruby|vm|insn|attr|Init)_/ - - # Predefined macros for compilers which are already supported by MJIT. - # We're going to support cl.exe too (WIP) but `cl.exe -E` can't produce macro. - SUPPORTED_CC_MACROS = [ - '__GNUC__', # gcc - '__clang__', # clang - ] - - # These macros are relied on this script's transformation - PREFIXED_MACROS = [ - 'ALWAYS_INLINE', - 'inline', - ] - - # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times. - RECURSIVE_MACROS = %w[ - RASCTRYINFO - RASIPADDR - ] - - IGNORED_FUNCTIONS = [ - 'rb_vm_search_method_slowpath', # This increases the time to compile when inlined. So we use it as external function. - 'rb_equal_opt', # Not used from VM and not compilable - ] - - ALWAYS_INLINED_FUNCTIONS = [ - 'vm_opt_plus', - 'vm_opt_minus', - 'vm_opt_mult', - 'vm_opt_div', - 'vm_opt_mod', - 'vm_opt_neq', - 'vm_opt_lt', - 'vm_opt_le', - 'vm_opt_gt', - 'vm_opt_ge', - 'vm_opt_ltlt', - 'vm_opt_and', - 'vm_opt_or', - 'vm_opt_aref', - 'vm_opt_aset', - 'vm_opt_aref_with', - 'vm_opt_aset_with', - 'vm_opt_not', - 'vm_getinstancevariable', - 'vm_setinstancevariable', - 'vm_setivar', - ] - - # Return start..stop of last decl in CODE ending STOP - def self.find_decl(code, stop) - level = 0 - i = stop - while i = code.rindex(/[;{}]/, i) - if level == 0 && stop != i && decl_found?($&, i) - return decl_start($&, i)..stop - end - case $& - when '}' - level += 1 - when '{' - level -= 1 - end - i -= 1 - end - nil - end - - def self.decl_found?(code, i) - i == 0 || code == ';' || code == '}' - end - - def self.decl_start(code, i) - if i == 0 && code != ';' && code != '}' - 0 - else - i + 1 - end - end - - # Given DECL return the name of it, nil if failed - def self.decl_name_of(decl) - ident_regex = /\w+/ - decl = decl.gsub(/^#.+$/, '') # remove macros - reduced_decl = decl.gsub(ATTR_REGEXP, '') # remove attributes - su1_regex = /{[^{}]*}/ - su2_regex = /{([^{}]|#{su1_regex})*}/ - su3_regex = /{([^{}]|#{su2_regex})*}/ # 3 nested structs/unions is probably enough - reduced_decl.gsub!(su3_regex, '') # remove structs/unions in the header - id_seq_regex = /\s*(#{ident_regex}(\s+|\s*[*]+\s*))*/ - # Process function header: - match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl) - return match[:name] if match - # Process non-function declaration: - reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization - match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl); - return match[:name] if match - nil - end - - # Return true if CC with CFLAGS compiles successfully the current code. - # Use STAGE in the message in case of a compilation failure - def self.check_code!(code, cc, cflags, stage) - with_code(code) do |path| - cmd = "#{cc} #{cflags} #{path}" - out = IO.popen(cmd, err: [:child, :out], &:read) - unless $?.success? - STDERR.puts "error in #{stage} header file:\n#{out}" - exit false - end - end - end - - # Remove unpreprocessable macros - def self.remove_harmful_macros!(code) - code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '') - end - - # -dD outputs those macros, and it produces redefinition warnings or errors - # This assumes common.mk passes `-DMJIT_HEADER` first when it creates rb_mjit_header.h. - def self.remove_predefined_macros!(code) - code.sub!(/\A(#define [^\n]+|\n)*(#define MJIT_HEADER 1\n)/, '\2') - end - - # Return [macro, others]. But others include PREFIXED_MACROS to be used in code. - def self.separate_macro_and_code(code) - code.lines.partition do |l| - l.start_with?('#') && PREFIXED_MACROS.all? { |m| !l.start_with?("#define #{m}") } - end.map! { |lines| lines.join('') } - end - - def self.write(code, out) - # create with strict permission, then will install proper - # permission - FileUtils.mkdir_p(File.dirname(out), mode: 0700) - File.binwrite("#{out}.new", code, perm: 0600) - FileUtils.mv("#{out}.new", out) - end - - # Note that this checks runruby. This conservatively covers platform names. - def self.windows? - RUBY_PLATFORM =~ /mswin|mingw|msys/ - end - - def self.cl_exe?(cc) - cc =~ /\Acl(\z| |\.exe)/ - end - - # If code has macro which only supported compilers predefine, return true. - def self.supported_header?(code) - SUPPORTED_CC_MACROS.any? { |macro| code =~ /^#\s*define\s+#{Regexp.escape(macro)}\b/ } - end - - # This checks if syntax check outputs one of the following messages. - # "error: conflicting types for 'restrict'" - # "error: redefinition of parameter 'restrict'" - # If it's true, this script regards platform as AIX or Solaris and adds -std=c99 as workaround. - def self.conflicting_types?(code, cc, cflags) - with_code(code) do |path| - cmd = "#{cc} #{cflags} #{path}" - out = IO.popen(cmd, err: [:child, :out], &:read) - !$?.success? && - (out.match?(/error: conflicting types for '[^']+'/) || - out.match?(/error: redefinition of parameter '[^']+'/)) - end - end - - def self.with_code(code) - # for `system_header` pragma which can't be in the main file. - Tempfile.open(['', '.h'], mode: File::BINARY) do |f| - f.puts code - f.close - Tempfile.open(['', '.c'], mode: File::BINARY) do |c| - c.puts <<SRC -#include "#{f.path}" -SRC - c.close - return yield(c.path) - end - end - end - private_class_method :with_code -end - -if ARGV.size != 3 - abort "Usage: #{$0} <c-compiler> <header file> <out>" -end - -cc = ARGV[0] -code = File.binread(ARGV[1]) # Current version of the header file. -outfile = ARGV[2] -if MJITHeader.cl_exe?(cc) - cflags = '-DMJIT_HEADER -Zs' -else - cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors' -end - -if !MJITHeader.cl_exe?(cc) && !MJITHeader.supported_header?(code) - puts "This compiler (#{cc}) looks not supported for MJIT. Giving up to generate MJIT header." - MJITHeader.write("#error MJIT does not support '#{cc}' yet", outfile) - exit -end - -MJITHeader.remove_predefined_macros!(code) - -if MJITHeader.windows? # transformation is broken with Windows headers for now - MJITHeader.remove_harmful_macros!(code) - MJITHeader.check_code!(code, cc, cflags, 'initial') - puts "\nSkipped transforming external functions to static on Windows." - MJITHeader.write(code, outfile) - exit -end - -macro, code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW -code = <<header + code -#ifdef __GNUC__ -# pragma GCC system_header -#endif -header -code_to_check = "#{code}#{macro}" # macro should not affect code again - -if MJITHeader.conflicting_types?(code_to_check, cc, cflags) - cflags = "#{cflags} -std=c99" # For AIX gcc -end - -# Check initial file correctness in the manner of final output. -MJITHeader.check_code!(code_to_check, cc, cflags, 'initial') -puts "\nTransforming external functions to static:" - -stop_pos = -1 -extern_names = [] - -# This loop changes function declarations to static inline. -while (decl_range = MJITHeader.find_decl(code, stop_pos)) - stop_pos = decl_range.begin - 1 - decl = code[decl_range] - decl_name = MJITHeader.decl_name_of(decl) - - if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - puts "#{PROGRAM}: changing definition of '#{decl_name}' to declaration" - code[decl_range] = decl.sub(/{.+}/m, ';') - elsif MJITHeader::ALWAYS_INLINED_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - header = match[0].sub(/{\z/, '').strip - header = "static inline #{header.sub(/\A((static|inline) )+/, '')}" - decl[match.begin(0)...match.end(0)] = '{' # remove header - code[decl_range] = "\nALWAYS_INLINE(#{header});\n#{header} #{decl}" - elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/) - decl.sub!(/(extern|static|inline) /, ' ') - unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. - puts "#{PROGRAM}: making declaration of '#{decl_name}' static inline" - end - - code[decl_range] = "static inline #{decl}" - elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/ - unless decl_name.match(MJITHeader::TARGET_NAME_REGEXP) - puts "#{PROGRAM}: SKIPPED to transform #{decl_name}" - next - end - - extern_names << decl_name - decl[match.begin(0)...match.end(0)] = '' - - if decl =~ /\bstatic\b/ - puts "warning: a static decl inside external definition of '#{decl_name}'" - end - - header.sub!(/(extern|inline) /, ' ') - unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. - puts "#{PROGRAM}: making external definition of '#{decl_name}' static inline" - end - code[decl_range] = "static inline #{header}#{decl}" - end -end - -code << macro - -# Check the final file correctness -MJITHeader.check_code!(code, cc, cflags, 'final') - -MJITHeader.write(code, outfile) diff --git a/tool/travis_retry.sh b/tool/travis_retry.sh new file mode 100755 index 0000000000..9b79c56550 --- /dev/null +++ b/tool/travis_retry.sh @@ -0,0 +1,13 @@ +#!/bin/sh -eu +# The modified version of `travis_retry` to support custom backoffs, which is used by .travis.yml. +# https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_retry.bash + +for sleep in 0 ${WAITS:- 1 25 100}; do + sleep "$sleep" + + echo "+ $@" + if "$@"; then + exit 0 + fi +done +exit 1 diff --git a/tool/travis_wait.sh b/tool/travis_wait.sh new file mode 100755 index 0000000000..471b765df8 --- /dev/null +++ b/tool/travis_wait.sh @@ -0,0 +1,18 @@ +#!/bin/bash -eu +# The modified version of `travis_wait` to output a log as the command goes. +# https://github.com/travis-ci/travis-ci/issues/4190#issuecomment-353342526 + +# Produce an output log every 9 minutes as the timeout without output is 10 +# minutes. A job finishes with a timeout if it takes longer than 50 minutes. +# https://docs.travis-ci.com/user/customizing-the-build#build-timeouts +while sleep 9m; do + # Print message with bash variable SECONDS. + echo "====[ $SECONDS seconds still running ]====" +done & + +echo "+ $@" +"$@" + +jobs +kill %1 +exit 0 diff --git a/tool/update-NEWS-gemlist.rb b/tool/update-NEWS-gemlist.rb new file mode 100755 index 0000000000..0b5503580d --- /dev/null +++ b/tool/update-NEWS-gemlist.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby +require 'json' +news = File.read("NEWS.md") +prev = news[/since the \*+(\d+\.\d+\.\d+)\*+/, 1] +prevs = [prev, prev.sub(/\.\d+\z/, '')] + +update = ->(list, type, desc = "updated") do + item = ->(mark = "* ") do + "### The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + + list.map {|g, v|"#{mark}#{g} #{v}\n"}.join("") + "\n" + end + news.sub!(/^(?:\*( +)|#+ *)?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do + item["#{$1&.<< " "}*#{$2 || ' '}"] + end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+( *\* *).*)*)*\n+\K/) do + item[$1 || "* "] + end +end + +load_gems_json = ->(type) do + JSON.parse(File.read("#{type}_gems.json"))['gems'].filter_map do |g| + v = g['versions'].values_at(*prevs).compact.first + g = g['gem'] + g = 'RubyGems' if g == 'rubygems' + [g, v] if v + end.to_h +end + +ARGV.each do |type| + last = load_gems_json[type] + changed = File.foreach("gems/#{type}_gems").filter_map do |l| + next if l.start_with?("#") + g, v = l.split(" ", 3) + next unless v + [g, v] unless last[g] == v + end + changed, added = changed.partition {|g, _| last[g]} + update[changed, type] or next + if added and !added.empty? + if type == 'bundled' + default_gems = load_gems_json['default'] + promoted = {} + added.delete_if do |k, v| + default_gems.key?(k) && promoted[k] = v + end + update[added, type, 'added'] + update[promoted, type, 'promoted from default gems'] or next + else + update[added, type, 'added'] or next + end + end + File.write("NEWS.md", news) +end diff --git a/tool/update-NEWS-refs.rb b/tool/update-NEWS-refs.rb new file mode 100644 index 0000000000..f48cac5ee1 --- /dev/null +++ b/tool/update-NEWS-refs.rb @@ -0,0 +1,38 @@ +# Usage: ruby tool/update-NEWS-refs.rb + +orig_src = File.read(File.join(__dir__, "../NEWS.md")) +lines = orig_src.lines(chomp: true) + +links = {} +while lines.last =~ %r{\A\[(.*?)\]:\s+(.*)\z} + links[$1] = $2 + lines.pop +end + +if links.empty? || lines.last != "" + raise "NEWS.md must end with a sequence of links" +end + +trackers = ["Feature", "Bug", "Misc"] +labels = links.keys.reject {|k| k.start_with?(*trackers)} +new_src = lines.join("\n").gsub(/\[?\[(#{Regexp.union(trackers)}\s+#(\d+))\]\]?/) do + links[$1] ||= "https://bugs.ruby-lang.org/issues/#$2" + "[[#$1]]" +end.gsub(/\[\[#{Regexp.union(labels)}\]\]?/) do + "[#$1]" +end.chomp + "\n\n" + +label_width = links.max_by {|k, _| k.size}.first.size + 4 +redmine_links, non_redmine_links = links.partition {|k,| k =~ /\A#{Regexp.union(trackers)}\s+#\d+\z/ } + +(redmine_links.sort_by {|k,| k[/\d+/].to_i } + non_redmine_links.reverse).each do |k, v| + new_src << "[#{k}]:".ljust(label_width) << v << "\n" +end + +if orig_src != new_src + print "Update NEWS.md? [y/N]" + $stdout.flush + if gets.chomp == "y" + File.write(File.join(__dir__, "../NEWS.md"), new_src) + end +end diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb new file mode 100755 index 0000000000..dec6b49cee --- /dev/null +++ b/tool/update-bundled_gems.rb @@ -0,0 +1,41 @@ +#!ruby -alpF\s+|#.* +BEGIN { + require 'rubygems' + date = nil + # STDOUT is not usable in inplace edit mode + output = $-i ? STDOUT : STDERR +} +output = STDERR if ARGF.file == STDIN +END { + output.print date.strftime("latest_date=%F") if date +} +if gem = $F[0] + ver = Gem::Version.new($F[1]) + (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| + s.platform == "ruby" && s.name == gem + } + if gem.version > ver + gem = src.fetch_spec(gem) + if ENV["UPDATE_BUNDLED_GEMS_ALL"] + uri = gem.metadata["source_code_uri"] || gem.homepage + uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") + else + uri = $F[2] + end + if (!date or gem.date && gem.date > date) and gem.date.to_i != 315_619_200 + # DEFAULT_SOURCE_DATE_EPOCH is meaningless + date = gem.date + end + if $F[3] + if $F[3].include?($F[1]) + $F[3][$F[1]] = gem.version.to_s + elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] + $F[3..-1] = [] + end + end + f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] + $_.gsub!(/\S+\s*(?=\s|$)/) {|s| (f.shift || "").ljust(s.size)} + $_ = [$_, *f].join(" ") unless f.empty? + $_.rstrip! + end +end diff --git a/tool/update-deps b/tool/update-deps index 4aaf812430..2d4a5674be 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -17,10 +17,18 @@ # 3. Use --fix to fix makefiles. # Ex. ./ruby tool/update-deps --fix # +# Usage to create a depend file initially: +# 1. Copy the dependency section from the Makefile generated by extconf.rb. +# Ex. ext/cgi/escape/Makefile +# 2. Add `# AUTOGENERATED DEPENDENCIES START` and `# AUTOGENERATED DEPENDENCIES END` +# sections to top and end of the depend file. +# 3. Run tool/update-deps --fix to fix the depend file. +# 4. Commit the depend file. +# # Other usages: # * Fix makefiles using previously detected dependency problems # Ex. ruby tool/update-deps --actual-fix [file] -# "ruby tool/update-deps --fix" is same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix". +# "ruby tool/update-deps --fix" is the same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix". require 'optparse' require 'stringio' @@ -88,7 +96,15 @@ result.each {|k,v| # They can be referenced as $(top_srcdir)/filename. # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' FILES_IN_SOURCE_DIRECTORY = %w[ - revision.h + prism/api_node.c + prism/ast.h + prism/diagnostic.c + prism/diagnostic.h + prism/node.c + prism/prettyprint.c + prism/serialize.c + prism/token_type.c + prism/version.h ] # Files built in the build directory (except extconf.h). @@ -112,6 +128,7 @@ FILES_NEED_VPATH = %w[ ext/ripper/eventids1.c ext/ripper/eventids2table.c ext/ripper/ripper.c + ext/ripper/ripper_init.c golf_prelude.c id.c id.h @@ -120,16 +137,14 @@ FILES_NEED_VPATH = %w[ known_errors.inc lex.c miniprelude.c - mjit_compile.inc newline.c node_name.inc - opt_sc.inc optinsn.inc optunifs.inc parse.c parse.h - prelude.c probes.dmyh + revision.h vm.inc vmtc.inc @@ -168,13 +183,17 @@ FILES_SAME_NAME_TOP = %w[ version.h ] +# Files that may or may not exist on CI for some reason. +# Windows build generally seems to have missing dependencies. +UNSTABLE_FILES = %r{\Awin32/[^/]+\.o\z} + # Other source files exist in the source directory. def in_makefile(target, source) target = target.to_s source = source.to_s case target - when %r{\A[^/]*\z}, %r{\Acoroutine/} + when %r{\A[^/]*\z}, %r{\Acoroutine/}, %r{\Aprism/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" @@ -194,7 +213,7 @@ def in_makefile(target, source) when %r{\Acoroutine/} then source2 = "{$(VPATH)}$(COROUTINE_H)" else source2 = "$(top_srcdir)/#{source}" end - ["common.mk", target2, source2] + ["depend", target2, source2] when %r{\Aenc/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source @@ -231,6 +250,10 @@ def in_makefile(target, source) else source2 = "$(top_srcdir)/#{source}" end ["#{File.dirname(target)}/depend", target2, source2] + # Files that may or may not exist on CI for some reason. + # Windows build generally seems to have missing dependencies. + when UNSTABLE_FILES + warn "warning: ignoring: #{target}" else raise "unexpected target: #{target}" end @@ -263,7 +286,7 @@ end def read_make_deps(cwd) dependencies = {} - make_p, make_p_stderr, make_p_status = Open3.capture3("make -p all miniruby ruby golf") + make_p, make_p_stderr, make_p_status = Open3.capture3("make -p all miniruby exe/ruby golf") File.open('update-deps.make.out.log', 'w') {|f| f.print make_p } File.open('update-deps.make.err.log', 'w') {|f| f.print make_p_stderr } if !make_p_status.success? @@ -300,7 +323,12 @@ def read_make_deps(cwd) target = rule_target deps = rule_sources deps = deps.scan(%r{[/0-9a-zA-Z._-]+}) + deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp next if /\.o\z/ !~ target.to_s + next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source) + next if /libzjit.o\z/ =~ target.to_s # skip ZJIT Rust object (no corresponding C source) + next if /target\/release\/libruby.o\z/ =~ target.to_s # skip YJIT+ZJIT Rust object (no corresponding C source) + next if /\.bundle\// =~ target.to_s next if /\A\./ =~ target.to_s # skip rules such as ".c.o" #p [curdir, target, deps] dir = curdir || dirstack.last @@ -332,24 +360,25 @@ end # raise ArgumentError, "can not find #{filename} (hint: #{hint0})" #end -def read_single_cc_deps(path_i, cwd) +def read_single_cc_deps(path_i, cwd, fn_o) files = {} compiler_wd = nil path_i.each_line {|line| next if /\A\# \d+ "(.*)"/ !~ line dep = $1 + next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc. - next if /\.erb\z/ =~ dep - compiler_wd ||= dep + next if /\.e?rb\z/ =~ dep + # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. + if /\/\/\z/ =~ dep + compiler_wd = Pathname(dep.sub(%r{//\z}, '')) + next + end + files[dep] = true } - # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. - if %r{\A/.*//\z} =~ compiler_wd - files.delete compiler_wd - compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, '')) - elsif !(compiler_wd = yield) - raise "compiler working directory not found: #{path_i}" - end + compiler_wd ||= fn_o.to_s.start_with?("enc/") ? cwd : path_i.parent + deps = [] files.each_key {|dep| dep = Pathname(dep) @@ -363,6 +392,9 @@ def read_single_cc_deps(path_i, cwd) next if !dep.to_s.start_with?(cwd.to_s) # omit system headers. deps << dep } + if deps.include?(cwd + "probes.h") + deps << (cwd + "probes.dmyh") + end deps end @@ -378,13 +410,7 @@ def read_cc_deps(cwd) end path_o = cwd + fn_o path_i = cwd + fn_i - deps[path_o] = read_single_cc_deps(path_i, cwd) do - if fn_o.to_s.start_with?("enc/") - cwd - else - path_i.parent - end - end + deps[path_o] = read_single_cc_deps(path_i, cwd, fn_o) } deps end @@ -472,7 +498,7 @@ def compare_deps(make_deps, cc_deps, out=$stdout) } } - makefiles.keys.sort.each {|makefile| + makefiles.keys.compact.sort.each {|makefile| cc_lines = cc_lines_hash[makefile] || Hash.new(false) make_lines = make_lines_hash[makefile] || Hash.new(false) content = begin @@ -525,7 +551,20 @@ def compare_deps(make_deps, cc_deps, out=$stdout) } end +def prepare_build + unless File.exist?("Makefile") + if File.exist?("autogen.sh") + system("./autogen.sh") + elsif !File.exist?("configure") + system("autoreconf", "-i", "-s") + end + system("./configure", "-q", "--enable-load-relative", "--prefix=/.", + "--disable-install-doc", "debugflags=-save-temps=obj -g") + end +end + def main_show(out=$stdout) + prepare_build make_deps, cc_deps = detect_dependencies(out) compare_deps(make_deps, cc_deps, out) end diff --git a/tool/wasm-clangw b/tool/wasm-clangw new file mode 100755 index 0000000000..9ebdfda75a --- /dev/null +++ b/tool/wasm-clangw @@ -0,0 +1,9 @@ +#!/bin/sh +# A Clang wrapper script to fake the clang linker driver. +# Clang linker automatically uses wasm-opt with -O if it found. +# However optimization before asyncify causes misoptimization, +# so wrap clang to insert our fake wasm-opt, which does nothing, in PATH. + +src_dir="$(cd "$(dirname "$0")/../wasm" && pwd)" +export PATH="$src_dir:$PATH" +exec "$@" diff --git a/tool/ytab.sed b/tool/ytab.sed deleted file mode 100755 index ba7566ac7f..0000000000 --- a/tool/ytab.sed +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sed -f -# This file is used when generating code for the Ruby parser. -/^int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^extern int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^yydestruct.*yymsg/,/{/{ - /^yydestruct/{ - /[, *]p)/!{ - H - s/^/ruby_parser_&/ - s/)$/, p)/ - /\*/s/p)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /yydestruct/{ - i\ -\ struct parser_params *p; - } - x - } - /^{/{ - x - /yydestruct/{ - i\ -#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, p) - } - x - } -} -/^yy_stack_print /,/{/{ - /^yy_stack_print/{ - /[, *]p)/!{ - H - s/^/ruby_parser_&/ - s/)$/, p)/ - /\*/s/p)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /yy_stack_print/{ - i\ -\ struct parser_params *p; - } - x - } - /^{/{ - x - /yy_stack_print/{ - i\ -#define yy_stack_print(b, t) ruby_parser_yy_stack_print(b, t, p) - } - x - } -} -/^yy_reduce_print/,/^}/{ - s/fprintf *(stderr,/YYFPRINTF (p,/g -} -s/^yysyntax_error (/&struct parser_params *p, / -s/ yysyntax_error (/&p, / -s/\( YYFPRINTF *(\)yyoutput,/\1p,/ -s/\( YYFPRINTF *(\)yyo,/\1p,/ -s/\( YYFPRINTF *(\)stderr,/\1p,/ -s/\( YYDPRINTF *((\)stderr,/\1p,/ -s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/ -s!^ *extern char \*getenv();!/* & */! -s/^\(#.*\)".*\.tab\.c"/\1"parse.c"/ -/^\(#.*\)".*\.y"/s:\\\\:/:g diff --git a/tool/zjit_bisect.rb b/tool/zjit_bisect.rb new file mode 100755 index 0000000000..997c572f51 --- /dev/null +++ b/tool/zjit_bisect.rb @@ -0,0 +1,158 @@ +#!/usr/bin/env ruby +require 'logger' +require 'optparse' +require 'shellwords' +require 'tempfile' +require 'timeout' + +ARGS = {timeout: 5} +OptionParser.new do |opts| + opts.banner += " <path_to_ruby> -- <options>" + opts.on("--timeout=TIMEOUT_SEC", "Seconds until child process is killed") do |timeout| + ARGS[:timeout] = Integer(timeout) + end + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end +end.parse! + +usage = "Usage: zjit_bisect.rb <path_to_ruby> -- <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +raise(usage) if OPTIONS.empty? +LOGGER = Logger.new($stdout) + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +# Attempt to reduce the `items` argument as much as possible, returning the +# shorter version. `fixed` will always be used as part of the items when +# running `command`. +# `command` should return True if the command succeeded (the failure did not +# reproduce) and False if the command failed (the failure reproduced). +def bisect_impl(command, fixed, items, indent="") + LOGGER.info("#{indent}step fixed[#{fixed.length}] and items[#{items.length}]") + while items.length > 1 + LOGGER.info("#{indent}#{fixed.length + items.length} candidates") + # Return two halves of the given list. For odd-length lists, the second + # half will be larger. + half = items.length / 2 + left = items[0...half] + right = items[half..] + if !command.call(fixed + left) + items = left + next + end + if !command.call(fixed + right) + items = right + next + end + # We need something from both halves to trigger the failure. Try + # holding each half fixed and bisecting the other half to reduce the + # candidates. + new_right = bisect_impl(command, fixed + left, right, indent + "< ") + new_left = bisect_impl(command, fixed + new_right, left, indent + "> ") + return new_left + new_right + end + items +end + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +def run_bisect(command, items) + LOGGER.info("Verifying items") + if command.call(items) + raise StandardError.new("Command succeeded with full items") + end + if !command.call([]) + raise StandardError.new("Command failed with empty items") + end + bisect_impl(command, [], items) +end + +def add_zjit_options cmd + if RUBY == "make" + # Automatically detect that we're running a make command instead of a Ruby + # one. Pass the bisection options via RUN_OPTS/SPECOPTS instead. + zjit_opts = cmd.select { |arg| arg.start_with?("--zjit") } + run_opts_index = cmd.find_index { |arg| arg.start_with?("RUN_OPTS=") } + specopts_index = cmd.find_index { |arg| arg.start_with?("SPECOPTS=") } + if run_opts_index + run_opts = Shellwords.split(cmd[run_opts_index].delete_prefix("RUN_OPTS=")) + run_opts.concat(zjit_opts) + cmd[run_opts_index] = "RUN_OPTS=#{run_opts.shelljoin}" + elsif specopts_index + specopts = Shellwords.split(cmd[specopts_index].delete_prefix("SPECOPTS=")) + specopts.concat(zjit_opts) + cmd[specopts_index] = "SPECOPTS=#{specopts.shelljoin}" + else + raise "Expected RUN_OPTS or SPECOPTS to be present in make command" + end + cmd = cmd - zjit_opts + end + cmd +end + +def run_ruby *cmd + cmd = add_zjit_options(cmd) + pid = Process.spawn(*cmd, { + in: :close, + out: [File::NULL, File::RDWR], + err: [File::NULL, File::RDWR], + }) + begin + status = Timeout.timeout(ARGS[:timeout]) do + Process::Status.wait(pid) + end + rescue Timeout::Error + Process.kill("KILL", pid) + LOGGER.warn("Timed out after #{ARGS[:timeout]} seconds") + status = Process::Status.wait(pid) + end + + status +end + +def run_with_jit_list(ruby, options, jit_list) + # Make a new temporary file containing the JIT list + Tempfile.create("jit_list") do |temp_file| + temp_file.write(jit_list.join("\n")) + temp_file.flush + temp_file.close + # Run the JIT with the temporary file + run_ruby ruby, "--zjit-allowed-iseqs=#{temp_file.path}", *options + end +end + +# Try running with no JIT list to get a stable baseline +unless run_with_jit_list(RUBY, OPTIONS, []).success? + cmd = [RUBY, "--zjit-allowed-iseqs=/dev/null", *OPTIONS].shelljoin + raise "The command failed unexpectedly with an empty JIT list. To reproduce, try running the following: `#{cmd}`" +end +# Collect the JIT list from the failing Ruby process +jit_list = nil +Tempfile.create "jit_list" do |temp_file| + run_ruby RUBY, "--zjit-log-compiled-iseqs=#{temp_file.path}", *OPTIONS + jit_list = File.readlines(temp_file.path).map(&:strip).reject(&:empty?) +end +LOGGER.info("Starting with JIT list of #{jit_list.length} items.") +# Try running without the optimizer +status = run_with_jit_list(RUBY, ["--zjit-disable-hir-opt", *OPTIONS], jit_list) +if status.success? + LOGGER.warn "*** Command suceeded with HIR optimizer disabled. HIR optimizer is probably at fault. ***" +end +# Now narrow it down +command = lambda do |items| + run_with_jit_list(RUBY, OPTIONS, items).success? +end +result = run_bisect(command, jit_list) +File.open("jitlist.txt", "w") do |file| + file.puts(result) +end +puts "Run:" +jitlist_path = File.expand_path("jitlist.txt") +puts add_zjit_options([RUBY, "--zjit-allowed-iseqs=#{jitlist_path}", *OPTIONS]).shelljoin +puts "Reduced JIT list (available in jitlist.txt):" +puts result diff --git a/tool/zjit_iongraph.html b/tool/zjit_iongraph.html new file mode 100644 index 0000000000..993cce9045 --- /dev/null +++ b/tool/zjit_iongraph.html @@ -0,0 +1,551 @@ +<!-- Copyright Mozilla and licensed under Mozilla Public License Version 2.0. + Source can be found at https://github.com/mozilla-spidermonkey/iongraph --> +<!-- Generated by `npm run build-www` on + 39b04fa18f23cbf3fd2ca7339a45341ff3351ba1 in tekknolagi/iongraph --> +<!DOCTYPE html> + +<head> + <title>iongraph</title> + <style>/* iongraph-specific styles inspired by Tachyons */ + +:root { + --ig-size-1: 1rem; + --ig-size-2: 2rem; + --ig-size-3: 4rem; + --ig-size-4: 8rem; + --ig-size-5: 16rem; + + --ig-spacing-1: .25rem; + --ig-spacing-2: .5rem; + --ig-spacing-3: 1rem; + --ig-spacing-4: 2rem; + --ig-spacing-5: 4rem; + --ig-spacing-6: 8rem; + --ig-spacing-7: 16rem; + + --ig-text-color: black; + --ig-text-color-dim: #777; + --ig-background-primary: #ffb54e; + --ig-background-light: white; + --ig-border-color: #0c0c0d; + + --ig-block-header-color: #0c0c0d; + --ig-loop-header-color: #1fa411; + --ig-movable-color: #1048af; + --ig-rob-color: #444; + --ig-in-worklist-color: red; + + --ig-block-selected: #ffc863; + --ig-block-last-selected: #ffb54e; + + --ig-highlight-0: #ffb54e; + --ig-highlight-1: #ffb5c5; + --ig-highlight-2: #a4cbff; + --ig-highlight-3: #8be182; + --ig-highlight-4: #d9a4fd; + + --ig-flash-color: #ffb54e; + + /* + * The heatmap of sample counts will effectively be sampled from a gradient: + * + * |----------|---------------------------------------| + * cold "cool" hot + * + * The "cold" color will simply be transparent. Therefore, the "cool" + * threshold indicates where the instruction will be fully colored and + * noticeable to the user. + */ + --ig-hot-color: #ff849e; + --ig-cool-color: #ffe546; + --ig-cool-threshold: 0.2; +} + +a.ig-link-normal { + color: inherit; + text-decoration: inherit; +} + +.ig-flex { + display: flex; +} + +.ig-flex-column { + flex-direction: column; +} + +.ig-flex-basis-0 { + flex-basis: 0; +} + +.ig-flex-grow-1 { + flex-grow: 1; +} + +.ig-flex-shrink-0 { + flex-shrink: 0; +} + +.ig-flex-shrink-1 { + flex-shrink: 1; +} + +.ig-items-center { + align-items: center; +} + +.ig-relative { + position: relative; +} + +.ig-absolute { + position: absolute; +} + +.ig-absolute-fill { + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.ig-g1 { + gap: var(--ig-spacing-1); +} + +.ig-g2 { + gap: var(--ig-spacing-2); +} + +.ig-g3 { + gap: var(--ig-spacing-3); +} + +.ig-w1 { + width: var(--ig-size-1); +} + +.ig-w2 { + width: var(--ig-size-2); +} + +.ig-w3 { + width: var(--ig-size-3); +} + +.ig-w4 { + width: var(--ig-size-4); +} + +.ig-w5 { + width: var(--ig-size-5); +} + +.ig-w-100 { + width: 100%; +} + +.ig-ba { + border-style: solid; + border-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bt { + border-top-style: solid; + border-top-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-br { + border-right-style: solid; + border-right-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bb { + border-bottom-style: solid; + border-bottom-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bl { + border-left-style: solid; + border-left-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-pa1 { + padding: var(--ig-spacing-1); +} + +.ig-pa2 { + padding: var(--ig-spacing-2); +} + +.ig-pa3 { + padding: var(--ig-spacing-3); +} + +.ig-ph1 { + padding-left: var(--ig-spacing-1); + padding-right: var(--ig-spacing-1); +} + +.ig-ph2 { + padding-left: var(--ig-spacing-2); + padding-right: var(--ig-spacing-2); +} + +.ig-ph3 { + padding-left: var(--ig-spacing-3); + padding-right: var(--ig-spacing-3); +} + +.ig-pv1 { + padding-top: var(--ig-spacing-1); + padding-bottom: var(--ig-spacing-1); +} + +.ig-pv2 { + padding-top: var(--ig-spacing-2); + padding-bottom: var(--ig-spacing-2); +} + +.ig-pv3 { + padding-top: var(--ig-spacing-3); + padding-bottom: var(--ig-spacing-3); +} + +.ig-pt1 { + padding-top: var(--ig-spacing-1); +} + +.ig-pt2 { + padding-top: var(--ig-spacing-2); +} + +.ig-pt3 { + padding-top: var(--ig-spacing-3); +} + +.ig-pr1 { + padding-right: var(--ig-spacing-1); +} + +.ig-pr2 { + padding-right: var(--ig-spacing-2); +} + +.ig-pr3 { + padding-right: var(--ig-spacing-3); +} + +.ig-pb1 { + padding-bottom: var(--ig-spacing-1); +} + +.ig-pb2 { + padding-bottom: var(--ig-spacing-2); +} + +.ig-pb3 { + padding-bottom: var(--ig-spacing-3); +} + +.ig-pl1 { + padding-left: var(--ig-spacing-1); +} + +.ig-pl2 { + padding-left: var(--ig-spacing-2); +} + +.ig-pl3 { + padding-left: var(--ig-spacing-3); +} + +.ig-f1 { + font-size: 3rem; +} + +.ig-f2 { + font-size: 2.25rem; +} + +.ig-f3 { + font-size: 1.5rem; +} + +.ig-f4 { + font-size: 1.25rem; +} + +.ig-f5 { + font-size: 1rem; +} + +.ig-f6 { + font-size: .875rem; +} + +.ig-f7 { + font-size: .75rem; +} + +.ig-text-normal { + color: var(--ig-text-color); +} + +.ig-text-dim { + color: var(--ig-text-color-dim); +} + +.ig-tl { + text-align: left; +} + +.ig-tr { + text-align: right; +} + +.ig-tc { + text-align: center; +} + +.ig-bg-white { + background-color: var(--ig-background-light); +} + +.ig-bg-primary { + background-color: var(--ig-background-primary); +} + +.ig-overflow-hidden { + overflow: hidden; +} + +.ig-overflow-auto { + overflow: auto; +} + +.ig-overflow-x-auto { + overflow-x: auto; +} + +.ig-overflow-y-auto { + overflow-y: auto; +} + +.ig-hide-if-empty:empty { + display: none; +} + +/* Non-utility styles */ + +.ig-graph { + color: var(--ig-text-color); + position: absolute; + left: 0; + top: 0; + isolation: isolate; +} + +.ig-block { + position: absolute; + + .ig-block-header { + font-weight: bold; + text-align: center; + background-color: var(--ig-block-header-color); + color: white; + padding: 0 1em; + border: 1px solid var(--ig-border-color); + border-width: 1px 1px 0; + } + + .ig-instructions { + padding: 0.5em; + border: 1px solid var(--ig-border-color); + border-width: 0 1px 1px; + + table { + border-collapse: collapse; + } + + td, + th { + white-space: nowrap; + padding: 0.1em 0.5em; + } + + th { + font-weight: normal; + } + } + + &.ig-selected { + outline: 4px solid var(--ig-block-selected); + } + + &.ig-last-selected { + outline-color: var(--ig-block-last-selected); + } +} + +.ig-block-att-loopheader { + .ig-block-header { + background-color: var(--ig-loop-header-color); + } +} + +.ig-block-att-splitedge { + .ig-instructions { + border-style: dotted; + border-width: 0 2px 2px; + } +} + +.ig-ins-num { + text-align: right; + cursor: pointer; +} + +.ig-ins-type { + text-align: right; +} + +.ig-ins-samples { + font-size: 0.875em; + text-align: right; + cursor: pointer; +} + +.ig-use { + padding: 0 0.25em; + border-radius: 2px; + cursor: pointer; +} + +.ig-edge-label { + position: absolute; + font-size: 0.8em; + line-height: 1; + bottom: -1em; + padding-left: 4px; +} + +.ig-ins-att-RecoveredOnBailout { + color: var(--ig-rob-color); +} + +.ig-ins-att-Movable { + color: var(--ig-movable-color); +} + +.ig-ins-att-Guard { + text-decoration: underline; +} + +.ig-ins-att-InWorklist { + color: var(--ig-in-worklist-color); +} + +.ig-can-flash { + transition: outline-color 1s ease-out; + outline: 3px solid color-mix(in srgb, var(--ig-flash-color) 0%, transparent); +} + +.ig-flash { + transition: outline-color 0s; + outline-color: var(--ig-flash-color); +} + +.ig-hotness { + --ig-hotness: 0; + + --ig-cold-color: color-mix(in srgb, var(--ig-cool-color) 20%, transparent); + background-color: + /* cool <-> hot */ + color-mix(in oklab, + /* cold <-> cool */ + color-mix(in oklab, + /* dead or cold */ + color-mix(in srgb, transparent, var(--ig-cold-color) clamp(0%, calc(var(--ig-hotness) * 100000000%), 100%)), + var(--ig-cool-color) clamp(0%, calc((var(--ig-hotness) / var(--ig-cool-threshold)) * 100%), 100%)), + var(--ig-hot-color) clamp(0%, calc(((var(--ig-hotness) - var(--ig-cool-threshold)) / (1 - var(--ig-cool-threshold))) * 100%), 100%)); +} + +.ig-highlight { + --ig-highlight-color: transparent; + background-color: var(--ig-highlight-color); +}</style> + <style>* { + box-sizing: border-box; +} + +:root { + font-size: 0.875rem; +} + +body { + margin: 0; + background-color: #e5e8ea; + /* Font, and many other styles, taken from the Firefox Profiler/ */ + font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen-Sans, Ubuntu, Noto Sans, Liberation Sans, Cantarell, Helvetica Neue, sans-serif; +} + +#container { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +.tweaks-panel { + position: absolute; + bottom: 0; + right: 0; + padding: 1rem; + border: 1px solid black; + border-width: 1px 0 0 1px; + background-color: white; +}</style> +</head> + +<body> + <script>"use strict";var iongraph=(()=>{var J=Object.defineProperty;var mt=Object.getOwnPropertyDescriptor;var gt=Object.getOwnPropertyNames;var ft=Object.prototype.hasOwnProperty;var bt=(a,t)=>{for(var e in t)J(a,e,{get:t[e],enumerable:!0})},kt=(a,t,e,s)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of gt(t))!ft.call(a,i)&&i!==e&&J(a,i,{get:()=>t[i],enumerable:!(s=mt(t,i))||s.enumerable});return a};var yt=a=>kt(J({},"__esModule",{value:!0}),a);var Yt={};bt(Yt,{StandaloneUI:()=>et,WebUI:()=>tt});function X(a){a.version===void 0&&(a.version=0);for(let t of a.functions)xt(t,a.version);return a.version=1,a}function xt(a,t){for(let e of a.passes){for(let s of e.mir.blocks)wt(s,t);for(let s of e.lir.blocks)It(s,t)}return a}function wt(a,t){t===0&&(a.ptr=(a.id??a.number)+1,a.id=a.number);for(let e of a.instructions)vt(e,t);return a}function vt(a,t){return t===0&&(a.ptr=a.id),a}function It(a,t){t===0&&(a.ptr=a.id??a.number,a.id=a.number);for(let e of a.instructions)Lt(e,t);return a}function Lt(a,t){return t===0&&(a.ptr=a.id,a.mirPtr=null),a}function q(a,t,e){return Math.max(t,Math.min(e,a))}function R(a,t,e,s){return(a-t)*Math.pow(e,s)+t}function k(a,t,e=!1){if(!a)if(e)console.error(t??"Assertion failed");else throw new Error(t??"Assertion failed")}function L(a,t){return k(a,t),a}function Pt(a){return typeof a=="string"?document.createTextNode(a):a}function Bt(a,t){for(let e of t)e&&a.appendChild(Pt(e))}function x(a,t,e,s){let i=document.createElement(a);if(t&&t.length>0){let c=t.filter(l=>!!l);i.classList.add(...c)}return e?.(i),s&&Bt(i,s),i}var st=Object.prototype.hasOwnProperty;function E(a,t){var e,s;if(a===t)return!0;if(a&&t&&(e=a.constructor)===t.constructor){if(e===Date)return a.getTime()===t.getTime();if(e===RegExp)return a.toString()===t.toString();if(e===Array){if((s=a.length)===t.length)for(;s--&&E(a[s],t[s]););return s===-1}if(!e||typeof a=="object"){s=0;for(e in a)if(st.call(a,e)&&++s&&!st.call(t,e)||!(e in t)||!E(a[e],t[e]))return!1;return Object.keys(t).length===s}}return a!==a&&t!==t}function N(a,t,e={}){let s=t,i=[],c={get(){return s},set(l){s=l;for(let o of i)o(s)},valueOf(){return s},toString(){return String(s)},[Symbol.toPrimitive](l){return l==="string"?String(s):s},onChange(l){i.push(l)},initial:t,name:a,min:e.min??0,max:e.max??100,step:e.step??1};return(e.tweaksObject??K).add(c)}var F=class{constructor(t){this.container=t.container,this.tweaks=[],this.callbacks=[]}add(t){let e=this.tweaks.find(r=>r.name===t.name);if(e)return e;this.tweaks.push(t);let s=document.createElement("div");this.container.appendChild(s),s.style.display="flex",s.style.alignItems="center",s.style.justifyContent="end",s.style.gap="0.5rem";let i=t.name.replace(/[a-zA-Z0-9]/g,"_"),c=document.createElement("label");s.appendChild(c),c.innerText=t.name,c.htmlFor=`tweak-${i}-input`;let l=document.createElement("input");s.appendChild(l),l.type="number",l.value=String(t),l.id=`tweak-${i}-input`,l.style.width="4rem",l.addEventListener("input",()=>{t.set(l.valueAsNumber)});let o=document.createElement("input");s.appendChild(o),o.type="range",o.value=String(t),o.min=String(t.min),o.max=String(t.max),o.step=t.step===0?"any":String(t.step),o.addEventListener("input",()=>{t.set(o.valueAsNumber)});let n=document.createElement("button");return s.appendChild(n),n.innerText="Reset",n.disabled=t.get()===t.initial,n.addEventListener("click",()=>{t.set(t.initial)}),t.onChange(r=>{l.value=String(r),o.value=String(r),n.disabled=t.get()===t.initial;for(let d of this.callbacks)d(t)}),t}onTweak(t){this.callbacks.push(t)}},nt=document.createElement("div");nt.classList.add("tweaks-panel");var K=new F({container:nt});K.onTweak(a=>{window.dispatchEvent(new CustomEvent("tweak",{detail:a}))});window.tweaks=K;var Nt=document.createElement("div"),St=new F({container:Nt}),ot=N("Test Value",3,{tweaksObject:St});ot.set(4);ot=4;var dt=N("Debug?",0,{min:0,max:1}),P=20,C=44,I=16,S=60,T=12,H=36,it=16,D=16,Tt=N("Layout Iterations",2,{min:0,max:6}),rt=N("Nearly Straight Threshold",30,{min:0,max:200}),$t=N("Nearly Straight Iterations",8,{min:0,max:10}),at=N("Stop At Pass",30,{min:0,max:30}),Mt=1.5,Et=.01,Ct=1,Ht=.1,z=40;function _(a){return a.mir.attributes.includes("loopheader")}function Dt(a){return a.loopHeight!==void 0}function $(a){if(k(a),Dt(a))return a;throw new Error(`Block ${a.id} is not a pseudo LoopHeader`)}var j=1,lt=2,U=4,O=0,W=1,w=new Proxy(console,{get(a,t){let e=a[t];return typeof e!="function"?e:+dt?e.bind(a):()=>{}}}),V=class{constructor(t,e,s={}){this.viewport=t;let i=t.getBoundingClientRect();this.viewportSize={x:i.width,y:i.height},this.graphContainer=document.createElement("div"),this.graphContainer.classList.add("ig-graph"),this.graphContainer.style.transformOrigin="top left",this.viewport.appendChild(this.graphContainer),this.sampleCounts=s.sampleCounts,this.maxSampleCounts=[0,0],this.heatmapMode=O;for(let[n,r]of this.sampleCounts?.totalLineHits??[])this.maxSampleCounts[O]=Math.max(this.maxSampleCounts[O],r);for(let[n,r]of this.sampleCounts?.selfLineHits??[])this.maxSampleCounts[W]=Math.max(this.maxSampleCounts[W],r);this.size={x:0,y:0},this.numLayers=0,this.zoom=1,this.translation={x:0,y:0},this.animating=!1,this.targetZoom=1,this.targetTranslation={x:0,y:0},this.startMousePos={x:0,y:0},this.lastMousePos={x:0,y:0},this.selectedBlockPtrs=new Set,this.lastSelectedBlockPtr=0,this.nav={visited:[],currentIndex:-1,siblings:[]},this.highlightedInstructions=[],this.instructionPalette=s.instructionPalette??[0,1,2,3,4].map(n=>`var(--ig-highlight-${n})`),this.blocks=e.mir.blocks.map(n=>{let r={ptr:n.ptr,id:n.id,mir:n,lir:e.lir.blocks.find(d=>d.id===n.id)??null,preds:[],succs:[],el:void 0,size:{x:0,y:0},layer:-1,loopID:-1,layoutNode:void 0};if(r.mir.attributes.includes("loopheader")){let d=r;d.loopHeight=0,d.parentLoop=null,d.outgoingEdges=[]}return k(r.ptr,"blocks must always have non-null ptrs"),r}),this.blocksByID=new Map,this.blocksByPtr=new Map,this.insPtrsByID=new Map,this.insIDsByPtr=new Map,this.loops=[];for(let n of this.blocks){this.blocksByID.set(n.id,n),this.blocksByPtr.set(n.ptr,n);for(let r of n.mir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id);if(n.lir)for(let r of n.lir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id)}for(let n of this.blocks)if(n.preds=n.mir.predecessors.map(r=>L(this.blocksByID.get(r))),n.succs=n.mir.successors.map(r=>L(this.blocksByID.get(r))),_(n)){let r=n.preds.filter(d=>d.mir.attributes.includes("backedge"));k(r.length===1),n.backedge=r[0]}for(let n of this.blocks)n.el=this.renderBlock(n);for(let n of this.blocks)n.size={x:n.el.clientWidth,y:n.el.clientHeight};let[c,l,o]=this.layout();this.render(c,l,o),this.addEventListeners()}layout(){let[t,e]=this.findLayoutRoots();w.log("Layout roots:",t.map(l=>l.id));for(let l of[...t,...e]){let o=l;o.loopHeight=0,o.parentLoop=null,o.outgoingEdges=[],Object.defineProperty(o,"backedge",{get(){throw new Error("Accessed .backedge on a pseudo loop header! Don't do that.")},configurable:!0})}for(let l of t)w.group("findLoops"),this.findLoops(l),w.groupEnd();for(let l of t)w.group("layer"),this.layer(l),w.groupEnd();for(let l of e)l.layer=0,l.loopID=l.id;let s=this.makeLayoutNodes();this.straightenEdges(s);let i=this.finagleJoints(s),c=this.verticalize(s,i);return[s,c,i]}findLayoutRoots(){let t=[],e=[],s=this.blocks.filter(i=>i.preds.length===0);for(let i of s){let c=i;if(i.mir.attributes.includes("osr")){k(i.succs.length>0);let l=i.succs[0];c=l;for(let o=0;;o++){if(o>=1e7)throw new Error("likely infinite loop");let n=c.preds.filter(r=>!r.mir.attributes.includes("osr")&&!r.mir.attributes.includes("backedge"));if(n.length===0)break;c=n[0]}c!==l?e.push(i):c=i}t.includes(c)||t.push(c)}return[t,e]}findLoops(t,e=null){if(e===null&&(e=[t.id]),!(t.loopID>=0)){if(w.log("block:",t.id,t.mir.loopDepth,"loopIDsByDepth:",e),w.log(t.mir.attributes),_(t)){let s=e[e.length-1],i=$(this.blocksByID.get(s));t.parentLoop=i,e=[...e,t.id],w.log("Block",t.id,"is true loop header, loopIDsByDepth is now",e)}if(t.mir.loopDepth>e.length-1&&(t.mir.loopDepth=e.length-1,w.log("Block",t.id,"has been forced back to loop depth",t.mir.loopDepth)),t.mir.loopDepth<e.length-1&&(e=e.slice(0,t.mir.loopDepth+1),w.log("Block",t.id,"has low loop depth, therefore we exited a loop. loopIDsByDepth:",e)),t.loopID=e[t.mir.loopDepth],!t.mir.attributes.includes("backedge"))for(let s of t.succs)this.findLoops(s,e)}}layer(t,e=0){if(w.log("block",t.id,"layer",e),t.mir.attributes.includes("backedge")){t.layer=t.succs[0].layer;return}if(e<=t.layer)return;t.layer=Math.max(t.layer,e),this.numLayers=Math.max(t.layer+1,this.numLayers);let s=$(this.blocksByID.get(t.loopID));for(;s;)s.loopHeight=Math.max(s.loopHeight,t.layer-s.layer+1),s=s.parentLoop;for(let i of t.succs)i.mir.loopDepth<t.mir.loopDepth?$(this.blocksByID.get(t.loopID)).outgoingEdges.push(i):this.layer(i,e+1);if(_(t))for(let i of t.outgoingEdges)this.layer(i,e+t.loopHeight)}makeLayoutNodes(){w.group("makeLayoutNodes");function t(o,n,r){o.dstNodes[n]=r,r.srcNodes.includes(o)||r.srcNodes.push(o),w.log("connected",o.id,"to",r.id)}let e;{let o={};for(let n of this.blocks)o[n.layer]||(o[n.layer]=[]),o[n.layer].push(n);e=Object.entries(o).map(([n,r])=>[Number(n),r]).sort((n,r)=>n[0]-r[0]).map(([n,r])=>r)}let s=0,i=e.map(()=>[]),c=[],l=new Map;for(let[o,n]of e.entries()){w.group("layer",o,"blocks",n.map(u=>u.id));let r=[];for(let u of n)for(let h=c.length-1;h>=0;h--){let p=c[h];p.dstBlock===u&&(r.unshift(p),c.splice(h,1))}let d=new Map;for(let u of c){let h,p=d.get(u.dstBlock.id);if(p)t(u.src,u.srcPort,p),h=p;else{let m={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:u.dstBlock,jointOffsets:[],flags:0};t(u.src,u.srcPort,m),i[o].push(m),d.set(u.dstBlock.id,m),h=m,w.log("Created dummy",h.id,"on the way to block",u.dstBlock.id)}u.src=h,u.srcPort=0}let b=[];for(let u of n){let h=$(this.blocksByID.get(u.loopID));for(;_(h);){let p=b.find(f=>f.loopID===h.id);p?p.block=u:b.push({loopID:h.id,block:u});let m=h.parentLoop;if(!m)break;h=m}}let g=[];for(let u of n){let h={id:s++,pos:{x:P,y:P},size:u.size,block:u,srcNodes:[],dstNodes:[],jointOffsets:[],flags:0};for(let p of r)p.dstBlock===u&&t(p.src,p.srcPort,h);i[o].push(h),u.layoutNode=h;for(let p of b.filter(m=>m.block===u)){let m=$(this.blocksByID.get(p.loopID)).backedge,f={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:m,jointOffsets:[],flags:0},y=l.get(m);y?t(f,0,y):(f.flags|=U,t(f,0,m.layoutNode)),i[o].push(f),l.set(m,f)}if(u.mir.attributes.includes("backedge"))t(u.layoutNode,0,u.succs[0].layoutNode);else for(let[p,m]of u.succs.entries())m.mir.attributes.includes("backedge")?g.push({src:h,srcPort:p,dstBlock:m}):c.push({src:h,srcPort:p,dstBlock:m})}for(let u of g){let h=L(l.get(u.dstBlock));t(u.src,u.srcPort,h)}w.groupEnd()}w.log("Pruning backedge dummies");{let o=[];for(let r of At(i))r.srcNodes.length===0&&o.push(r);let n=new Set;for(let r of o){let d=r;for(;d.block===null&&d.srcNodes.length===0;)Ot(d),n.add(d),k(d.dstNodes.length===1),d=d.dstNodes[0]}for(let r of i)for(let d=r.length-1;d>=0;d--)n.has(r[d])&&r.splice(d,1)}w.log("Marking leftmost and rightmost dummies");for(let o of i){for(let n=0;n<o.length&&o[n].block===null;n++)o[n].flags|=j;for(let n=o.length-1;n>=0&&o[n].block===null;n--)o[n].flags|=lt}w.log("Verifying integrity of all nodes");for(let o of i)for(let n of o){n.block?k(n.dstNodes.length===n.block.succs.length,`expected node ${n.id} for block ${n.block.id} to have ${n.block.succs.length} destination nodes, but got ${n.dstNodes.length} instead`):k(n.dstNodes.length===1,`expected dummy node ${n.id} to have only one destination node, but got ${n.dstNodes.length} instead`);for(let r=0;r<n.dstNodes.length;r++)k(n.dstNodes[r]!==void 0,`dst slot ${r} of node ${n.id} was undefined`)}return w.groupEnd(),i}straightenEdges(t){let e=g=>{for(let u=0;u<g.length-1;u++){let h=g[u],p=g[u+1],m=h.block===null&&p.block!==null,f=h.pos.x+h.size.x+(m?I:0)+C;p.pos.x=Math.max(p.pos.x,f)}},s=()=>{for(let g of t)for(let u of g){if(u.block===null)continue;let h=u.block.loopID!==null?$(this.blocksByID.get(u.block.loopID)):null;if(h){let p=h.layoutNode;u.pos.x=Math.max(u.pos.x,p.pos.x)}}},i=()=>{let g=new Map;for(let u of Q(t)){let h=u.dstBlock,p=u.pos.x;g.set(h,Math.max(g.get(h)??0,p))}for(let u of Q(t)){let h=u.dstBlock,p=g.get(h);k(p,`no position for backedge ${h.id}`),u.pos.x=p}for(let u of t)e(u)},c=()=>{let g=new Map;for(let u of t){let h=0,p=0;for(;h<u.length;h++)if(!(u[h].flags&j)){p=u[h].pos.x;break}for(h-=1,p-=C+I;h>=0;h--){let m=u[h];k(m.block===null&&m.flags&j);let f=p;for(let y of m.srcNodes){let v=y.pos.x+y.dstNodes.indexOf(m)*S;v<f&&(f=v)}m.pos.x=f,p=m.pos.x-C,g.set(m.dstBlock,Math.min(g.get(m.dstBlock)??1/0,f))}}for(let u of Q(t)){if(!(u.flags&j))continue;let h=g.get(u.dstBlock);k(h,`no position for run to block ${u.dstBlock.id}`),u.pos.x=h}},l=()=>{for(let g=0;g<t.length-1;g++){let u=t[g];e(u);let h=-1;for(let p of u)for(let[m,f]of p.dstNodes.entries()){let y=t[g+1].indexOf(f);if(y>h&&f.srcNodes[0]===p){let v=I+S*m,B=I,Z=f.pos.x;f.pos.x=Math.max(f.pos.x,p.pos.x+v-B),f.pos.x!==Z&&(h=y)}}}},o=()=>{for(let g of t){for(let u=g.length-1;u>=0;u--){let h=g[u];if(!h.block||h.block.mir.attributes.includes("backedge"))continue;let p=[];for(let m of h.srcNodes){let f=I+m.dstNodes.indexOf(h)*S,y=I;p.push(m.pos.x+f-(h.pos.x+y))}for(let[m,f]of h.dstNodes.entries()){if(f.block===null&&f.dstBlock.mir.attributes.includes("backedge"))continue;let y=I+m*S,v=I;p.push(f.pos.x+v-(h.pos.x+y))}if(!p.includes(0)){p=p.filter(m=>m>0).sort((m,f)=>m-f);for(let m of p){let f=!1;for(let y=u+1;y<g.length;y++){let v=g[y];if(v.flags<)continue;let B=h.pos.x+m,Z=h.pos.x+m+h.size.x,ht=v.pos.x-C,pt=v.pos.x+v.size.x+C;Z>=ht&&B<=pt&&(f=!0)}if(!f){h.pos.x+=m;break}}}}e(g)}},n=()=>{for(let g=t.length-1;g>=0;g--){let u=t[g];e(u);for(let h of u)for(let p of h.srcNodes){if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}},r=()=>{for(let g=0;g<t.length;g++){let u=t[g];e(u);for(let h of u){if(h.dstNodes.length===0)continue;let p=h.dstNodes[0];if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}};function d(g,u){let h=[];for(let p=0;p<u;p++)for(let m of g)h.push(m);return h}let b=[...d([l,s,i],Tt),i,...d([n,r],$t),o,i,c];k(b.length<=(at.initial??1/0),`STOP_AT_PASS was too small - should be at least ${b.length}`),w.group("Running passes");for(let[g,u]of b.entries())g<at&&(w.log(u.name??u.toString()),u());w.groupEnd()}finagleJoints(t){let e=[];for(let s of t){let i=[];for(let r of s)if(r.jointOffsets=new Array(r.dstNodes.length).fill(0),!r.block?.mir.attributes.includes("backedge"))for(let[d,b]of r.dstNodes.entries()){let g=r.pos.x+I+S*d,u=b.pos.x+I;Math.abs(u-g)<2*T||i.push({x1:g,x2:u,src:r,srcPort:d,dst:b})}i.sort((r,d)=>r.x1-d.x1);let c=[],l=[];t:for(let r of i){let d=r.x2-r.x1>=0?c:l,b=null;for(let g=d.length-1;g>=0;g--){let u=d[g],h=!1;for(let p of u){if(r.dst===p.dst){u.push(r);continue t}let m=Math.min(r.x1,r.x2),f=Math.max(r.x1,r.x2),y=Math.min(p.x1,p.x2),v=Math.max(p.x1,p.x2);if(f>=y&&m<=v){h=!0;break}}if(h)break;b=u}b?b.push(r):d.push([r])}let o=Math.max(0,c.length+l.length-1)*it,n=-o/2;for(let r of[...c.reverse(),...l]){for(let d of r)d.src.jointOffsets[d.srcPort]=n;n+=it}e.push(o)}return k(e.length===t.length),e}verticalize(t,e){let s=new Array(t.length),i=P;for(let c=0;c<t.length;c++){let l=t[c],o=0;for(let n of l)n.pos.y=i,o=Math.max(o,n.size.y);s[c]=o,i+=o+H+e[c]+H}return s}renderBlock(t){let e=document.createElement("div");this.graphContainer.appendChild(e),e.classList.add("ig-block","ig-bg-white");for(let o of t.mir.attributes)e.classList.add(`ig-block-att-${o}`);e.setAttribute("data-ig-block-ptr",`${t.ptr}`),e.setAttribute("data-ig-block-id",`${t.id}`);let s="";t.mir.attributes.includes("loopheader")?s=" (loop header)":t.mir.attributes.includes("backedge")?s=" (backedge)":t.mir.attributes.includes("splitedge")&&(s=" (split edge)");let i=document.createElement("div");i.classList.add("ig-block-header"),i.innerText=`Block ${t.id}${s}`,e.appendChild(i);let c=document.createElement("div");c.classList.add("ig-instructions"),e.appendChild(c);let l=document.createElement("table");if(t.lir){l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + ${this.sampleCounts?` + <col style="width: 1px"> + <col style="width: 1px"> + `:""} + </colgroup> + ${this.sampleCounts?` + <thead> + <tr> + <th></th> + <th></th> + <th class="ig-f6">Total</th> + <th class="ig-f6">Self</th> + </tr> + </thead> + `:""} + `;for(let o of t.lir.instructions)l.appendChild(this.renderLIRInstruction(o))}else{l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + <col style="width: 1px"> + </colgroup> + `;for(let o of t.mir.instructions)l.appendChild(this.renderMIRInstruction(o))}if(c.appendChild(l),t.succs.length===2)for(let[o,n]of[1,0].entries()){let r=document.createElement("div");r.innerText=`${n}`,r.classList.add("ig-edge-label"),r.style.left=`${I+S*o}px`,e.appendChild(r)}return i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",o=>{o.stopPropagation(),o.shiftKey||this.selectedBlockPtrs.clear(),this.setSelection([],t.ptr)}),e}render(t,e,s){for(let n of t)for(let r of n)if(r.block!==null){let d=r.block;d.el.style.left=`${r.pos.x}px`,d.el.style.top=`${r.pos.y}px`}let i=0,c=0;for(let n of t)for(let r of n)i=Math.max(i,r.pos.x+r.size.x+P),c=Math.max(c,r.pos.y+r.size.y+P);let l=document.createElementNS("http://www.w3.org/2000/svg","svg");this.graphContainer.appendChild(l);let o=(n,r)=>{for(let d of n)i=Math.max(i,d+P);for(let d of r)c=Math.max(c,d+P)};for(let n=0;n<t.length;n++){let r=t[n];for(let d of r){d.block||k(d.dstNodes.length===1,`dummy nodes must have exactly one destination, but dummy ${d.id} had ${d.dstNodes.length}`),k(d.dstNodes.length===d.jointOffsets.length,"must have a joint offset for each destination");for(let[b,g]of d.dstNodes.entries()){let u=d.pos.x+I+S*b,h=d.pos.y+d.size.y;if(d.block?.mir.attributes.includes("backedge")){let p=d.block.succs[0],m=d.pos.x,f=d.pos.y+D,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=jt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(d.flags&U){let p=L(g.block),m=d.pos.x+I,f=d.pos.y+D+T,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=zt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(g.block===null&&g.dstBlock.mir.attributes.includes("backedge")){let p=g.pos.x+I,m=g.pos.y+(g.flags&U?D+T:0);if(d.block===null){let f=h-H,y=Ft(u,h,p,m,f,!1);l.appendChild(y),o([u,p],[h,m,f])}else{let f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=_t(u,h,p,m,f);l.appendChild(y),o([u,p],[h,m,f])}}else{let p=g.pos.x+I,m=g.pos.y,f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=Rt(u,h,p,m,f,g.block!==null);l.appendChild(y),o([u,p],[h,m,f])}}}}if(l.setAttribute("width",`${i}`),l.setAttribute("height",`${c}`),this.size={x:i,y:c},+dt)for(let n of t)for(let r of n){let d=document.createElement("div");d.innerHTML=`${r.id}<br><- ${r.srcNodes.map(b=>b.id)}<br>-> ${r.dstNodes.map(b=>b.id)}<br>${r.flags}`,d.style.position="absolute",d.style.border="1px solid black",d.style.backgroundColor="white",d.style.left=`${r.pos.x}px`,d.style.top=`${r.pos.y}px`,d.style.whiteSpace="nowrap",this.graphContainer.appendChild(d)}this.updateHighlightedInstructions(),this.updateHotness()}renderMIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-mir","ig-can-flash",...t.attributes.map(o=>`ig-ins-att-${o}`)),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=`v${t.id}`,s.appendChild(i);let c=document.createElement("td");c.innerHTML=e.replace(/(v)(\d+)/g,(o,n,r)=>`<span class="ig-use ig-highlightable" data-ig-use="${r}">${n}${r}</span>`),s.appendChild(c);let l=document.createElement("td");return l.classList.add("ig-ins-type"),l.innerText=t.type==="None"?"":t.type,s.appendChild(l),i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),c.querySelectorAll(".ig-use").forEach(o=>{o.addEventListener("pointerdown",n=>{n.preventDefault(),n.stopPropagation()}),o.addEventListener("click",n=>{let r=parseInt(L(o.getAttribute("data-ig-use")),10);this.jumpToInstruction(r,{zoom:1})})}),s}renderLIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-lir","ig-hotness"),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=String(t.id),s.appendChild(i);let c=document.createElement("td");if(c.innerText=e,s.appendChild(c),this.sampleCounts){let l=this.sampleCounts?.totalLineHits.get(t.id)??0,o=this.sampleCounts?.selfLineHits.get(t.id)??0,n=document.createElement("td");n.classList.add("ig-ins-samples"),n.classList.toggle("ig-text-dim",l===0),n.innerText=`${l}`,n.title="Color by total count",s.appendChild(n);let r=document.createElement("td");r.classList.add("ig-ins-samples"),r.classList.toggle("ig-text-dim",o===0),r.innerText=`${o}`,r.title="Color by self count",s.appendChild(r);for(let[d,b]of[n,r].entries())b.addEventListener("pointerdown",g=>{g.preventDefault(),g.stopPropagation()}),b.addEventListener("click",()=>{k(d===O||d===W),this.heatmapMode=d,this.updateHotness()})}return i.addEventListener("pointerdown",l=>{l.preventDefault(),l.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),s}renderSelection(){this.graphContainer.querySelectorAll(".ig-block").forEach(t=>{let e=parseInt(L(t.getAttribute("data-ig-block-ptr")),10);t.classList.toggle("ig-selected",this.selectedBlockPtrs.has(e)),t.classList.toggle("ig-last-selected",this.lastSelectedBlockPtr===e)})}removeNonexistentHighlights(){this.highlightedInstructions=this.highlightedInstructions.filter(t=>this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`))}updateHighlightedInstructions(){for(let t of this.highlightedInstructions)k(this.highlightedInstructions.filter(e=>e.ptr===t.ptr).length===1,`instruction ${t.ptr} was highlighted more than once`);this.graphContainer.querySelectorAll(".ig-ins, .ig-use").forEach(t=>{Vt(t)});for(let t of this.highlightedInstructions){let e=this.instructionPalette[t.paletteColor%this.instructionPalette.length],s=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`);if(s){ct(s,e);let i=this.insIDsByPtr.get(t.ptr);this.graphContainer.querySelectorAll(`.ig-use[data-ig-use="${i}"]`).forEach(c=>{ct(c,e)})}}}updateHotness(){this.graphContainer.querySelectorAll(".ig-ins-lir").forEach(t=>{k(t.classList.contains("ig-hotness"));let e=parseInt(L(t.getAttribute("data-ig-ins-id")),10),s=0;this.sampleCounts&&(s=((this.heatmapMode===O?this.sampleCounts.totalLineHits:this.sampleCounts.selfLineHits).get(e)??0)/this.maxSampleCounts[this.heatmapMode]),t.style.setProperty("--ig-hotness",`${s}`)})}addEventListeners(){this.viewport.addEventListener("wheel",e=>{e.preventDefault();let s=this.zoom;if(e.ctrlKey){s=Math.max(Ht,Math.min(Ct,this.zoom*Math.pow(Mt,-e.deltaY*Et)));let c=s/this.zoom-1;this.zoom=s;let{x:l,y:o}=this.viewport.getBoundingClientRect(),n=e.clientX-l-this.translation.x,r=e.clientY-o-this.translation.y;this.translation.x-=n*c,this.translation.y-=r*c}else this.translation.x-=e.deltaX,this.translation.y-=e.deltaY;let i=this.clampTranslation(this.translation,s);this.translation.x=i.x,this.translation.y=i.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerdown",e=>{e.pointerType==="mouse"&&!(e.button===0||e.button===1)||(e.preventDefault(),this.viewport.setPointerCapture(e.pointerId),this.startMousePos={x:e.clientX,y:e.clientY},this.lastMousePos={x:e.clientX,y:e.clientY},this.animating=!1)}),this.viewport.addEventListener("pointermove",e=>{if(!this.viewport.hasPointerCapture(e.pointerId))return;let s=e.clientX-this.lastMousePos.x,i=e.clientY-this.lastMousePos.y;this.translation.x+=s,this.translation.y+=i,this.lastMousePos={x:e.clientX,y:e.clientY};let c=this.clampTranslation(this.translation,this.zoom);this.translation.x=c.x,this.translation.y=c.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerup",e=>{this.viewport.releasePointerCapture(e.pointerId);let s=2,i=this.startMousePos.x-e.clientX,c=this.startMousePos.y-e.clientY;Math.abs(i)<=s&&Math.abs(c)<=s&&this.setSelection([]),this.animating=!1}),new ResizeObserver(e=>{k(e.length===1);let s=e[0].contentRect;this.viewportSize.x=s.width,this.viewportSize.y=s.height}).observe(this.viewport)}setSelection(t,e=0){this.setSelectionRaw(t,e),e?this.nav={visited:[e],currentIndex:0,siblings:[e]}:this.nav={visited:[],currentIndex:-1,siblings:[]}}setSelectionRaw(t,e){this.selectedBlockPtrs.clear();for(let s of[...t,e])this.blocksByPtr.has(s)&&this.selectedBlockPtrs.add(s);this.lastSelectedBlockPtr=this.blocksByPtr.has(e)?e:0,this.renderSelection()}navigate(t){let e=this.lastSelectedBlockPtr;if(t==="down"||t==="up")if(e){let s=L(this.blocksByPtr.get(e)),i=(t==="down"?s.succs:s.preds).map(l=>l.ptr);s.ptr!==this.nav.visited[this.nav.currentIndex]&&(this.nav.visited=[s.ptr],this.nav.currentIndex=0);let c=this.nav.currentIndex+(t==="down"?1:-1);if(0<=c&&c<this.nav.visited.length)this.nav.currentIndex=c,this.nav.siblings=i;else{let l=i[0];l!==void 0&&(t==="down"?(this.nav.visited.push(l),this.nav.currentIndex+=1,k(this.nav.currentIndex===this.nav.visited.length-1)):(this.nav.visited.unshift(l),k(this.nav.currentIndex===0)),this.nav.siblings=i)}this.setSelectionRaw([],this.nav.visited[this.nav.currentIndex])}else{let s=[...this.blocks].sort((n,r)=>n.id-r.id),i=s.filter(n=>n.preds.length===0),c=s.filter(n=>n.succs.length===0),l=t==="down"?i:c,o=l[0];k(o),this.setSelectionRaw([],o.ptr),this.nav={visited:[o.ptr],currentIndex:0,siblings:l.map(n=>n.ptr)}}else if(e!==void 0){let s=this.nav.siblings.indexOf(e);k(s>=0,"currently selected node should be in siblings array");let i=s+(t==="right"?1:-1);0<=i&&i<this.nav.siblings.length&&this.setSelectionRaw([],this.nav.siblings[i])}k(this.nav.visited.length===0||this.nav.siblings.includes(this.nav.visited[this.nav.currentIndex]),"expected currently visited node to be in the siblings array"),k(this.lastSelectedBlockPtr===0||this.nav.siblings.includes(this.lastSelectedBlockPtr),"expected currently selected block to be in siblings array")}toggleInstructionHighlight(t,e){this.removeNonexistentHighlights();let s=this.highlightedInstructions.findIndex(c=>c.ptr===t),i=s>=0;if(e!==void 0&&(i=!e),i)s>=0&&this.highlightedInstructions.splice(s,1);else if(s<0){let c=0;for(;;){if(this.highlightedInstructions.find(l=>l.paletteColor===c)){c+=1;continue}break}this.highlightedInstructions.push({ptr:t,paletteColor:c})}this.updateHighlightedInstructions()}clampTranslation(t,e){let s=z-this.size.x*e,i=this.viewportSize.x-z,c=z-this.size.y*e,l=this.viewportSize.y-z,o=q(t.x,s,i),n=q(t.y,c,l);return{x:o,y:n}}updatePanAndZoom(){let t=this.clampTranslation(this.translation,this.zoom);this.graphContainer.style.transform=`translate(${t.x}px, ${t.y}px) scale(${this.zoom})`}graph2viewport(t,e=this.translation,s=this.zoom){return{x:t.x*s+e.x,y:t.y*s+e.y}}viewport2graph(t,e=this.translation,s=this.zoom){return{x:(t.x-e.x)/s,y:(t.y-e.y)/s}}async goToGraphCoordinates(t,{zoom:e=this.zoom,animate:s=!0}){let i={x:-t.x*e,y:-t.y*e};if(!s){this.animating=!1,this.translation.x=i.x,this.translation.y=i.y,this.zoom=e,this.updatePanAndZoom(),await new Promise(l=>setTimeout(l,0));return}if(this.targetTranslation=i,this.targetZoom=e,this.animating)return;this.animating=!0;let c=performance.now();for(;this.animating;){let l=await new Promise(h=>requestAnimationFrame(h)),o=(l-c)/1e3;c=l;let n=1,r=.01,d=1e-6,b=this.targetTranslation.x-this.translation.x,g=this.targetTranslation.y-this.translation.y,u=this.targetZoom-this.zoom;if(this.translation.x=R(this.translation.x,this.targetTranslation.x,d,o),this.translation.y=R(this.translation.y,this.targetTranslation.y,d,o),this.zoom=R(this.zoom,this.targetZoom,d,o),this.updatePanAndZoom(),Math.abs(b)<=n&&Math.abs(g)<=n&&Math.abs(u)<=r){this.translation.x=this.targetTranslation.x,this.translation.y=this.targetTranslation.y,this.zoom=this.targetZoom,this.animating=!1,this.updatePanAndZoom();break}}await new Promise(l=>setTimeout(l,0))}jumpToBlock(t,{zoom:e=this.zoom,animate:s=!0,viewportPos:i}={}){let c=this.blocksByPtr.get(t);if(!c)return Promise.resolve();let l;return i?l={x:c.layoutNode.pos.x-i.x/e,y:c.layoutNode.pos.y-i.y/e}:l=this.graphPosToCenterRect(c.layoutNode.pos,c.layoutNode.size,e),this.goToGraphCoordinates(l,{zoom:e,animate:s})}async jumpToInstruction(t,{zoom:e=this.zoom,animate:s=!0}){let i=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-id="${t}"]`);if(!i)return;let c=i.getBoundingClientRect(),l=this.graphContainer.getBoundingClientRect(),o=(c.x-l.x)/this.zoom,n=(c.y-l.y)/this.zoom,r=c.width/this.zoom,d=c.height/this.zoom,b=this.graphPosToCenterRect({x:o,y:n},{x:r,y:d},e);i.classList.add("ig-flash"),await this.goToGraphCoordinates(b,{zoom:e,animate:s}),i.classList.remove("ig-flash")}graphPosToCenterRect(t,e,s){let i=this.viewportSize.x/s,c=this.viewportSize.y/s,l=Math.max(20/s,(i-e.x)/2),o=Math.max(20/s,(c-e.y)/2),n=t.x-l,r=t.y-o;return{x:n,y:r}}exportState(){let t={translation:this.translation,zoom:this.zoom,heatmapMode:this.heatmapMode,highlightedInstructions:this.highlightedInstructions,selectedBlockPtrs:this.selectedBlockPtrs,lastSelectedBlockPtr:this.lastSelectedBlockPtr,viewportPosOfSelectedBlock:void 0};return this.lastSelectedBlockPtr&&(t.viewportPosOfSelectedBlock=this.graph2viewport(L(this.blocksByPtr.get(this.lastSelectedBlockPtr)).layoutNode.pos)),t}restoreState(t,e){this.translation.x=t.translation.x,this.translation.y=t.translation.y,this.zoom=t.zoom,this.heatmapMode=t.heatmapMode,this.highlightedInstructions=t.highlightedInstructions,this.setSelection(Array.from(t.selectedBlockPtrs),t.lastSelectedBlockPtr),this.updatePanAndZoom(),this.updateHotness(),this.updateHighlightedInstructions(),e.preserveSelectedBlockPosition&&this.jumpToBlock(this.lastSelectedBlockPtr,{zoom:this.zoom,animate:!1,viewportPos:t.viewportPosOfSelectedBlock})}};function Ot(a){for(let t of a.dstNodes){let e=t.srcNodes.indexOf(a);k(e!==-1),t.srcNodes.splice(e,1)}}function*Q(a){for(let t of a)for(let e of t)e.block===null&&(yield e)}function*At(a){for(let t of a)for(let e of t)e.block===null&&e.dstBlock.mir.attributes.includes("backedge")&&(yield e)}function Rt(a,t,e,s,i,c,l=1){let o=T;k(t+o<=i&&i<s-o,`downward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i-o} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${e} ${i+o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,180);r.appendChild(b)}return r}function Ft(a,t,e,s,i,c,l=1){let o=T;k(s+o<=i&&i<=t-o,`upward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i+o} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${e} ${i-o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,0);r.appendChild(b)}return r}function zt(a,t,e,s,i=1){let c=T;k(t-c>=s&&a-c>=e,`to backedge: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, r = ${c} `,!0),i%2===1&&(a+=.5,s+=.5);let l="";l+=`M ${a} ${t} `,l+=`A ${c} ${c} 0 0 0 ${a-c} ${s} `,l+=`L ${e} ${s} `;let o=document.createElementNS("http://www.w3.org/2000/svg","g"),n=document.createElementNS("http://www.w3.org/2000/svg","path");n.setAttribute("d",l),n.setAttribute("fill","none"),n.setAttribute("stroke","black"),n.setAttribute("stroke-width",`${i} `),o.appendChild(n);let r=G(e,s,270);return o.appendChild(r),o}function _t(a,t,e,s,i,c=1){let l=T;k(t+l<=i&&a<=e&&s<=t,`block to backedge dummy: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${l} `,!0),c%2===1&&(a+=.5,e+=.5,i+=.5);let o="";o+=`M ${a} ${t} `,o+=`L ${a} ${i-l} `,o+=`A ${l} ${l} 0 0 0 ${a+l} ${i} `,o+=`L ${e-l} ${i} `,o+=`A ${l} ${l} 0 0 0 ${e} ${i-l} `,o+=`L ${e} ${s} `;let n=document.createElementNS("http://www.w3.org/2000/svg","g"),r=document.createElementNS("http://www.w3.org/2000/svg","path");return r.setAttribute("d",o),r.setAttribute("fill","none"),r.setAttribute("stroke","black"),r.setAttribute("stroke-width",`${c} `),n.appendChild(r),n}function jt(a,t,e,s,i=1){k(e<a&&s===t,`x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s} `,!0),i%2===1&&(t+=.5,s+=.5);let c="";c+=`M ${a} ${t} `,c+=`L ${e} ${s} `;let l=document.createElementNS("http://www.w3.org/2000/svg","g"),o=document.createElementNS("http://www.w3.org/2000/svg","path");o.setAttribute("d",c),o.setAttribute("fill","none"),o.setAttribute("stroke","black"),o.setAttribute("stroke-width",`${i} `),l.appendChild(o);let n=G(e,s,270);return l.appendChild(n),l}function G(a,t,e,s=5){let i=document.createElementNS("http://www.w3.org/2000/svg","path");return i.setAttribute("d",`M 0 0 L ${-s} ${s*1.5} L ${s} ${s*1.5} Z`),i.setAttribute("transform",`translate(${a}, ${t}) rotate(${e})`),i}function ct(a,t){a.classList.add("ig-highlight"),a.style.setProperty("--ig-highlight-color",t)}function Vt(a){a.classList.remove("ig-highlight"),a.style.setProperty("--ig-highlight-color","transparent")}var A=class{constructor(t,{func:e,pass:s=0,sampleCounts:i}){this.graph=null,this.func=e,this.passNumber=s,this.sampleCounts=i,this.keyPasses=[null,null,null,null];{let c=null;for(let[l,o]of e.passes.entries())o.mir.blocks.length>0&&(this.keyPasses[0]===null&&(this.keyPasses[0]=l),o.lir.blocks.length===0&&(this.keyPasses[1]=l)),o.lir.blocks.length>0&&(c?.lir.blocks.length===0&&(this.keyPasses[2]=l),this.keyPasses[3]=l),c=o}this.redundantPasses=[];{let c=null;for(let[l,o]of e.passes.entries()){if(c===null){c=o;continue}E(c.mir,o.mir)&&E(c.lir,o.lir)&&this.redundantPasses.push(l),c=o}}this.viewport=x("div",["ig-flex-grow-1","ig-overflow-hidden"],c=>{c.style.position="relative"}),this.sidebarLinks=e.passes.map((c,l)=>x("a",["ig-link-normal","ig-pv1","ig-ph2","ig-flex","ig-g2"],o=>{o.href="#",o.addEventListener("click",n=>{n.preventDefault(),this.switchPass(l)})},[x("div",["ig-w1","ig-tr","ig-f6","ig-text-dim"],o=>{o.style.paddingTop="0.08rem"},[`${l}`]),x("div",[this.redundantPasses.includes(l)&&"ig-text-dim"],()=>{},[c.name])])),this.container=x("div",["ig-absolute","ig-absolute-fill","ig-flex"],()=>{},[x("div",["ig-w5","ig-br","ig-flex-shrink-0","ig-overflow-y-auto","ig-bg-white"],()=>{},[...this.sidebarLinks]),this.viewport]),t.appendChild(this.container),this.keydownHandler=this.keydownHandler.bind(this),this.tweakHandler=this.tweakHandler.bind(this),window.addEventListener("keydown",this.keydownHandler),window.addEventListener("tweak",this.tweakHandler),this.update()}destroy(){this.container.remove(),window.removeEventListener("keydown",this.keydownHandler),window.removeEventListener("tweak",this.tweakHandler)}update(){for(let[s,i]of this.sidebarLinks.entries())i.classList.toggle("ig-bg-primary",this.passNumber===s);let t=this.graph?.exportState();this.viewport.innerHTML="",this.graph=null;let e=this.func.passes[this.passNumber];if(e)try{this.graph=new V(this.viewport,e,{sampleCounts:this.sampleCounts}),t&&this.graph.restoreState(t,{preserveSelectedBlockPosition:!0})}catch(s){this.viewport.innerHTML="An error occurred while laying out the graph. See console.",console.error(s)}}switchPass(t){this.passNumber=t,this.update()}keydownHandler(t){switch(t.key){case"w":case"s":this.graph?.navigate(t.key==="s"?"down":"up"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"a":case"d":this.graph?.navigate(t.key==="d"?"right":"left"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"f":for(let e=this.passNumber+1;e<this.func.passes.length;e++)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"r":for(let e=this.passNumber-1;e>=0;e--)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"1":case"2":case"3":case"4":{let e=["1","2","3","4"].indexOf(t.key),s=this.keyPasses[e];typeof s=="number"&&this.switchPass(s)}break;case"c":{let e=this.graph?.blocksByPtr.get(this.graph?.lastSelectedBlockPtr??-1);e&&this.graph?.jumpToBlock(e.ptr,{zoom:1})}break}}tweakHandler(){this.update()}};var M=new URL(window.location.toString()).searchParams,Gt=M.has("func")?parseInt(M.get("func"),10):void 0,ut=M.has("pass")?parseInt(M.get("pass"),10):void 0,Y=class{constructor(t){this.exportButton=null,this.ionjson=null,this.funcIndex=Gt??0,this.funcSelected=t.funcSelected,this.funcSelector=x("div",[],()=>{},["Function",x("input",["ig-w3"],e=>{e.type="number",e.min="1",e.addEventListener("input",()=>{this.switchFunc(parseInt(e.value,10)-1)})},[])," / ",x("span",["num-functions"])]),this.funcSelectorNone=x("div",[],()=>{},["No functions to display."]),this.funcName=x("div"),this.root=x("div",["ig-bb","ig-flex","ig-bg-white"],()=>{},[x("div",["ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center","ig-br","ig-hide-if-empty"],()=>{},[t.browse&&x("div",[],()=>{},[x("input",[],e=>{e.type="file",e.addEventListener("change",s=>{let i=s.target;i.files?.length&&this.fileSelected(i.files[0])})})]),this.funcSelector,this.funcSelectorNone]),x("div",["ig-flex-grow-1","ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center"],()=>{},[this.funcName,x("div",["ig-flex-grow-1"]),t.export&&x("div",[],()=>{},[x("button",[],e=>{this.exportButton=e,e.addEventListener("click",()=>{this.exportStandalone()})},["Export"])])])]),this.update()}async fileSelected(t){let e=JSON.parse(await t.text());this.ionjson=X(e),this.switchFunc(0),this.update()}switchIonJSON(t){this.ionjson=t,this.switchFunc(this.funcIndex)}switchFunc(t){t=Math.max(0,Math.min(this.numFunctions()-1,t)),this.funcIndex=isNaN(t)?0:t,this.funcSelected(this.ionjson?.functions[this.funcIndex]??null),this.update()}numFunctions(){return this.ionjson?.functions.length??0}update(){let t=0<=this.funcIndex&&this.funcIndex<this.numFunctions();this.funcSelector.hidden=this.numFunctions()<=1,this.funcSelectorNone.hidden=!(this.ionjson&&this.numFunctions()===0);let e=this.funcSelector.querySelector("input");e.max=`${this.numFunctions()}`,e.value=`${this.funcIndex+1}`,this.funcSelector.querySelector(".num-functions").innerHTML=`${this.numFunctions()}`,this.funcName.hidden=!t,this.funcName.innerText=`${this.ionjson?.functions[this.funcIndex].name??""}`,this.exportButton&&(this.exportButton.disabled=!this.ionjson||!t)}async exportStandalone(){let t=L(this.ionjson),e=t.functions[this.funcIndex].name,s={version:1,functions:[t.functions[this.funcIndex]]},c=(await(await fetch("./standalone.html")).text()).replace(/\{\{\s*IONJSON\s*\}\}/,JSON.stringify(s)),l=URL.createObjectURL(new Blob([c],{type:"text/html;charset=utf-8"})),o=document.createElement("a");o.href=l,o.download=`iongraph-${e}.html`,document.body.appendChild(o),o.click(),o.remove(),URL.revokeObjectURL(l)}},tt=class{constructor(){this.menuBar=new Y({browse:!0,export:!0,funcSelected:t=>this.switchFunc(t)}),this.func=null,this.sampleCountsFromFile=void 0,this.graph=null,this.loadStuffFromQueryParams(),this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer]),this.update()}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut,sampleCounts:this.sampleCountsFromFile}))}loadStuffFromQueryParams(){(async()=>{let t=M.get("file");if(t){let s=await(await fetch(t)).json(),i=X(s);this.menuBar.switchIonJSON(i)}})(),(async()=>{let t=M.get("sampleCounts");if(t){let s=await(await fetch(t)).json();this.sampleCountsFromFile={selfLineHits:new Map(s.selfLineHits),totalLineHits:new Map(s.totalLineHits)},this.update()}})()}switchFunc(t){this.func=t,this.update()}},et=class{constructor(){this.menuBar=new Y({funcSelected:t=>this.switchFunc(t)}),this.func=null,this.graph=null,this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer])}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut}))}setIonJSON(t){this.menuBar.switchIonJSON(t)}switchFunc(t){this.func=t,this.update()}};return yt(Yt);})(); +</script> + <script>window.__exportedIonJSON = {{ IONJSON }}</script> + <script> + const ui = new iongraph.StandaloneUI(); + document.body.appendChild(ui.root); + ui.setIonJSON(window.__exportedIonJSON); + </script> +</body> diff --git a/tool/zjit_iongraph.rb b/tool/zjit_iongraph.rb new file mode 100755 index 0000000000..0cb7701614 --- /dev/null +++ b/tool/zjit_iongraph.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +require 'json' +require 'logger' + +LOGGER = Logger.new($stderr) + +def run_ruby *cmd + # Find the first --zjit* option and add --zjit-dump-hir-iongraph after it + zjit_index = cmd.find_index { |arg| arg.start_with?("--zjit") } + raise "No --zjit option found in command" unless zjit_index + cmd.insert(zjit_index + 1, "--zjit-dump-hir-iongraph") + pid = Process.spawn(*cmd) + _, status = Process.wait2(pid) + if status.exitstatus != 0 + LOGGER.warn("Command failed with exit status #{status.exitstatus}") + end + pid +end + +usage = "Usage: zjit_iongraph.rb <path_to_ruby> <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +pid = run_ruby(RUBY, *OPTIONS) +functions = Dir["/tmp/zjit-iongraph-#{pid}/fun*.json"].map do |path| + JSON.parse(File.read(path)) +end + +if functions.empty? + LOGGER.warn("No iongraph functions found for PID #{pid}") +end + +json = JSON.dump({version: 1, functions: functions}) +# Get zjit_iongraph.html from the sibling file next to this script +html = File.read(File.join(File.dirname(__FILE__), "zjit_iongraph.html")) +html.sub!("{{ IONJSON }}", json) +output_path = "zjit_iongraph_#{pid}.html" +File.write(output_path, html) +puts "Wrote iongraph to #{output_path}" |