diff options
Diffstat (limited to 'tool')
330 files changed, 22022 insertions, 14643 deletions
diff --git a/tool/annocheck/Dockerfile b/tool/annocheck/Dockerfile index 138adc48de..d1fb1839c9 100644 --- a/tool/annocheck/Dockerfile +++ b/tool/annocheck/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/fedora:latest +FROM ghcr.io/ruby/fedora:latest RUN dnf -y install annobin-annocheck WORKDIR /work diff --git a/tool/annocheck/Dockerfile-copy b/tool/annocheck/Dockerfile-copy index e658d12ddc..d437f27387 100644 --- a/tool/annocheck/Dockerfile-copy +++ b/tool/annocheck/Dockerfile-copy @@ -1,7 +1,6 @@ -FROM docker.io/fedora:latest -ARG FILES +FROM ghcr.io/ruby/fedora:latest +ARG IN_DIR RUN dnf -y install annobin-annocheck -RUN mkdir /work -COPY ${FILES} /work +COPY ${IN_DIR} /work WORKDIR /work diff --git a/tool/auto-style.rb b/tool/auto-style.rb new file mode 100755 index 0000000000..3b93c8c317 --- /dev/null +++ b/tool/auto-style.rb @@ -0,0 +1,284 @@ +#!/usr/bin/env ruby +# Usage: +# auto-style.rb oldrev newrev [pushref] + +require 'shellwords' +require 'tmpdir' +ENV['LC_ALL'] = 'C' + +class Git + attr_reader :depth + + def initialize(oldrev, newrev, branch = nil) + @oldrev = oldrev + @newrev = !newrev || newrev.empty? ? 'HEAD' : newrev + @branch = branch + + return unless oldrev + + # GitHub may not fetch github.event.pull_request.base.sha at checkout + git('log', '--format=%H', '-1', @oldrev, out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=1', 'origin', @oldrev) + git('log', '--format=%H', '-1', "#@newrev~99", out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=100', 'origin', @newrev) + + with_clean_env do + @revs = {} + IO.popen(['git', 'log', '--format=%H %s', "#{@oldrev}..#{@newrev}"]) do |f| + f.each do |line| + line.chomp! + rev, subj = line.split(' ', 2) + @revs[rev] = subj + end + end + @depth = @revs.size + end + end + + # ["foo/bar.c", "baz.h", ...] + def updated_paths + with_clean_env do + IO.popen(['git', 'diff', '--name-only', @oldrev, @newrev], &:readlines).each(&:chomp!) + end + end + + # [0, 1, 4, ...] + def updated_lines(file) # NOTE: This doesn't work well on pull requests, so not used anymore + lines = [] + revs = @revs.map {|rev, subj| rev unless subj.start_with?("Revert ")}.compact + revs_pattern = /\A(?:#{revs.join('|')}) / + with_clean_env { IO.popen(['git', 'blame', '-l', '--', file], &:readlines) }.each_with_index do |line, index| + if revs_pattern =~ line + lines << index + end + end + lines + end + + def commit(log, *files) + git('add', *files) + git('commit', '-m', log) + end + + def push + git('push', 'origin', @branch) if @branch + end + + def diff + git('--no-pager', 'diff') + end + + private + + def git(*args, **opts) + cmd = ['git', *args] + puts "+ #{cmd.shelljoin}" + ret = with_clean_env { system(*cmd, **opts) } + unless ret or opts[:err] + abort "Failed to run: #{cmd}" + end + ret + end + + def with_clean_env + git_dir = ENV.delete('GIT_DIR') # this overcomes '-C' or pwd + yield + ensure + ENV['GIT_DIR'] = git_dir if git_dir + end +end + +DEFAULT_GEM_LIBS = %w[ + bundler + cmath + csv + e2mmap + fileutils + forwardable + ipaddr + irb + logger + matrix + mutex_m + ostruct + prime + rdoc + rexml + rss + scanf + shell + sync + thwait + tracer + webrick +] + +DEFAULT_GEM_EXTS = %w[ + bigdecimal + date + dbm + digest + etc + fcntl + fiddle + gdbm + io/console + io/nonblock + json + openssl + psych + racc + sdbm + stringio + strscan + zlib +] + +IGNORED_FILES = [ + # default gems whose master is GitHub + %r{\Abin/(?!erb)\w+\z}, + *(DEFAULT_GEM_LIBS + DEFAULT_GEM_EXTS).flat_map { |lib| + [ + %r{\Alib/#{lib}/}, + %r{\Alib/#{lib}\.gemspec\z}, + %r{\Alib/#{lib}\.rb\z}, + %r{\Atest/#{lib}/}, + ] + }, + *DEFAULT_GEM_EXTS.flat_map { |ext| + [ + %r{\Aext/#{ext}/}, + %r{\Atest/#{ext}/}, + ] + }, + + # vendoring (ccan) + %r{\Accan/}, + + # vendoring (io/) + %r{\Aext/io/}, + + # vendoring (nkf) + %r{\Aext/nkf/nkf-utf8/}, + + # vendoring (onigmo) + %r{\Aenc/}, + %r{\Ainclude/ruby/onigmo\.h\z}, + %r{\Areg.+\.(c|h)\z}, + + # explicit or implicit `c-file-style: "linux"` + %r{\Aaddr2line\.c\z}, + %r{\Amissing/}, + %r{\Astrftime\.c\z}, + %r{\Avsnprintf\.c\z}, + + # to respect the original statements of licenses + %r{\ALEGAL\z}, + + # trailing spaces could be intentional in TRICK code + %r{\Asample/trick[^/]*/}, +] + +DIFFERENT_STYLE_FILES = %w[ + addr2line.c io_buffer.c prism*.c scheduler.c +] + +def adjust_styles(files) + trailing = eofnewline = expandtab = indent = false + + edited_files = files.select do |f| + src = File.binread(f) rescue next + eofnewline = eofnewline0 = true if src.sub!(/(?<!\A|\n)\z/, "\n") + + trailing0 = false + expandtab0 = false + indent0 = false + + src.gsub!(/^.*$/).with_index do |line, lineno| + trailing = trailing0 = true if line.sub!(/[ \t]+$/, '') + line + end + + if f.end_with?('.c') || f.end_with?('.h') || f == 'insns.def' + # If and only if unedited lines did not have tab indentation, prevent introducing tab indentation to the file. + expandtab_allowed = src.each_line.with_index.all? do |line, lineno| + !line.start_with?("\t") + end + + if expandtab_allowed + src.gsub!(/^.*$/).with_index do |line, lineno| + if line.start_with?("\t") # last-committed line with hard tabs + expandtab = expandtab0 = true + line.sub(/\A\t+/) { |tabs| ' ' * (8 * tabs.length) } + else + line + end + end + end + end + + if File.fnmatch?("*.[chy]", f, File::FNM_PATHNAME) && + !DIFFERENT_STYLE_FILES.any? {|pat| File.fnmatch?(pat, f, File::FNM_PATHNAME)} + indent0 = true if src.gsub!(/^\w+\([^\n]*?\)\K[ \t]*(?=\{( *\\)?$)/, '\1' "\n") + indent0 = true if src.gsub!(/^([ \t]*)\}\K[ \t]*(?=else\b.*?( *\\)?$)/, '\2' "\n" '\1') + indent0 = true if src.gsub!(/^[ \t]*\}\n\K\n+(?=[ \t]*else\b)/, '') + indent ||= indent0 + end + + if trailing0 or eofnewline0 or expandtab0 or indent0 + File.binwrite(f, src) + true + end + end + if edited_files.empty? + return + else + msg = [('remove trailing spaces' if trailing), + ('append newline at EOF' if eofnewline), + ('expand tabs' if expandtab), + ('adjust indents' if indent), + ].compact + message = "* #{msg.join(', ')}. [ci skip]" + if expandtab + message += "\nPlease consider using misc/expand_tabs.rb as a pre-commit hook." + end + return message, edited_files + end +end + +oldrev, newrev, pushref = ARGV +if (dry_run = oldrev == '-n') or oldrev == '--' + _, *updated_files = ARGV + git = Git.new(nil, nil) +else + unless dry_run = pushref.nil? + branch = IO.popen(['git', 'rev-parse', '--symbolic', '--abbrev-ref', pushref], &:read).strip + end + git = Git.new(oldrev, newrev, branch) + + updated_files = git.updated_paths +end + +files = updated_files.select {|l| + /^\d/ !~ l and /\.bat\z/ !~ l and + (/\A(?:config|[Mm]akefile|GNUmakefile|README)/ =~ File.basename(l) or + /\A\z|\.(?:[chsy]|\d+|e?rb|tmpl|bas[eh]|z?sh|in|ma?k|def|src|trans|rdoc|ja|en|el|sed|awk|p[ly]|scm|mspec|html|rs)\z/ =~ File.extname(l)) +} +files.select! {|n| File.file?(n) } +files.reject! do |f| + IGNORED_FILES.any? { |re| f.match(re) } +end + +if files.empty? + puts "No files are an auto-style target:\n#{updated_files.join("\n")}" +elsif !(message, edited_files = adjust_styles(files)) + puts "All edited lines are formatted well:\n#{files.join("\n")}" +else + if dry_run + git.diff + abort message + else + git.commit(message, *edited_files) + git.push + end +end diff --git a/tool/auto_review_pr.rb b/tool/auto_review_pr.rb new file mode 100755 index 0000000000..07c98c7e0a --- /dev/null +++ b/tool/auto_review_pr.rb @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'uri' +require_relative './sync_default_gems' + +class GitHubAPIClient + def initialize(token) + @token = token + end + + def get(path) + response = Net::HTTP.get_response(URI("https://api.github.com#{path}"), { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end + + def post(path, body = {}) + body = JSON.dump(body) + response = Net::HTTP.post(URI("https://api.github.com#{path}"), body, { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + 'Content-Type' => 'application/json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end +end + +class AutoReviewPR + REPO = 'ruby/ruby' + + COMMENT_USER = 'github-actions[bot]' + COMMENT_PREFIX = 'The following files are maintained in the following upstream repositories:' + COMMENT_SUFFIX = 'Please file a pull request to the above instead. Thank you!' + + def initialize(client) + @client = client + end + + def review(pr_number) + # Fetch the list of files changed by the PR + changed_files = @client.get("/repos/#{REPO}/pulls/#{pr_number}/files").map { it.fetch(:filename) } + + # Build a Hash: { upstream_repo => files, ... } + upstream_repos = SyncDefaultGems::Repository.group(changed_files) + upstream_repos.delete(nil) # exclude no-upstream files + upstream_repos.delete('prism') if changed_files.include?('prism_compile.c') # allow prism changes in this case + if upstream_repos.empty? + puts "Skipped: The PR ##{pr_number} doesn't have upstream repositories." + return + end + + # Check if the PR is already reviewed + existing_comments = @client.get("/repos/#{REPO}/issues/#{pr_number}/comments") + existing_comments.map! { [it.fetch(:user).fetch(:login), it.fetch(:body)] } + if existing_comments.any? { |user, comment| user == COMMENT_USER && comment.start_with?(COMMENT_PREFIX) } + puts "Skipped: The PR ##{pr_number} already has an automated review comment." + return + end + + # Post a comment + comment = format_comment(upstream_repos) + result = @client.post("/repos/#{REPO}/issues/#{pr_number}/comments", { body: comment }) + puts "Success: #{JSON.pretty_generate(result)}" + end + + private + + # upstream_repos: { upstream_repo => files, ... } + def format_comment(upstream_repos) + comment = +'' + comment << "#{COMMENT_PREFIX}\n\n" + + upstream_repos.each do |upstream_repo, files| + comment << "* https://github.com/ruby/#{upstream_repo}\n" + files.each do |file| + comment << " * #{file}\n" + end + end + + comment << "\n#{COMMENT_SUFFIX}" + comment + end +end + +pr_number = ARGV[0] || abort("Usage: #{$0} <pr_number>") +client = GitHubAPIClient.new(ENV.fetch('GITHUB_TOKEN')) + +AutoReviewPR.new(client).review(pr_number) diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb index ef63263414..91ac5628f1 100644 --- a/tool/bundler/dev_gems.rb +++ b/tool/bundler/dev_gems.rb @@ -3,17 +3,18 @@ source "https://rubygems.org" gem "test-unit", "~> 3.0" -gem "rake", "~> 13.0" +gem "test-unit-ruby-core" +gem "rake", "~> 13.1" gem "rb_sys" -gem "webrick", "~> 1.6" -gem "parallel_tests", "~> 2.29" +gem "turbo_tests", "~> 2.2.3" +gem "parallel_tests", "~> 4.10.1" gem "parallel", "~> 1.19" gem "rspec-core", "~> 3.12" gem "rspec-expectations", "~> 3.12" gem "rspec-mocks", "~> 3.12" -gem "uri", "~> 0.12.0" +gem "rubygems-generate_index", "~> 1.1" group :doc do - gem "ronn", "~> 0.7.3", :platform => :ruby + gem "ronn-ng", "~> 0.10.1", platform: :ruby end diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock index 694109aada..832127fb4c 100644 --- a/tool/bundler/dev_gems.rb.lock +++ b/tool/bundler/dev_gems.rb.lock @@ -1,57 +1,132 @@ GEM remote: https://rubygems.org/ specs: - diff-lcs (1.5.0) - hpricot (0.8.6) - hpricot (0.8.6-java) + compact_index (0.15.0) + diff-lcs (1.6.2) + kramdown (2.5.1) + rexml (>= 3.3.9) + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + mini_portile2 (2.8.9) mustache (1.1.1) - parallel (1.22.1) - parallel_tests (2.32.0) + nokogiri (1.19.0) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.19.0-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-java) + racc (~> 1.4) + nokogiri (1.19.0-x64-mingw-ucrt) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-gnu) + racc (~> 1.4) + parallel (1.27.0) + parallel_tests (4.10.1) parallel - power_assert (2.0.2) - rake (13.0.6) - rb_sys (0.9.52) - rdiscount (2.2.7) - ronn (0.7.3) - hpricot (>= 0.8.2) - mustache (>= 0.7.0) - rdiscount (>= 1.5.8) - rspec-core (3.12.0) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.0) + power_assert (3.0.1) + racc (1.8.1) + racc (1.8.1-java) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rexml (3.4.4) + ronn-ng (0.10.1) + kramdown (~> 2, >= 2.1) + kramdown-parser-gfm (~> 1, >= 1.0.1) + mustache (~> 1) + nokogiri (~> 1, >= 1.14.3) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.1) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.0) - test-unit (3.5.5) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + test-unit (3.7.7) power_assert - uri (0.12.0) - webrick (1.7.0) + test-unit-ruby-core (1.0.14) + test-unit (>= 3.7.2) + turbo_tests (2.2.5) + parallel_tests (>= 3.3.0, < 5) + rspec (>= 3.10) PLATFORMS + aarch64-darwin + aarch64-linux + arm-linux + arm64-darwin java ruby - universal-java-11 - universal-java-18 + universal-java x64-mingw-ucrt - x64-mingw32 - x86_64-darwin-20 + x64-mswin64-140 + x86-linux + x86_64-darwin x86_64-linux DEPENDENCIES parallel (~> 1.19) - parallel_tests (~> 2.29) - rake (~> 13.0) + parallel_tests (~> 4.10.1) + rake (~> 13.1) rb_sys - ronn (~> 0.7.3) + ronn-ng (~> 0.10.1) rspec-core (~> 3.12) rspec-expectations (~> 3.12) rspec-mocks (~> 3.12) + rubygems-generate_index (~> 1.1) test-unit (~> 3.0) - uri (~> 0.12.0) - webrick (~> 1.6) + test-unit-ruby-core + turbo_tests (~> 2.2.3) + +CHECKSUMS + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + kramdown (2.5.1) sha256=87bbb6abd9d3cebe4fc1f33e367c392b4500e6f8fa19dd61c0972cf4afe7368c + kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 + mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289 + mustache (1.1.1) sha256=90891fdd50b53919ca334c8c1031eada1215e78d226d5795e523d6123a2717d0 + nokogiri (1.19.0) sha256=e304d21865f62518e04f2bf59f93bd3a97ca7b07e7f03952946d8e1c05f45695 + nokogiri (1.19.0-aarch64-linux-gnu) sha256=11a97ecc3c0e7e5edcf395720b10860ef493b768f6aa80c539573530bc933767 + nokogiri (1.19.0-arm-linux-gnu) sha256=572a259026b2c8b7c161fdb6469fa2d0edd2b61cd599db4bbda93289abefbfe5 + nokogiri (1.19.0-arm64-darwin) sha256=0811dfd936d5f6dd3f6d32ef790568bf29b2b7bead9ba68866847b33c9cf5810 + nokogiri (1.19.0-java) sha256=5f3a70e252be641d8a4099f7fb4cc25c81c632cb594eec9b4b8f2ca8be4374f3 + nokogiri (1.19.0-x64-mingw-ucrt) sha256=05d7ed2d95731edc9bef2811522dc396df3e476ef0d9c76793a9fca81cab056b + nokogiri (1.19.0-x86_64-darwin) sha256=1dad56220b603a8edb9750cd95798bffa2b8dd9dd9aa47f664009ee5b43e3067 + nokogiri (1.19.0-x86_64-linux-gnu) sha256=f482b95c713d60031d48c44ce14562f8d2ce31e3a9e8dd0ccb131e9e5a68b58c + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parallel_tests (4.10.1) sha256=df05458c691462b210f7a41fc2651d4e4e8a881e8190e6d1e122c92c07735d70 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + ronn-ng (0.10.1) sha256=4eeb0185c0fbfa889efed923b5b50e949cd869e7d82ac74138acd0c9c7165ec0 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + test-unit-ruby-core (1.0.14) sha256=d2e997796c9c5c5e8e31ac014f83a473ff5c2523a67cfa491b08893e12d43d22 + turbo_tests (2.2.5) sha256=3fa31497d12976d11ccc298add29107b92bda94a90d8a0a5783f06f05102509f BUNDLED WITH - 2.5.0.dev + 4.1.0.dev diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb index 9cb740cd15..a9b6fda11b 100644 --- a/tool/bundler/rubocop_gems.rb +++ b/tool/bundler/rubocop_gems.rb @@ -2,9 +2,10 @@ source "https://rubygems.org" -gem "rubocop", "~> 1.7" +gem "rubocop", ">= 1.52.1", "< 2" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock index 6f888e1d10..d0c3120e11 100644 --- a/tool/bundler/rubocop_gems.rb.lock +++ b/tool/bundler/rubocop_gems.rb.lock @@ -1,73 +1,159 @@ GEM remote: https://rubygems.org/ specs: - ast (2.4.2) - diff-lcs (1.5.0) - json (2.6.3) - json (2.6.3-java) - minitest (5.16.3) - parallel (1.22.1) - parser (3.1.3.0) + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) ast (~> 2.4.1) - power_assert (2.0.2) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) rainbow (3.1.1) - rake (13.0.6) - rake-compiler (1.2.0) + rake (13.3.1) + rake-compiler (1.3.1) rake - rb_sys (0.9.52) - regexp_parser (2.6.1) - rexml (3.2.5) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.0) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.0) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.1) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.0) - rubocop (1.40.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.82.1) json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) - parser (>= 3.1.2.1) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.23.0, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.48.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.24.0) - parser (>= 3.1.1.0) - ruby-progressbar (1.11.0) - test-unit (3.5.5) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + stringio (3.2.0) + test-unit (3.7.7) power_assert - unicode-display_width (2.3.0) + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) PLATFORMS + aarch64-darwin aarch64-linux - arm64-darwin-20 - arm64-darwin-21 - arm64-darwin-22 - universal-java-11 - universal-java-18 + arm64-darwin + ruby + universal-java x64-mingw-ucrt - x86_64-darwin-19 - x86_64-darwin-20 - x86_64-darwin-21 + x64-mswin64-140 + x86_64-darwin x86_64-linux DEPENDENCIES - minitest + irb + minitest (~> 5.1) rake rake-compiler rb_sys rspec - rubocop (~> 1.7) + rubocop (>= 1.52.1, < 2) test-unit +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.82.1) sha256=09f1a6a654a960eda767aebea33e47603080f8e9c9a3f019bf9b94c9cab5e273 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + BUNDLED WITH - 2.5.0.dev + 4.1.0.dev diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb index 20c1ecd827..f7bc34cf5e 100644 --- a/tool/bundler/standard_gems.rb +++ b/tool/bundler/standard_gems.rb @@ -4,7 +4,8 @@ source "https://rubygems.org" gem "standard", "~> 1.0" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock index d74de2452b..f3792f8611 100644 --- a/tool/bundler/standard_gems.rb.lock +++ b/tool/bundler/standard_gems.rb.lock @@ -1,75 +1,121 @@ GEM remote: https://rubygems.org/ specs: - ast (2.4.2) - diff-lcs (1.5.0) - json (2.6.3) - json (2.6.3-java) - language_server-protocol (3.17.0.2) - minitest (5.16.3) - parallel (1.22.1) - parser (3.1.3.0) + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) ast (~> 2.4.1) - power_assert (2.0.2) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) rainbow (3.1.1) - rake (13.0.6) - rake-compiler (1.2.0) + rake (13.3.1) + rake-compiler (1.3.1) rake - rb_sys (0.9.52) - regexp_parser (2.6.1) - rexml (3.2.5) - rspec (3.12.0) - rspec-core (~> 3.12.0) - rspec-expectations (~> 3.12.0) - rspec-mocks (~> 3.12.0) - rspec-core (3.12.0) - rspec-support (~> 3.12.0) - rspec-expectations (3.12.0) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-mocks (3.12.1) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.12.0) - rspec-support (3.12.0) - rubocop (1.39.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.81.7) json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) - parser (>= 3.1.2.1) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.23.0, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.47.1, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.24.0) - parser (>= 3.1.1.0) - rubocop-performance (1.15.1) - rubocop (>= 1.7.0, < 2.0) - rubocop-ast (>= 0.4.0) - ruby-progressbar (1.11.0) - standard (1.19.1) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + rubocop-performance (1.26.1) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (1.13.0) + standard (1.52.0) language_server-protocol (~> 3.17.0.2) - rubocop (= 1.39.0) - rubocop-performance (= 1.15.1) - test-unit (3.5.5) + lint_roller (~> 1.0) + rubocop (~> 1.81.7) + standard-custom (~> 1.0.0) + standard-performance (~> 1.8) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.9.0) + lint_roller (~> 1.1) + rubocop-performance (~> 1.26.0) + stringio (3.2.0) + test-unit (3.7.7) power_assert - unicode-display_width (2.3.0) + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) PLATFORMS + aarch64-darwin aarch64-linux - arm64-darwin-20 - arm64-darwin-21 - arm64-darwin-22 - universal-java-11 - universal-java-18 + arm64-darwin + ruby + universal-java x64-mingw-ucrt - x86_64-darwin-19 - x86_64-darwin-20 - x86_64-darwin-21 + x64-mswin64-140 + x86_64-darwin x86_64-linux DEPENDENCIES - minitest + irb + minitest (~> 5.1) rake rake-compiler rb_sys @@ -77,5 +123,57 @@ DEPENDENCIES standard (~> 1.0) test-unit +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.81.7) sha256=6fb5cc298c731691e2a414fe0041a13eb1beed7bab23aec131da1bcc527af094 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834 + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + standard (1.52.0) sha256=ec050e63228e31fabe40da3ef96da7edda476f7acdf3e7c2ad47b6e153f6a076 + standard-custom (1.0.2) sha256=424adc84179a074f1a2a309bb9cf7cd6bfdb2b6541f20c6bf9436c0ba22a652b + standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + BUNDLED WITH - 2.5.0.dev + 4.1.0.dev diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb index 9ba5763a3b..ddc19e2939 100644 --- a/tool/bundler/test_gems.rb +++ b/tool/bundler/test_gems.rb @@ -2,11 +2,17 @@ source "https://rubygems.org" -gem "rack", "2.0.8" -gem "webrick", "1.7.0" -gem "rack-test", "~> 1.1" -gem "compact_index", "~> 0.13.0" -gem "sinatra", "~> 2.0" -gem "rake", "13.0.1" +gem "rack", "~> 3.1" +gem "rack-test", "~> 2.1" +gem "compact_index", "~> 0.15.0" +gem "sinatra", "~> 4.1" +gem "rake", "~> 13.1" gem "builder", "~> 3.2" gem "rb_sys" +gem "fiddle" +gem "rubygems-generate_index", "~> 1.1" +gem "concurrent-ruby" +gem "psych" +gem "etc", platforms: [:ruby, :windows] +gem "open3" +gem "shellwords" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock index 16e25ffd5c..fdffc1f09d 100644 --- a/tool/bundler/test_gems.rb.lock +++ b/tool/bundler/test_gems.rb.lock @@ -1,45 +1,106 @@ GEM remote: https://rubygems.org/ specs: - builder (3.2.4) - compact_index (0.13.0) - mustermann (1.1.2) + base64 (0.3.0) + builder (3.3.0) + compact_index (0.15.0) + concurrent-ruby (1.3.6) + date (3.5.1) + date (3.5.1-java) + etc (1.4.6) + fiddle (1.1.8) + jar-dependencies (0.5.5) + logger (1.7.0) + mustermann (3.0.4) ruby2_keywords (~> 0.0.1) - rack (2.0.8) - rack-protection (2.0.8.1) - rack - rack-test (1.1.0) - rack (>= 1.0, < 3) - rake (13.0.1) - rb_sys (0.9.52) + open3 (0.2.1) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + rack (3.2.4) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) ruby2_keywords (0.0.5) - sinatra (2.0.8.1) - mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.8.1) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + shellwords (0.2.2) + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) tilt (~> 2.0) - tilt (2.0.11) - webrick (1.7.0) + stringio (3.2.0) + tilt (2.6.1) PLATFORMS java ruby - universal-java-11 - universal-java-18 + universal-java x64-mingw-ucrt - x64-mingw32 - x86_64-darwin-20 + x64-mswin64-140 + x86_64-darwin x86_64-linux DEPENDENCIES builder (~> 3.2) - compact_index (~> 0.13.0) - rack (= 2.0.8) - rack-test (~> 1.1) - rake (= 13.0.1) + compact_index (~> 0.15.0) + concurrent-ruby + etc + fiddle + open3 + psych + rack (~> 3.1) + rack-test (~> 2.1) + rake (~> 13.1) rb_sys - sinatra (~> 2.0) - webrick (= 1.7.0) + rubygems-generate_index (~> 1.1) + shellwords + sinatra (~> 4.1) + +CHECKSUMS + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + etc (1.4.6) sha256=0f7e9e7842ea5e3c3bd9bc81746ebb8c65ea29e4c42a93520a0d638129c7de01 + fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mustermann (3.0.4) sha256=85fadcb6b3c6493a8b511b42426f904b7f27b282835502233dd154daab13aa22 + open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + rack (3.2.4) sha256=5d74b6f75082a643f43c1e76b419c40f0e5527fcfee1e669ac1e6b73c0ccb6f6 + rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac + rack-session (2.1.1) sha256=0b6dc07dea7e4b583f58a48e8b806d4c9f1c6c9214ebc202ec94562cbea2e4e9 + rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94 + sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + tilt (2.6.1) sha256=35a99bba2adf7c1e362f5b48f9b581cce4edfba98117e34696dde6d308d84770 BUNDLED WITH - 2.5.0.dev + 4.1.0.dev diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb new file mode 100644 index 0000000000..8d12c5adde --- /dev/null +++ b/tool/bundler/vendor_gems.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "fileutils", "1.8.0" +gem "molinillo", github: "cocoapods/molinillo", ref: "1d62d7d5f448e79418716dc779a4909509ccda2a" +gem "net-http", "0.7.0" # net-http-0.8.0 is broken with JRuby +gem "net-http-persistent", "4.0.6" +gem "net-protocol", "0.2.2" +gem "optparse", "0.8.0" +gem "pub_grub", github: "jhawthorn/pub_grub", ref: "df6add45d1b4d122daff2f959c9bd1ca93d14261" +gem "resolv", "0.6.2" +gem "securerandom", "0.4.1" +gem "timeout", "0.4.4" +gem "thor", "1.4.0" +gem "tsort", "0.2.0" +gem "uri", "1.1.1" diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock new file mode 100644 index 0000000000..cc7886e60b --- /dev/null +++ b/tool/bundler/vendor_gems.rb.lock @@ -0,0 +1,75 @@ +GIT + remote: https://github.com/cocoapods/molinillo.git + revision: 1d62d7d5f448e79418716dc779a4909509ccda2a + ref: 1d62d7d5f448e79418716dc779a4909509ccda2a + specs: + molinillo (0.8.0) + +GIT + remote: https://github.com/jhawthorn/pub_grub.git + revision: df6add45d1b4d122daff2f959c9bd1ca93d14261 + ref: df6add45d1b4d122daff2f959c9bd1ca93d14261 + specs: + pub_grub (0.5.0) + +GEM + remote: https://rubygems.org/ + specs: + connection_pool (2.5.4) + fileutils (1.8.0) + net-http (0.7.0) + uri + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + net-protocol (0.2.2) + timeout + optparse (0.8.0) + resolv (0.6.2) + securerandom (0.4.1) + thor (1.4.0) + timeout (0.4.4) + tsort (0.2.0) + uri (1.1.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + fileutils (= 1.8.0) + molinillo! + net-http (= 0.7.0) + net-http-persistent (= 4.0.6) + net-protocol (= 0.2.2) + optparse (= 0.8.0) + pub_grub! + resolv (= 0.6.2) + securerandom (= 0.4.1) + thor (= 1.4.0) + timeout (= 0.4.4) + tsort (= 0.2.0) + uri (= 1.1.1) + +CHECKSUMS + connection_pool (2.5.4) sha256=e9e1922327416091f3f6542f5f4446c2a20745276b9aa796dd0bb2fd0ea1e70a + fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02 + molinillo (0.8.0) + net-http (0.7.0) sha256=4db7d9f558f8ffd4dcf832d0aefd02320c569c7d4f857def49e585069673a425 + net-http-persistent (4.0.6) sha256=2abb3a04438edf6cb9e0e7e505969605f709eda3e3c5211beadd621a2c84dd5d + net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 + optparse (0.8.0) sha256=ef6b7fbaf7ec331474f325bc08dd5622e6e1e651007a5341330ee4b08ce734f0 + pub_grub (0.5.0) + resolv (0.6.2) sha256=61efe545cedddeb1b14f77e51f85c85ca66af5098fdbf567fadf32c34590fb14 + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + thor (1.4.0) sha256=8763e822ccb0f1d7bee88cde131b19a65606657b847cc7b7b4b82e772bcd8a3d + timeout (0.4.4) sha256=f0f6f970104b82427cd990680f539b6bbb8b1e55efa913a55c6492935e4e0edb + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + +BUNDLED WITH + 4.0.0.dev diff --git a/tool/ci_functions.sh b/tool/ci_functions.sh deleted file mode 100644 index 7066bbe4ec..0000000000 --- a/tool/ci_functions.sh +++ /dev/null @@ -1,29 +0,0 @@ -# -*- BASH -*- -# Manage functions used on a CI. -# Run `. tool/ci_functions.sh` to use it. - -# Create options with patterns `-n !/name1/ -n !/name2/ ..` to exclude the test -# method names by the method names `name1 name2 ..`. -# See `ruby tool/test/runner.rb --help` `-n` option. -function ci_to_excluded_test_opts { - local tests_str="${1}" - # Use the backward matching `!/name$/`, as the perfect matching doesn't work. - # https://bugs.ruby-lang.org/issues/16936 - ruby <<EOF - opts = "${tests_str}".split.map { |test| "-n \!/#{test}\$$/" } - puts opts.join(' ') -EOF - return 0 -} - -# Create options with patterns `-n name1 -n name2 ..` to include the test -# method names by the method names `name1 name2 ..`. -# See `ruby tool/test/runner.rb --help` `-n` option. -function ci_to_included_test_opts { - local tests_str="${1}" - ruby <<EOF - opts = "${tests_str}".split.map { |test| "-n #{test}" } - puts opts.join(' ') -EOF - return 0 -} diff --git a/tool/commit-email.rb b/tool/commit-email.rb new file mode 100755 index 0000000000..c887f8783e --- /dev/null +++ b/tool/commit-email.rb @@ -0,0 +1,372 @@ +#!/usr/bin/env ruby + +require "optparse" +require "nkf" +require "shellwords" + +CommitEmailInfo = Struct.new( + :author, + :author_email, + :revision, + :entire_sha256, + :date, + :log, + :branch, + :diffs, + :added_files, :deleted_files, :updated_files, + :added_dirs, :deleted_dirs, :updated_dirs, +) + +class GitInfoBuilder + GitCommandFailure = Class.new(RuntimeError) + + def initialize(repo_path) + @repo_path = repo_path + end + + def build(oldrev, newrev, refname) + diffs = build_diffs(oldrev, newrev) + + info = CommitEmailInfo.new + info.author = git_show(newrev, format: '%an') + info.author_email = normalize_email(git_show(newrev, format: '%aE')) + info.revision = newrev[0...10] + info.entire_sha256 = newrev + info.date = Time.at(Integer(git_show(newrev, format: '%at'))) + info.log = git_show(newrev, format: '%B') + info.branch = git('rev-parse', '--symbolic', '--abbrev-ref', refname).strip + info.diffs = diffs + info.added_files = find_files(diffs, status: :added) + info.deleted_files = find_files(diffs, status: :deleted) + info.updated_files = find_files(diffs, status: :modified) + info.added_dirs = [] # git does not deal with directory + info.deleted_dirs = [] # git does not deal with directory + info.updated_dirs = [] # git does not deal with directory + info + end + + private + + # Force git-svn email address to @ruby-lang.org to avoid email bounce by invalid email address. + def normalize_email(email) + if email.match(/\A[^@]+@\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/) # git-svn + svn_user, _ = email.split('@', 2) + "#{svn_user}@ruby-lang.org" + else + email + end + end + + def find_files(diffs, status:) + files = [] + diffs.each do |path, values| + if values.keys.first == status + files << path + end + end + files + end + + # SVN version: + # { + # "filename" => { + # "[modified|added|deleted|copied|property_changed]" => { + # type: "[modified|added|deleted|copied|property_changed]", + # body: "diff body", # not implemented because not used + # added: Integer, + # deleted: Integer, + # } + # } + # } + def build_diffs(oldrev, newrev) + diffs = {} + + numstats = git('diff', '--numstat', oldrev, newrev).lines.map { |l| l.strip.split("\t", 3) } + git('diff', '--name-status', oldrev, newrev).each_line do |line| + status, path, _newpath = line.strip.split("\t", 3) + diff = build_diff(path, numstats) + + case status + when 'A' + diffs[path] = { added: { type: :added, **diff } } + when 'M' + diffs[path] = { modified: { type: :modified, **diff } } + when 'C' + diffs[path] = { copied: { type: :copied, **diff } } + when 'D' + diffs[path] = { deleted: { type: :deleted, **diff } } + when /\AR/ # R100 (which does not exist in git.ruby-lang.org's git 2.1.4) + # TODO: implement something + else + $stderr.puts "unexpected git diff status: #{status}" + end + end + + diffs + end + + def build_diff(path, numstats) + diff = { added: 0, deleted: 0 } # :body not implemented because not used + line = numstats.find { |(_added, _deleted, file, *)| file == path } + return diff if line.nil? + + added, deleted, _ = line + if added + diff[:added] = Integer(added) + end + if deleted + diff[:deleted] = Integer(deleted) + end + diff + end + + def git_show(revision, format:) + git('show', '--no-show-signature', "--pretty=#{format}", '--no-patch', revision).strip + end + + def git(*args) + command = ['git', '-C', @repo_path, *args] + output = with_gitenv { IO.popen(command, external_encoding: 'UTF-8', &:read) } + unless $?.success? + raise GitCommandFailure, "failed to execute '#{command.join(' ')}':\n#{output}" + end + output + end + + def with_gitenv + orig = ENV.to_h.dup + begin + ENV.delete('GIT_DIR') + yield + ensure + ENV.replace(orig) + end + end +end + +CommitEmailOptions = Struct.new(:error_to, :viewer_uri) + +CommitEmail = Module.new +class << CommitEmail + SENDMAIL = ENV.fetch('SENDMAIL', '/usr/sbin/sendmail') + private_constant :SENDMAIL + + def parse(args) + options = CommitEmailOptions.new + + opts = OptionParser.new do |opts| + opts.separator('') + + opts.on('-e', '--error-to [TO]', + 'Add [TO] to to address when error is occurred') do |to| + options.error_to = to + end + + opts.on('--viewer-uri [URI]', + 'Use [URI] as URI of revision viewer') do |uri| + options.viewer_uri = uri + end + + opts.on_tail('--help', 'Show this message') do + puts opts + exit + end + end + + return opts.parse(args), options + end + + def main(repo_path, to, rest) + args, options = parse(rest) + + infos = args.each_slice(3).flat_map do |oldrev, newrev, refname| + revisions = IO.popen(['git', 'log', '--no-show-signature', '--reverse', '--pretty=%H', "#{oldrev}^..#{newrev}"], &:read).lines.map(&:strip) + revisions[0..-2].zip(revisions[1..-1]).map do |old, new| + GitInfoBuilder.new(repo_path).build(old, new, refname) + end + end + + infos.each do |info| + next if info.branch.start_with?('notes/') + puts "#{info.branch}: #{info.revision} (#{info.author})" + + from = make_from(name: info.author, email: "noreply@ruby-lang.org") + sendmail(to, from, make_mail(to, from, info, viewer_uri: options.viewer_uri)) + end + end + + def sendmail(to, from, mail) + IO.popen([*SENDMAIL.shellsplit, to], 'w') do |f| + f.print(mail) + end + unless $?.success? + raise "Failed to run `#{SENDMAIL} #{to}` with: '#{mail}'" + end + end + + private + + def b_encode(str) + NKF.nkf('-WwM', str) + end + + def make_body(info, viewer_uri:) + body = +'' + body << "#{info.author}\t#{format_time(info.date)}\n" + body << "\n" + body << " New Revision: #{info.revision}\n" + body << "\n" + body << " #{viewer_uri}#{info.revision}\n" + body << "\n" + body << " Log:\n" + body << info.log.lstrip.gsub(/^\t*/, ' ').rstrip + body << "\n\n" + body << added_dirs(info) + body << added_files(info) + body << deleted_dirs(info) + body << deleted_files(info) + body << modified_dirs(info) + body << modified_files(info) + [body.rstrip].pack('M') + end + + def format_time(time) + time.strftime('%Y-%m-%d %X %z (%a, %d %b %Y)') + end + + def changed_items(title, type, items) + rv = '' + unless items.empty? + rv << " #{title} #{type}:\n" + rv << items.collect {|item| " #{item}\n"}.join('') + end + rv + end + + def changed_files(title, files) + changed_items(title, 'files', files) + end + + def added_files(info) + changed_files('Added', info.added_files) + end + + def deleted_files(info) + changed_files('Removed', info.deleted_files) + end + + def modified_files(info) + changed_files('Modified', info.updated_files) + end + + def changed_dirs(title, files) + changed_items(title, 'directories', files) + end + + def added_dirs(info) + changed_dirs('Added', info.added_dirs) + end + + def deleted_dirs(info) + changed_dirs('Removed', info.deleted_dirs) + end + + def modified_dirs(info) + changed_dirs('Modified', info.updated_dirs) + end + + def changed_dirs_info(info, uri) + (info.added_dirs.collect do |dir| + " Added: #{dir}\n" + end + info.deleted_dirs.collect do |dir| + " Deleted: #{dir}\n" + end + info.updated_dirs.collect do |dir| + " Modified: #{dir}\n" + end).join("\n") + end + + def diff_info(info, uri) + info.diffs.collect do |key, values| + [ + key, + values.collect do |type, value| + case type + when :added + rev = "?revision=#{info.revision}&view=markup" + when :modified, :property_changed + prev_revision = (info.revision.is_a?(Integer) ? info.revision - 1 : "#{info.revision}^") + rev = "?r1=#{info.revision}&r2=#{prev_revision}&diff_format=u" + when :deleted, :copied + rev = '' + else + raise "unknown diff type: #{value[:type]}" + end + + link = [uri, key.sub(/ .+/, '') || ''].join('/') + rev + + desc = '' + + [desc, link] + end + ] + end + end + + def make_header(to, from, info) + <<~EOS + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: #{from} + To: #{to} + Subject: #{make_subject(info)} + EOS + end + + def make_subject(info) + subject = +'' + subject << "#{info.revision}" + subject << " (#{info.branch})" + subject << ': ' + subject << info.log.lstrip.lines.first.to_s.strip + b_encode(subject) + end + + # https://tools.ietf.org/html/rfc822#section-4.1 + # https://tools.ietf.org/html/rfc822#section-6.1 + # https://tools.ietf.org/html/rfc822#appendix-D + # https://tools.ietf.org/html/rfc2047 + def make_from(name:, email:) + if name.ascii_only? + escaped_name = name.gsub(/["\\\n]/) { |c| "\\#{c}" } + %Q["#{escaped_name}" <#{email}>] + else + escaped_name = "=?UTF-8?B?#{NKF.nkf('-WwMB', name)}?=" + %Q[#{escaped_name} <#{email}>] + end + end + + def make_mail(to, from, info, viewer_uri:) + make_header(to, from, info) + make_body(info, viewer_uri: viewer_uri) + end +end + +repo_path, to, *rest = ARGV +begin + CommitEmail.main(repo_path, to, rest) +rescue StandardError => e + $stderr.puts "#{e.class}: #{e.message}" + $stderr.puts e.backtrace + + _, options = CommitEmail.parse(rest) + to = options.error_to + CommitEmail.sendmail(to, to, <<-MAIL) +From: #{to} +To: #{to} +Subject: Error + +#{$!.class}: #{$!.message} +#{$@.join("\n")} +MAIL + exit 1 +end diff --git a/tool/darwin-ar b/tool/darwin-ar new file mode 100755 index 0000000000..8b25425cfe --- /dev/null +++ b/tool/darwin-ar @@ -0,0 +1,6 @@ +#!/bin/bash +export LANG=C LC_ALL=C # Suppress localication +exec 2> >(exec grep -v \ + -e ' no symbols$' \ + >&2) +exec "$@" diff --git a/tool/darwin-cc b/tool/darwin-cc index 6eee96e435..42637022a4 100755 --- a/tool/darwin-cc +++ b/tool/darwin-cc @@ -2,5 +2,8 @@ exec 2> >(exec grep -v \ -e '^ld: warning: The [a-z0-9_][a-z0-9_]* architecture is deprecated for macOS' \ -e '^ld: warning: text-based stub file /System/Library/Frameworks/' \ + -e '^ld: warning: ignoring duplicate libraries:' \ + -e "warning: '\.debug_macinfo' is not currently supported:" \ + -e "note: while processing" \ >&2) exec "$@" diff --git a/tool/downloader.rb b/tool/downloader.rb index 0bcd8f31c3..39ebf44a83 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -1,39 +1,14 @@ # Used by configure and make to download or update mirrored Ruby and GCC -# files. This will use HTTPS if possible, falling back to HTTP. +# files. + +# -*- frozen-string-literal: true -*- require 'fileutils' require 'open-uri' require 'pathname' -begin - require 'net/https' -rescue LoadError - https = 'http' -else - https = 'https' - - # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old - # versions do not. so, patching OpenSSL::X509::Store#add_file instead. - class OpenSSL::X509::Store - alias orig_add_file add_file - def add_file(pems) - Array(pems).each do |pem| - if File.directory?(pem) - add_path pem - else - orig_add_file pem - end - end - end - end - # since open-uri internally checks ssl_ca_cert using File.directory?, - # allow to accept an array. - class <<File - alias orig_directory? directory? - def File.directory? files - files.is_a?(Array) ? false : orig_directory?(files) - end - end -end +verbose, $VERBOSE = $VERBOSE, nil +require 'net/https' +$VERBOSE = verbose class Downloader def self.find(dlname) @@ -42,44 +17,37 @@ class Downloader end end - def self.https=(https) - @@https = https - end - - def self.https? - @@https == 'https' - end - - def self.https - @@https - end - def self.get_option(argv, options) false end class GNU < self - def self.download(name, *rest) - if https? - begin - super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest) - rescue => e - m1, m2 = e.message.split("\n", 2) - STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" - super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) - end + Mirrors = %w[ + https://raw.githubusercontent.com/autotools-mirror/autoconf/refs/heads/master/build-aux/ + https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master + ] + + def self.download(name, *rest, **options) + Mirrors.each_with_index do |url, i| + super("#{url}/#{name}", name, *rest, **options) + rescue => e + raise if i + 1 == Mirrors.size # no more URLs + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" else - super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) + return end end end class RubyGems < self - def self.download(name, dir = nil, since = true, options = {}) + def self.download(name, dir = nil, since = true, **options) require 'rubygems' - options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) - super("https://rubygems.org/downloads/#{name}", name, dir, since, options) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true + end + super("https://rubygems.org/downloads/#{name}", name, dir, since, **options) end end @@ -104,16 +72,13 @@ class Downloader end end - def self.download(name, dir = nil, since = true, options = {}) - options = options.dup - unicode_beta = options.delete(:unicode_beta) + def self.download(name, dir = nil, since = true, unicode_beta: nil, **options) name_dir_part = name.sub(/[^\/]+$/, '') if unicode_beta == 'YES' if INDEX.size == 0 - index_options = options.dup - index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file + cache_save = false # TODO: make sure caching really doesn't work for index file index_data = File.read(under(dir, "index.html")) rescue nil - index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options) + index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, cache_save: cache_save, **options) INDEX[:index] = File.read(index_file) since = true unless INDEX[:index] == index_data end @@ -122,7 +87,7 @@ class Downloader beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] # make sure we always check for new versions of files, # because they can easily change in the beta period - super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options) + super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, **options) else index_file = Pathname.new(under(dir, name_dir_part+'index.html')) if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ @@ -130,7 +95,7 @@ class Downloader "Remove all files in this directory and in .downloaded-cache/ " + "because they may be leftovers from the beta period." end - super(UNICODE_PUBLIC+name, name, dir, since, options) + super(UNICODE_PUBLIC+name, name, dir, since, **options) end end end @@ -195,64 +160,53 @@ class Downloader # Example usage: # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', # 'UnicodeData.txt', 'enc/unicode/data' - def self.download(url, name, dir = nil, since = true, options = {}) - options = options.dup + def self.download(url, name, dir = nil, since = true, + cache_save: ENV["CACHE_SAVE"] != "no", cache_dir: nil, + ignore_http_client_errors: nil, + dryrun: nil, verbose: false, **options) url = URI(url) - dryrun = options.delete(:dryrun) - if name file = Pathname.new(under(dir, name)) else name = File.basename(url.path) end - cache_save = options.delete(:cache_save) { - ENV["CACHE_SAVE"] != "no" - } - cache = cache_file(url, name, options.delete(:cache_dir)) + cache = cache_file(url, name, cache_dir) file ||= cache if since.nil? and file.exist? - if $VERBOSE + if verbose $stdout.puts "#{file} already exists" $stdout.flush end - if cache_save - save_cache(cache, file, name) - end return file.to_path end if dryrun puts "Download #{url} into #{file}" return end - if link_cache(cache, file, name, $VERBOSE) + if link_cache(cache, file, name, verbose: verbose) return file.to_path end - if !https? and URI::HTTPS === url - warn "*** using http instead of https ***" - url.scheme = 'http' - url = URI(url.to_s) - end - if $VERBOSE + if verbose $stdout.print "downloading #{name} ... " $stdout.flush end mtime = nil options = options.merge(http_options(file, since.nil? ? true : since)) begin - data = with_retry(10) do - data = url.read(options) - if mtime = data.meta["last-modified"] - mtime = Time.httpdate(mtime) - end - data - end + data = with_retry(10) {url.read(options)} rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified - if $VERBOSE + case http_error.message + when /^304 / # 304 Not Modified + if verbose $stdout.puts "#{name} not modified" $stdout.flush end return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error @@ -267,9 +221,14 @@ class Downloader return file.to_path end raise + else + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end end dest = (cache_save && cache && !cache.exist? ? cache : file) dest.parent.mkpath + dest.unlink if dest.symlink? && !dest.exist? dest.open("wb", 0600) do |f| f.write(data) f.chmod(mode_for(data)) @@ -277,7 +236,7 @@ class Downloader if mtime dest.utime(mtime, mtime) end - if $VERBOSE + if verbose $stdout.puts "done" $stdout.flush end @@ -295,20 +254,24 @@ class Downloader dir ? File.join(dir, File.basename(name)) : name end + def self.default_cache_dir + if cache_dir = ENV['CACHE_DIR'] + return cache_dir unless cache_dir.empty? + end + ".downloaded-cache" + end + def self.cache_file(url, name, cache_dir = nil) case cache_dir when false return nil when nil - cache_dir = ENV['CACHE_DIR'] - if !cache_dir or cache_dir.empty? - cache_dir = ".downloaded-cache" - end + cache_dir = default_cache_dir end Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) end - def self.link_cache(cache, file, name, verbose = false) + def self.link_cache(cache, file, name, verbose: false) return false unless cache and cache.exist? return true if cache.eql?(file) if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] @@ -380,8 +343,6 @@ class Downloader private_class_method :with_retry end -Downloader.https = https.freeze - if $0 == __FILE__ since = true options = {} @@ -406,26 +367,42 @@ if $0 == __FILE__ case ARGV[0] when '-d', '--destdir' + ## -d, --destdir DIRECTORY Download into the directory destdir = ARGV[1] ARGV.shift when '-p', '--prefix' - # strip directory names from the name to download, and add the - # prefix instead. + ## -p, --prefix Strip directory names from the name to download, + ## and add the prefix instead. prefix = ARGV[1] ARGV.shift when '-e', '--exist', '--non-existent-only' + ## -e, --exist, --non-existent-only Skip already existent files. since = nil when '-a', '--always' + ## -a, --always Download all files. since = false when '-u', '--update', '--if-modified' + ## -u, --update, --if-modified Download newer files only. since = true - when '-n', '--dryrun' + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Do not download actually. options[:dryrun] = true when '--cache-dir' + ## --cache-dir DIRECTORY Cache downloaded files in the directory. options[:cache_dir] = ARGV[1] ARGV.shift when /\A--cache-dir=(.*)/m options[:cache_dir] = $1 + when /\A--help\z/ + ## --help Print this message + puts "Usage: #$0 [options] relative-url..." + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ abort "#{$0}: unknown option #{ARGV[0]}" else @@ -433,7 +410,7 @@ if $0 == __FILE__ end ARGV.shift end - $VERBOSE = true + options[:verbose] = true if dl args.each do |name| dir = destdir @@ -452,10 +429,10 @@ if $0 == __FILE__ end name = "#{prefix}/#{name}" end - dl.download(name, dir, since, options) + dl.download(name, dir, since, **options) end else abort "usage: #{$0} url name" unless args.size == 2 - Downloader.download(args[0], args[1], destdir, since, options) + Downloader.download(args[0], args[1], destdir, since, **options) end end diff --git a/tool/dummy-rake-compiler/rake/extensiontask.rb b/tool/dummy-rake-compiler/rake/extensiontask.rb deleted file mode 100644 index 62b7ff8018..0000000000 --- a/tool/dummy-rake-compiler/rake/extensiontask.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Rake - class ExtensionTask < TaskLib - def initialize(...) - task :compile do - puts "Dummy `compile` task defined in #{__FILE__}" - end - end - end -end diff --git a/tool/enc-case-folding.rb b/tool/enc-case-folding.rb index 76c6b5c48b..82fec7b625 100755 --- a/tool/enc-case-folding.rb +++ b/tool/enc-case-folding.rb @@ -61,7 +61,7 @@ class CaseFolding @version = nil turkic = [] - IO.foreach(filename, mode: "rb") do |line| + File.foreach(filename, mode: "rb") do |line| @version ||= line[/-([0-9.]+).txt/, 1] next unless res = pattern.match(line) ch_from = res[1].to_i(16) @@ -230,7 +230,7 @@ class CaseMapping @specials = [] @specials_length = 0 @version = nil - IO.foreach(File.join(mapping_directory, 'UnicodeData.txt'), mode: "rb") do |line| + File.foreach(File.join(mapping_directory, 'UnicodeData.txt'), mode: "rb") do |line| next if line =~ /^</ code, _, _, _, _, _, _, _, _, _, _, _, upper, lower, title = line.chomp.split ';' unless upper and lower and title and (upper+lower+title)=='' @@ -239,7 +239,7 @@ class CaseMapping end @filename = File.join(mapping_directory, 'SpecialCasing.txt') - IO.foreach(@filename, mode: "rb") do |line| + File.foreach(@filename, mode: "rb") do |line| @version ||= line[/-([0-9.]+).txt/, 1] line.chomp! line, comment = line.split(/ *#/) diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index 6f2576cc37..a89390ad8f 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -5,14 +5,33 @@ # # To use this, get UnicodeData.txt, Scripts.txt, PropList.txt, # PropertyAliases.txt, PropertyValueAliases.txt, DerivedCoreProperties.txt, -# DerivedAge.txt and Blocks.txt from unicode.org. +# DerivedAge.txt, Blocks.txt, emoji/emoji-data.txt, +# auxiliary/GraphemeBreakProperty.txt from unicode.org # (http://unicode.org/Public/UNIDATA/) And run following command. -# ruby1.9 tool/enc-unicode.rb data_dir > enc/unicode/name2ctype.kwd +# tool/enc-unicode.rb data_dir emoji_data_dir > enc/unicode/name2ctype.kwd # You can get source file for gperf. After this, simply make ruby. - -if ARGV[0] == "--header" - header = true - ARGV.shift +# Or directly run: +# tool/enc-unicode.rb --header data_dir emoji_data_dir > enc/unicode/<VERSION>/name2ctype.h +# +# There are Makefile rules that automate steps above: `make update-unicode` and +# `make enc/unicode/<VERSION>/name2ctype.h`. + +while arg = ARGV.shift + case arg + when "--" + break + when "--header" + header = true + when "--diff" + diff = ARGV.shift or abort "#{$0}: --diff=DIFF-COMMAND" + when /\A--diff=(.+)/m + diff = $1 + when /\A-/ + abort "#{$0}: unknown option #{arg}" + else + ARGV.unshift(arg) + break + end end unless ARGV.size == 2 abort "Usage: #{$0} data_directory emoji_data_directory" @@ -59,7 +78,7 @@ def parse_unicode_data(file) data = {'Any' => (0x0000..0x10ffff).to_a, 'Assigned' => [], 'ASCII' => (0..0x007F).to_a, 'NEWLINE' => [0x0a], 'Cn' => []} beg_cp = nil - IO.foreach(file) do |line| + File.foreach(file) do |line| fields = line.split(';') cp = fields[0].to_i(16) @@ -127,7 +146,8 @@ def define_posix_props(data) data['Space'] = data['White_Space'] data['Blank'] = data['Space_Separator'] + [0x0009] data['Cntrl'] = data['Cc'] - data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + + data['Connector_Punctuation'] + data['Join_Control'] data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - data['Surrogate'] - data['Unassigned'] data['Print'] = data['Graph'] + data['Space_Separator'] @@ -145,14 +165,24 @@ def parse_scripts(data, categories) names = {} files.each do |file| data_foreach(file[:fn]) do |line| + # Parse Unicode data files and store code points and properties. if /^# Total (?:code points|elements): / =~ line data[current] = cps categories[current] = file[:title] (names[file[:title]] ||= []) << current cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line - current = $3 + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w(?:[\w\s;]*\w)?)/ =~ line + # $1: The first hexadecimal code point or the start of a range. + # $2: The end code point of the range, if present. + # If there's no range (just a single code point), $2 is nil. + # $3: The property or other info. + # Example: + # line = "0915..0939 ; InCB; Consonant # Lo [37] DEVANAGARI LETTER KA..DEVANAGARI LETTER HA" + # $1 = "0915" + # $2 = "0939" + # $3 = "InCB; Consonant" $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + current = $3.gsub(/\W+/, '_') end end end @@ -205,7 +235,7 @@ def parse_age(data) ages << current last_constname = constname cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\d+\.\d+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\d+\.\d+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -224,7 +254,7 @@ def parse_GraphemeBreakProperty(data) make_const(constname, cps, "Grapheme_Cluster_Break=#{current}") ages << current cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -236,7 +266,7 @@ def parse_block(data) cps = [] blocks = [] data_foreach('Blocks.txt') do |line| - if /^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+);\s*(.*)/ =~ line + if /^(\h+)\.\.(\h+);\s*(.*)/ =~ line cps = ($1.to_i(16)..$2.to_i(16)).to_a constname = constantize_blockname($3) data[constname] = cps @@ -253,23 +283,12 @@ def parse_block(data) blocks << constname end -# shim for Ruby 1.8 -unless {}.respond_to?(:key) - class Hash - alias key index - end -end - $const_cache = {} # make_const(property, pairs, name): Prints a 'static const' structure for a # given property, group of paired codepoints, and a human-friendly name for # the group def make_const(prop, data, name) - if name.empty? - puts "\n/* '#{prop}' */" - else - puts "\n/* '#{prop}': #{name} */" - end + puts "\n/* '#{prop}': #{name} */" # comment used to generate documentation if origprop = $const_cache.key(data) puts "#define CR_#{prop} CR_#{origprop}" else @@ -421,8 +440,6 @@ define_posix_props(data) POSIX_NAMES.each do |name| if name == 'XPosixPunct' make_const(name, data[name], "[[:Punct:]]") - elsif name == 'Punct' - make_const(name, data[name], "") else make_const(name, data[name], "[[:#{name}:]]") end @@ -483,7 +500,11 @@ end output.ifdef :USE_UNICODE_PROPERTIES do props.each do |name| i += 1 - name = normalize_propname(name) + name = if name.start_with?('InCB') + name.downcase.gsub(/_/, '=') + else + normalize_propname(name) + end name_to_index[name] = i puts "%-40s %3d" % [name + ',', i] end @@ -544,6 +565,47 @@ output.restore if header require 'tempfile' + def diff_args(diff) + ok = IO.popen([diff, "-DDIFF_TEST", IO::NULL, "-"], "r+") do |f| + f.puts "Test for diffutils 3.8" + f.close_write + /^#if/ =~ f.read + end + if ok + proc {|macro, *inputs| + [diff, "-D#{macro}", *inputs] + } + else + IO.popen([diff, "--old-group-format=%<", "--new-group-format=%>", IO::NULL, IO::NULL], err: %i[child out], &:read) + unless $?.success? + abort "#{$0}: #{diff} -D does not work" + end + warn "Avoiding diffutils 3.8 bug#61193" + proc {|macro, *inputs| + [diff] + [ + "--old-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#endif /* ! @ */\n", + + "--new-group-format=" \ + "#ifdef @\n" \ + "%>" \ + "#endif /* @ */\n", + + "--changed-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#else /* @ */\n" \ + "%>" \ + "#endif /* @ */\n" + ].map {|opt| opt.gsub(/@/) {macro}} + inputs + } + end + end + + ifdef = diff_args(diff || "diff") + NAME2CTYPE = %w[gperf -7 -c -j1 -i1 -t -C -P -T -H uniname2ctype_hash -Q uniname2ctype_pool -N uniname2ctype_p] fds = [] @@ -553,9 +615,8 @@ if header IO.popen([*NAME2CTYPE, out: tmp], "w") {|f| output.show(f, *syms)} end while syms.pop fds.each(&:close) - ff = nil - IO.popen(%W[diff -DUSE_UNICODE_AGE_PROPERTIES #{fds[1].path} #{fds[0].path}], "r") {|age| - IO.popen(%W[diff -DUSE_UNICODE_PROPERTIES #{fds[2].path} -], "r", in: age) {|f| + IO.popen(ifdef["USE_UNICODE_AGE_PROPERTIES", fds[1].path, fds[0].path], "r") {|age| + IO.popen(ifdef["USE_UNICODE_PROPERTIES", fds[2].path, "-"], "r", in: age) {|f| ansi = false f.each {|line| if /ANSI-C code produced by gperf/ =~ line @@ -564,7 +625,7 @@ if header line.sub!(/\/\*ANSI\*\//, '1') if ansi line.gsub!(/\(int\)\((?:long|size_t)\)&\(\(struct uniname2ctype_pool_t \*\)0\)->uniname2ctype_pool_(str\d+),\s+/, 'uniname2ctype_offset(\1), ') - if ff = (!ff ? /^(uniname2ctype_hash) /=~line : /^\}/!~line) # no line can match both, exclusive flip-flop + if line.start_with?("uniname2ctype_hash\s") ... line.start_with?("}") line.sub!(/^( *(?:register\s+)?(.*\S)\s+hval\s*=\s*)(?=len;)/, '\1(\2)') end puts line diff --git a/tool/extlibs.rb b/tool/extlibs.rb index b482258a2c..cef6712833 100755 --- a/tool/extlibs.rb +++ b/tool/extlibs.rb @@ -43,7 +43,7 @@ class ExtLibs end def do_download(url, cache_dir) - Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) + Downloader.download(url, nil, nil, nil, cache_dir: cache_dir) end def do_checksum(cache, chksums) @@ -185,7 +185,7 @@ class ExtLibs extracted = false dest = File.dirname(list) url = chksums = nil - IO.foreach(list) do |line| + File.foreach(list) do |line| line.sub!(/\s*#.*/, '') if /^(\w+)\s*=\s*(.*)/ =~ line vars[$1] = vars.expand($2) diff --git a/tool/fake.rb b/tool/fake.rb index 91dfb041c4..2c458985d8 100644 --- a/tool/fake.rb +++ b/tool/fake.rb @@ -9,6 +9,15 @@ class File end end +[[libpathenv, "."], [preloadenv, libruby_so]].each do |env, path| + env or next + e = ENV[env] or next + e = e.split(File::PATH_SEPARATOR) + path = File.realpath(path, builddir) rescue next + e.delete(path) or next + ENV[env] = (e.join(File::PATH_SEPARATOR) unless e.empty?) +end + static = !!(defined?($static) && $static) $:.unshift(builddir) posthook = proc do @@ -54,6 +63,8 @@ prehook = proc do |extmk| RbConfig.fire_update!("extout", $extout) RbConfig.fire_update!("rubyhdrdir", "$(top_srcdir)/include") RbConfig.fire_update!("rubyarchhdrdir", "$(extout)/include/$(arch)") + RbConfig.fire_update!("rubyarchdir", "$(extout)/$(arch)") + RbConfig.fire_update!("rubylibdir", "$(extout)/common") RbConfig.fire_update!("libdirname", "buildlibdir") trace_var(:$ruby, posthook) untrace_var(:$extmk, prehook) diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb index cb59b326ba..127ea236f3 100755 --- a/tool/fetch-bundled_gems.rb +++ b/tool/fetch-bundled_gems.rb @@ -1,6 +1,16 @@ -#!ruby -an +#!ruby -alnF\s+|#.* BEGIN { require 'fileutils' + require_relative 'lib/colorize' + + color = Colorize.new + + if ARGV.first.start_with?("BUNDLED_GEMS=") + bundled_gems = ARGV.shift[13..-1] + sep = bundled_gems.include?(",") ? "," : " " + bundled_gems = bundled_gems.split(sep) + bundled_gems = nil if bundled_gems.empty? + end dir = ARGV.shift ARGF.eof? @@ -11,22 +21,34 @@ BEGIN { n, v, u, r = $F next unless n -next if n =~ /^#/ +next if bundled_gems&.all? {|pat| !File.fnmatch?(pat, n)} -if File.directory?(n) - puts "updating #{n} ..." - system("git", "fetch", chdir: n) or abort -else - puts "retrieving #{n} ..." - system(*%W"git clone #{u} #{n}") or abort +unless File.exist?("#{n}/.git") + puts "retrieving #{color.notice(n)} ..." + system(*%W"git clone --depth=1 --no-tags #{u} #{n}") or abort end + if r - puts "fetching #{r} ..." + puts "fetching #{color.notice(r)} ..." system("git", "fetch", "origin", r, chdir: n) or abort + c = r +else + c = ["v#{v}", v].find do |c| + puts "fetching #{color.notice(c)} ..." + system("git", "fetch", "origin", "refs/tags/#{c}:refs/tags/#{c}", chdir: n) + end or abort end -c = r || "v#{v}" + checkout = %w"git -c advice.detachedHead=false checkout" -puts "checking out #{c} (v=#{v}, r=#{r}) ..." +info = %[, r=#{color.info(r)}] if r +puts "checking out #{color.notice(c)} (v=#{color.info(v)}#{info}) ..." unless system(*checkout, c, "--", chdir: n) abort if r or !system(*checkout, v, "--", chdir: n) end + +if r + unless File.exist? "#{n}/#{n}.gemspec" + require_relative "lib/bundled_gem" + BundledGem.dummy_gemspec("#{n}/#{n}.gemspec") + end +end diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb index 6200e78a56..2de8379606 100755 --- a/tool/file2lastrev.rb +++ b/tool/file2lastrev.rb @@ -98,6 +98,7 @@ ok = true data.sub!(/(?<!\A|\n)\z/, "\n") @output.write(data, overwrite: true, create_only: create_only) rescue => e + next if @suppress_not_found and VCS::NotFoundError === e warn "#{File.basename(Program)}: #{e.message}" ok = false end diff --git a/tool/format-release b/tool/format-release index 7b0f8e78e7..8bb6154243 100755 --- a/tool/format-release +++ b/tool/format-release @@ -1,8 +1,15 @@ #!/usr/bin/env ruby -# https://rubygems.org/gems/diffy -require "diffy" + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "diffy" +end + require "open-uri" require "yaml" +require_relative "./ruby-version" Diffy::Diff.default_options.merge!( include_diff_info: true, @@ -24,10 +31,9 @@ class Tarball def gz?; @url.end_with?('.gz'); end def zip?; @url.end_with?('.zip'); end - def bz2?; @url.end_with?('.bz2'); end def xz?; @url.end_with?('.xz'); end - def ext; @url[/(?:zip|tar\.(?:gz|bz2|xz))\z/]; end + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end def to_md <<eom @@ -49,25 +55,10 @@ eom unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" - info = YAML.load(URI(uri).read) + info = YAML.unsafe_load(URI(uri).read) if info.size != 1 raise "unexpected info.yml '#{uri}'" end @@ -82,10 +73,11 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" - stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) + stat = `git -C #{rubydir} diff -l0 --shortstat #{prev_tag}..#{tag}` files_changed, insertions, deletions = stat.scan(/\d+/) end @@ -178,7 +170,7 @@ eom if /\.0(?:-\w+)?\z/ =~ ver # preview, rc, or first release entry <<= <<eom - tag: v#{ver.tr('.-', '_')} + tag: #{RubyVersion.tag(ver)} stats: files_changed: #{files_changed} insertions: #{insertions} @@ -190,34 +182,25 @@ eom url: gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip - bz2: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.bz2 xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz size: gz: #{ary.find{|x|x.gz? }.size} zip: #{ary.find{|x|x.zip?}.size} - bz2: #{ary.find{|x|x.bz2?}&.size} xz: #{ary.find{|x|x.xz? }.size} sha1: gz: #{ary.find{|x|x.gz? }.sha1} zip: #{ary.find{|x|x.zip?}.sha1} - bz2: #{ary.find{|x|x.bz2?}&.sha1} xz: #{ary.find{|x|x.xz? }.sha1} sha256: gz: #{ary.find{|x|x.gz? }.sha256} zip: #{ary.find{|x|x.zip?}.sha256} - bz2: #{ary.find{|x|x.bz2?}&.sha256} xz: #{ary.find{|x|x.xz? }.sha256} sha512: gz: #{ary.find{|x|x.gz? }.sha512} zip: #{ary.find{|x|x.zip?}.sha512} - bz2: #{ary.find{|x|x.bz2?}&.sha512} xz: #{ary.find{|x|x.xz? }.sha512} eom - if ver.start_with?("3.") - entry = entry.gsub(/ bz2: .*\n/, "") - end - if data.include?("\n- version: #{ver}\n") # update existing entry data.sub!(/\n- version: #{ver}\n(^ .*\n)*\n/, "\n#{entry}\n") @@ -235,11 +218,11 @@ eom diff.each_with_index do |line, index| case index when 0 - line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do "--- a/#{filename}\t#{$2}" end when 1 - line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do "+++ b/#{filename}\t#{$2}" end end diff --git a/tool/gen-github-release.rb b/tool/gen-github-release.rb new file mode 100755 index 0000000000..cdb66080d9 --- /dev/null +++ b/tool/gen-github-release.rb @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby + +if ARGV.size < 2 + puts "Usage: #{$0} <from version tag> <to version tag> [--no-dry-run]" + puts " : if --no-dry-run is specified, it will create a release on GitHub" + exit 1 +end + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "octokit" + gem "faraday-retry" + gem "nokogiri" +end + +require "open-uri" + +Octokit.configure do |c| + c.access_token = ENV['GITHUB_TOKEN'] + c.auto_paginate = true + c.per_page = 100 +end + +client = Octokit::Client.new + +note = "## What's Changed\n\n" + +notes = [] + +diff = client.compare("ruby/ruby", ARGV[0], ARGV[1]) +diff[:commits].each do |c| + if c[:commit][:message] =~ /\[(Backport|Feature|Bug) #(\d*)\]/ + url = "https://bugs.ruby-lang.org/issues/#{$2}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ - Ruby master - Ruby Issue Tracking System/, "") + elsif c[:commit][:message] =~ /\(#(\d*)\)/ + url = "https://github.com/ruby/ruby/pull/#{$1}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ · ruby\/ruby · GitHub/, "") + else + next + end + notes << "* [#{title}](#{url})" +rescue OpenURI::HTTPError + puts "Error: #{url}" +end + +notes.uniq! + +note << notes.join("\n") + +note << "\n\n" +note << "Note: This list is automatically generated by tool/gen-github-release.rb. Because of this, some commits may be missing.\n\n" +note << "## Full Changelog\n\n" +note << "https://github.com/ruby/ruby/compare/#{ARGV[0]}...#{ARGV[1]}\n\n" + +if ARGV[2] == "--no-dry-run" + name = ARGV[1].gsub(/^v/, "").gsub(/_/, ".") + prerelease = ARGV[1].match?(/rc|preview/) ? true : false + client.create_release("ruby/ruby", ARGV[1], name: name, body: note, make_latest: "false", prerelease: prerelease) + puts "Created a release: https://github.com/ruby/ruby/releases/tag/#{ARGV[1]}" +else + puts note +end diff --git a/tool/generic_erb.rb b/tool/generic_erb.rb index 6607d5c256..b6cb51474e 100644 --- a/tool/generic_erb.rb +++ b/tool/generic_erb.rb @@ -12,7 +12,7 @@ source = false templates = [] ARGV.options do |o| - o.on('-i', '--input=PATH') {|v| template << v} + o.on('-i', '--input=PATH') {|v| templates << v} o.on('-x', '--source') {source = true} out.def_options(o) o.order!(ARGV) @@ -27,11 +27,7 @@ vpath = out.vpath output, vpath = output, vpath result = templates.map do |template| - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(File.read(template), trim_mode: '%-') - else - erb = ERB.new(File.read(template), nil, '%-') - end + erb = ERB.new(File.read(template), trim_mode: '%-') erb.filename = template source ? erb.src : proc{erb.result(binding)}.call end diff --git a/tool/gperf.sed b/tool/gperf.sed index 52915f80d5..9550a522b9 100644 --- a/tool/gperf.sed +++ b/tool/gperf.sed @@ -1,23 +1,4 @@ -/ANSI-C code/{ - h - s/.*/ANSI:offset:/ - x -} -/\/\*!ANSI{\*\//{ - G - s/\/\*!ANSI{\*\/\(.*\)\/\*}!ANSI\*\/\(.*\)\nANSI:.*/\/\*\1\*\/\2/ -} -s/(int)([a-z_]*)&((struct \([a-zA-Z_0-9][a-zA-Z_0-9]*\)_t *\*)0)->\1_str\([1-9][0-9]*\),/gperf_offsetof(\1, \2),/g -/^#line/{ - G - x - s/:offset:/:/ - x - s/\(.*\)\(\n\).*:offset:.*/#define gperf_offsetof(s, n) (short)offsetof(struct s##_t, s##_str##n)\2\1/ - s/\n[^#].*// -} /^[a-zA-Z_0-9]*hash/,/^}/{ s/ hval = / hval = (unsigned int)/ s/ return / return (unsigned int)/ } -s/^\(static\) \(unsigned char gperf_downcase\)/\1 const \2/ diff --git a/tool/ifchange b/tool/ifchange index 5af41e0156..9e4a89533c 100755 --- a/tool/ifchange +++ b/tool/ifchange @@ -18,7 +18,7 @@ HELP set -e timestamp= keepsuffix= -empty= +srcavail=f color=auto until [ $# -eq 0 ]; do case "$1" in @@ -39,7 +39,7 @@ until [ $# -eq 0 ]; do keepsuffix=`expr \( "$1" : '[^=]*=\(.*\)' \)` ;; --empty) - empty=yes + srcavail=s ;; --color) color=always @@ -97,7 +97,7 @@ fi targetdir= case "$target" in */*) targetdir=`dirname "$target"`;; esac -if [ -f "$target" -a ! -${empty:+f}${empty:-s} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then +if [ -f "$target" -a ! -${srcavail} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then echo "$target ${msg_unchanged}unchanged${msg_reset}" rm -f "$temp" else diff --git a/tool/leaked-globals b/tool/leaked-globals index d95f3794e8..6118cd56e8 100755 --- a/tool/leaked-globals +++ b/tool/leaked-globals @@ -1,23 +1,33 @@ #!/usr/bin/ruby require_relative 'lib/colorize' +require 'shellwords' until ARGV.empty? case ARGV[0] - when /\ASYMBOL_PREFIX=(.*)/ + when /\A SYMBOL_PREFIX=(.*)/x SYMBOL_PREFIX = $1 - when /\ANM=(.*)/ # may be multiple words - NM = $1 - when /\APLATFORM=(.+)?/ + when /\A NM=(.*)/x # may be multiple words + NM = $1.shellsplit + when /\A PLATFORM=(.+)?/x platform = $1 + when /\A SOEXT=(.+)?/x + soext = $1 + when /\A SYMBOLS_IN_EMPTYLIB=(.*)/x + SYMBOLS_IN_EMPTYLIB = $1.split(" ") + when /\A EXTSTATIC=(.+)?/x + EXTSTATIC = true else break end ARGV.shift end +SYMBOLS_IN_EMPTYLIB ||= nil +EXTSTATIC ||= false config = ARGV.shift count = 0 col = Colorize.new + config_code = File.read(config) REPLACE = config_code.scan(/\bAC_(?:REPLACE|CHECK)_FUNCS?\((\w+)/).flatten # REPLACE << 'memcmp' if /\bAC_FUNC_MEMCMP\b/ =~ config_code @@ -29,27 +39,68 @@ if platform and !platform.empty? else REPLACE.concat( h .gsub(%r[/\*.*?\*/]m, " ") # delete block comments - .gsub(%r[//.*], ' ') # delete oneline comments + .gsub(%r[//.*], " ") # delete oneline comments .gsub(/^\s*#.*(?:\\\n.*)*/, "") # delete preprocessor directives + .gsub(/(?:\A|;)\K\s*typedef\s.*?;/m, "") .scan(/\b((?!rb_|DEPRECATED|_)\w+)\s*\(.*\);/) .flatten) end end missing = File.dirname(config) + "/missing/" ARGV.reject! do |n| - unless (src = Dir.glob(missing + File.basename(n, ".*") + ".[cS]")).empty? - puts "Ignore #{n} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" + base = File.basename(n, ".*") + next true if REPLACE.include?(base) + unless (src = Dir.glob(missing + base + ".[cS]")).empty? + puts "Ignore #{col.skip(n)} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" true end end + +# darwin's ld64 seems to require exception handling personality functions to be +# extern, so we allow the Rust one. +REPLACE.push("rust_eh_personality") if RUBY_PLATFORM.include?("darwin") + print "Checking leaked global symbols..." STDOUT.flush -IO.foreach("|#{NM} -Pgp #{ARGV.join(' ')}") do |line| +if soext + soext = /\.#{soext}(?:$|\.)/ + if EXTSTATIC + ARGV.delete_if {|n| soext =~ n} + elsif ARGV.size == 1 + so = soext =~ ARGV.first + end +end + +Pipe = Struct.new(:command) do + def open(&block) IO.popen(command, &block) end + def each(&block) open {|f| f.each(&block)} end +end + +Pipe.new(NM + ARGV).each do |line| + line.chomp! + next so = nil if line.empty? + if so.nil? and line.chomp!(":") + so = soext =~ line || false + next + end n, t, = line.split next unless /[A-TV-Z]/ =~ t next unless n.sub!(/^#{SYMBOL_PREFIX}/o, "") next if n.include?(".") - next if /\A(?:Init_|InitVM_|RUBY_|ruby_|rb_|[Oo]nig|dln_|mjit_|coroutine_)/ =~ n + next if !so and n.start_with?("___asan_") + next if !so and n.start_with?("__odr_asan_") + next if !so and n.start_with?("__retguard_") + next if !so and n.start_with?("__dtrace") + case n + when /\A(?:Init_|InitVM_|pm_|[Oo]nig|dln_|coroutine_)/ + next + when /\Aruby_static_id_/ + next unless so + when /\A(?:RUBY_|ruby_|rb_)/ + next unless so and /_(threadptr|ec)_/ =~ n + when *SYMBOLS_IN_EMPTYLIB + next + end next if REPLACE.include?(n) puts col.fail("leaked") if count.zero? count += 1 diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb new file mode 100644 index 0000000000..daa1a1f235 --- /dev/null +++ b/tool/lib/_tmpdir.rb @@ -0,0 +1,100 @@ +template = "rubytest." + +# This path is only for tests. +# Assume the directory by these environment variables are safe. +base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| + next unless tmp and tmp.size <= 50 and File.directory?(tmp) + # On macOS, the default TMPDIR is very long, in spite of UNIX socket + # path length being limited. + # + # Also Rubygems creates its own temporary directory per tests, and + # some tests copy the full path of gemhome there. In that case, the + # path contains both temporary names twice, and can exceed path name + # limit very easily. + tmp +end +begin + tmpdir = File.join(base, template + Random.new_seed.to_s(36)[-6..-1]) + Dir.mkdir(tmpdir, 0o700) +rescue Errno::EEXIST + retry +end +# warn "tmpdir(#{tmpdir.size}) = #{tmpdir}" + +pid = $$ +END { + if pid == $$ + begin + Dir.rmdir(tmpdir) + rescue Errno::ENOENT + rescue Errno::ENOTEMPTY + require_relative "colorize" + colorize = Colorize.new + ls = Struct.new(:colorize) do + def mode_inspect(m, s) + [ + (m & 0o4 == 0 ? ?- : ?r), + (m & 0o2 == 0 ? ?- : ?w), + (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)), + ] + end + def decorate_path(path, st) + case + when st.directory? + color = "bold;blue" + type = "/" + when st.symlink? + color = "bold;cyan" + # type = "@" + when st.executable? + color = "bold;green" + type = "*" + when path.end_with?(".gem") + color = "green" + end + colorize.decorate(path, color) + (type || "") + end + def list_tree(parent, indent = "", &block) + children = Dir.children(parent).map do |child| + [child, path = File.join(parent, child), File.lstat(path)] + end + nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size + size_width = children.map {|child, path, st| st.size}.max.to_s.size + + children.each do |child, path, st| + m = st.mode + m = [ + (st.file? ? ?- : st.ftype[0]), + mode_inspect(m >> 6, (?s unless m & 04000 == 0)), + mode_inspect(m >> 3, (?s unless m & 02000 == 0)), + mode_inspect(m, (?t unless m & 01000 == 0)), + ].join("") + warn sprintf("%s* %s %*d %*d %s % s%s", + indent, m, nlink_width, st.nlink, size_width, st.size, + st.mtime.to_s, decorate_path(child, st), + (" -> " + decorate_path(File.readlink(path), File.stat(path)) if + st.symlink?)) + if st.directory? + list_tree(File.join(parent, child), indent + " ", &block) + end + yield path, st if block + end + end + end.new(colorize) + warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":" + Dir.chdir(tmpdir) do + ls.list_tree(".") do |path, st| + if st.directory? + Dir.rmdir(path) + else + File.unlink(path) + end + end + end + require "fileutils" + FileUtils.rm_rf(tmpdir) + end + end +} + +ENV["TMPDIR"] = ENV["SPEC_TEMP_DIR"] = ENV["GEM_TEST_TMPDIR"] = tmpdir diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb new file mode 100644 index 0000000000..9ad5ea220b --- /dev/null +++ b/tool/lib/bundle_env.rb @@ -0,0 +1,4 @@ +ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_PATH__SYSTEM"] = "true" +ENV["BUNDLE_WITHOUT"] = "lint doc" diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index 38c331183d..d2ed61a508 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -6,12 +6,45 @@ require 'rubygems/package' # unpack bundled gem files. module BundledGem + DEFAULT_GEMS_DEPENDENCIES = [ + "net-protocol", # net-ftp + "time", # net-ftp + "singleton", # prime + "ipaddr", # rinda + "forwardable", # prime, rinda + "strscan", # rexml + "psych" # rdoc + ] + module_function def unpack(file, *rest) pkg = Gem::Package.new(file) - prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)} + prepare_test(pkg.spec, *rest) do |dir| + pkg.extract_files(dir) + FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)}) + end puts "Unpacked #{file}" + rescue Gem::Package::FormatError, Errno::ENOENT + puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." + if file =~ /^gems\/(\w+)-/ + file = Dir.glob("gems/#{$1}-*.gem").first + end + retry + end + + def build(gemspec, version, outdir = ".", validation: true) + outdir = File.expand_path(outdir) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.load(gemfile) + abort "Failed to load #{gemspec}" unless spec + output = File.join(outdir, spec.file_name) + FileUtils.rm_rf(output) + package = Gem::Package.new(output) + package.spec = spec + package.build(validation == false) + end end def copy(path, *rest) @@ -35,6 +68,9 @@ module BundledGem gem_dir = File.join(dir, "gems", target) yield gem_dir spec_dir = spec.extensions.empty? ? "specifications" : File.join("gems", target) + if spec.extensions.empty? + spec.dependencies.reject! {|dep| DEFAULT_GEMS_DEPENDENCIES.include?(dep.name)} + end File.binwrite(File.join(dir, spec_dir, "#{target}.gemspec"), spec.to_ruby) unless spec.extensions.empty? spec.dependencies.clear @@ -52,4 +88,39 @@ module BundledGem end FileUtils.rm_rf(Dir.glob("#{gem_dir}/.git*")) end + + def dummy_gemspec(gemspec) + return if File.exist?(gemspec) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.new do |s| + s.name = gemfile.chomp(".gemspec") + s.version = + File.read("lib/#{s.name}.rb")[/VERSION = "(.+?)"/, 1] || + begin File.read("lib/#{s.name}/version.rb")[/VERSION = "(.+?)"/, 1]; rescue; nil; end || + raise("cannot find the version of #{ s.name } gem") + s.authors = ["DUMMY"] + s.email = ["dummy@ruby-lang.org"] + s.files = Dir.glob("{lib,ext}/**/*").select {|f| File.file?(f)} + s.licenses = ["Ruby"] + s.description = "DO NOT USE; dummy gemspec only for test" + s.summary = "(dummy gemspec)" + end + File.write(gemfile, spec.to_ruby) + end + end + + def checkout(gemdir, repo, rev, git: $git) + return unless rev or !git or git.empty? + unless File.exist?("#{gemdir}/.git") + puts "Cloning #{repo}" + command = "#{git} clone #{repo} #{gemdir}" + system(command) or raise "failed: #{command}" + end + puts "Update #{File.basename(gemdir)} to #{rev}" + command = "#{git} fetch origin #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + command = "#{git} checkout --detach #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + end end diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb index 8fb90e1833..0904312119 100644 --- a/tool/lib/colorize.rb +++ b/tool/lib/colorize.rb @@ -7,7 +7,7 @@ class Colorize def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color)) @colors = @reset = nil @color = opts && opts[:color] || color - if color or (color == nil && STDOUT.tty? && (ENV["NO_COLOR"] || "").empty?) + if color or (color == nil && coloring?) if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil) @beg = "\e[" colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} @@ -27,23 +27,48 @@ class Colorize end DEFAULTS = { - "pass"=>"32", "fail"=>"31;1", "skip"=>"33;1", + # color names "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", "bold"=>"1", "underline"=>"4", "reverse"=>"7", + "bright_black"=>"90", "bright_red"=>"91", "bright_green"=>"92", "bright_yellow"=>"93", + "bright_blue"=>"94", "bright_magenta"=>"95", "bright_cyan"=>"96", "bright_white"=>"97", + + # abstract decorations + "pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold", + "note"=>"bright_yellow", "notice"=>"bright_yellow", "info"=>"bright_magenta", } - NO_COLOR = (nc = ENV['NO_COLOR']) && !nc.empty? + def coloring? + STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?) + end # colorize.decorate(str, name = color_name) def decorate(str, name = @color) - if !NO_COLOR and @colors and color = (@colors[name] || DEFAULTS[name]) + if coloring? and color = resolve_color(name) "#{@beg}#{color}m#{str}#{@reset}" else str end end + def resolve_color(color = @color, seen = {}, colors = nil) + return unless @colors + color.to_s.gsub(/\b[a-z][\w ]+/) do |n| + n.gsub!(/\W+/, "_") + n.downcase! + c = seen[n] and next c + if colors + c = colors[n] + elsif (c = (tbl = @colors)[n] || (tbl = DEFAULTS)[n]) + colors = tbl + else + next n + end + seen[n] = resolve_color(c, seen, colors) + end + end + DEFAULTS.each_key do |name| define_method(name) {|str| decorate(str, name) diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index fdae050bb5..e29a0e3c25 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -1,6 +1,43 @@ # frozen_string_literal: true module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + module Unit module Assertions def assert_raises(*exp, &b) @@ -37,6 +74,20 @@ module Test module CoreAssertions require_relative 'envutil' require 'pp' + begin + require '-test-/sanitizers' + rescue LoadError + # in test-unit-ruby-core gem + def sanitizers + nil + end + else + def sanitizers + Test::Sanitizers + end + end + module_function :sanitizers + nil.pretty_inspect def mu_pp(obj) #:nodoc: @@ -55,11 +106,14 @@ module Test end def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, - success: nil, **opt) + success: nil, failed: nil, gems: false, **opt) args = Array(args).dup - args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') + unless gems.nil? + args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems") + end stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) - desc = FailDesc[status, message, stderr] + desc = failed[status, message, stderr] if failed + desc ||= FailDesc[status, message, stderr] if block_given? raise "test_stdout ignored, use block only or without block" if test_stdout != [] raise "test_stderr ignored, use block only or without block" if test_stderr != [] @@ -111,8 +165,13 @@ module Test end def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) - # TODO: consider choosing some appropriate limit for MJIT and stop skipping this once it does not randomly fail + # TODO: consider choosing some appropriate limit for RJIT and stop skipping this once it does not randomly fail + pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + # For previous versions which implemented MJIT pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + # ASAN has the same problem - its shadow memory greatly increases memory usage + # (plus asan has better ways to detect memory leaks than this assertion) + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled? require_relative 'memory_status' raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) @@ -244,9 +303,34 @@ module Test def separated_runner(token, out = nil) include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + out = out ? IO.new(out, 'w') : STDOUT + + # avoid method redefinitions + out_write = out.method(:write) + integer_to_s = Integer.instance_method(:to_s) + array_pack = Array.instance_method(:pack) + marshal_dump = Marshal.method(:dump) + assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set) + assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get) + Test::Unit::Assertions.module_eval do + @_assertions = 0 + + undef _assertions= + define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)}) + + undef _assertions + define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)}) + end + # assume Method#call and UnboundMethod#bind_call need to work as the original + at_exit { - out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" + assertions = assertions_ivar_get.call(:@_assertions) + out_write.call <<~OUT + <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}> + #{array_pack.bind_call([marshal_dump.call($!)], 'm0')} + </error id="#{token}"> + OUT } if defined?(Test::Unit::Runner) Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) @@ -280,7 +364,16 @@ eom args = args.dup args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + # power_assert 3 requires ruby 3.1 or later + args << "-W:no-experimental" if RUBY_VERSION < "3.1." stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + + if sanitizers&.lsan_enabled? + # LSAN may output messages like the following line into stderr. We should ignore it. + # ==276855==Running thread 276851 was not suspended. False leaks are possible. + # See https://github.com/google/sanitizers/issues/1479 + stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "") + end ensure if res_c res_c.close @@ -291,15 +384,16 @@ eom end raise if $! abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + marshal_error = nil assert(!abort, FailDesc[status, nil, stderr]) - self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i - begin - res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m")) + res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do + self._assertions += $1.to_i + res = Marshal.load($2.unpack1("m")) or next rescue => marshal_error ignore_stderr = nil res = nil - end - if res and !(SystemExit === res) + else + next if SystemExit === res if bt = res.backtrace bt.each do |l| l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} @@ -311,7 +405,7 @@ eom raise res end - # really is it succeed? + # really did it succeed? unless ignore_stderr # the body of assert_separately must not output anything to detect error assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) @@ -322,9 +416,17 @@ eom # Run Ractor-related test without influencing the main test suite def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) - return unless defined?(Ractor) + omit unless defined?(Ractor) + + # https://bugs.ruby-lang.org/issues/21262 + shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value) + shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join) + + if require + require = [require] unless require.is_a?(Array) + require = require.map {|r| "require #{r.inspect}"}.join("\n") + end - require = "require #{require.inspect}" if require if require_relative dir = File.dirname(caller_locations[0,1][0].absolute_path) full_path = File.expand_path(require_relative, dir) @@ -332,6 +434,8 @@ eom end assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{shim_value} + #{shim_join} #{require} previous_verbose = $VERBOSE $VERBOSE = nil @@ -442,19 +546,15 @@ eom case expected when String assert = :assert_equal - when Regexp - assert = :assert_match else - raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" + assert_respond_to(expected, :===) + assert = :assert_match end - ex = m = nil - EnvUtil.with_default_internal(expected.encoding) do - ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do - yield - end - m = ex.message + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield end + m = ex.message msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} if assert == :assert_equal @@ -467,6 +567,43 @@ eom ex end + # :call-seq: + # assert_raise_kind_of(*args, &block) + # + #Tests if the given block raises one of the given exceptions or + #sub exceptions of the given exceptions. If the last argument + #is a String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise SystemCallErr do + # Dir.chdir(__FILE__) #Raises Errno::ENOTDIR, so assertion succeeds + # end + def assert_raise_kind_of(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + raise e unless exp.include? Test::Unit::PendedError + rescue *exp => e + pass + rescue Exception => e + flunk(message(msg) {"#{mu_pp(exp)} family exception expected, not #{mu_pp(e)}"}) + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} family expected but nothing was raised"}) + end + end + e + end + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: # :call-seq: @@ -539,11 +676,11 @@ eom refute_respond_to(obj, meth, msg) end - # pattern_list is an array which contains regexp and :*. + # pattern_list is an array which contains regexp, string and :*. # :* means any sequence. # # pattern_list is anchored. - # Use [:*, regexp, :*] for non-anchored match. + # Use [:*, regexp/string, :*] for non-anchored match. def assert_pattern_list(pattern_list, actual, message=nil) rest = actual anchored = true @@ -586,7 +723,7 @@ eom def assert_warning(pat, msg = nil) result = nil - stderr = EnvUtil.with_default_internal(pat.encoding) { + stderr = EnvUtil.with_default_internal(of: pat) { EnvUtil.verbose_warning { result = yield } @@ -600,17 +737,15 @@ eom assert_warning(*args) {$VERBOSE = false; yield} end - def assert_deprecated_warning(mesg = /deprecated/) + def assert_deprecated_warning(mesg = /deprecated/, &block) assert_warning(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end - def assert_deprecated_warn(mesg = /deprecated/) + def assert_deprecated_warn(mesg = /deprecated/, &block) assert_warn(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end @@ -701,7 +836,7 @@ eom msg = "exceptions on #{errs.length} threads:\n" + errs.map {|t, err| "#{t.inspect}:\n" + - err.full_message(highlight: false, order: :top) + (err.respond_to?(:full_message) ? err.full_message(highlight: false, order: :top) : err.message) }.join("\n---\n") if message msg = "#{message}\n#{msg}" @@ -736,6 +871,67 @@ eom end alias all_assertions_foreach assert_all_assertions_foreach + %w[ + CLOCK_THREAD_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID + CLOCK_MONOTONIC + ].find do |c| + if Process.const_defined?(c) + [c.to_sym, Process.const_get(c)].find do |clk| + begin + Process.clock_gettime(clk) + rescue + # Constants may be defined but not implemented, e.g., mingw. + else + unless Process.clock_getres(clk) < 1.0e-03 + next # needs msec precision + end + PERFORMANCE_CLOCK = clk + end + end + end + end + + # Expect +seq+ to respond to +first+ and +each+ methods, e.g., + # Array, Range, Enumerator::ArithmeticSequence and other + # Enumerable-s, and each elements should be size factors. + # + # :yield: each elements of +seq+. + def assert_linear_performance(seq, rehearsal: nil, pre: ->(n) {n}) + pend "No PERFORMANCE_CLOCK found" unless defined?(PERFORMANCE_CLOCK) + + # Timeout testing generally doesn't work when RJIT compilation happens. + rjit_enabled = defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + measure = proc do |arg, message| + st = Process.clock_gettime(PERFORMANCE_CLOCK) + yield(*arg) + t = (Process.clock_gettime(PERFORMANCE_CLOCK) - st) + assert_operator 0, :<=, t, message unless rjit_enabled + t + end + + first = seq.first + *arg = pre.call(first) + times = (0..(rehearsal || (2 * first))).map do + measure[arg, "rehearsal"].nonzero? + end + times.compact! + tmin, tmax = times.minmax + + # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) + tbase = 10 * tmax * [(tmax / tmin) ** 2 / 4, 1].max + info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" + + seq.each do |i| + next if i == first + t = tbase * i.fdiv(first) + *arg = pre.call(i) + message = "[#{i}]: in #{t}s #{info}" + Timeout.timeout(t, Timeout::Error, message) do + measure[arg, message] + end + end + end + def diff(exp, act) require 'pp' q = PP.new(+"") @@ -756,6 +952,82 @@ eom token = "\e[7;1m#{$$.to_s}:#{Time.now.strftime('%s.%L')}:#{rand(0x10000).to_s(16)}:\e[m" return token.dump, Regexp.quote(token) end + + # Platform predicates + + def self.mswin? + defined?(@mswin) ? @mswin : @mswin = RUBY_PLATFORM.include?('mswin') + end + private def mswin? + CoreAssertions.mswin? + end + + def self.mingw? + defined?(@mingw) ? @mingw : @mingw = RUBY_PLATFORM.include?('mingw') + end + private def mingw? + CoreAssertions.mingw? + end + + module_function def windows? + mswin? or mingw? + end + + def self.version_compare(expected, actual) + expected.zip(actual).each {|e, a| z = (e <=> a); return z if z.nonzero?} + 0 + end + + def self.version_match?(expected, actual) + if !actual + false + elsif expected.empty? + true + elsif expected.size == 1 and Range === (range = expected.first) + b, e = range.begin, range.end + return false if b and (c = version_compare(Array(b), actual)) > 0 + return false if e and (c = version_compare(Array(e), actual)) < 0 + return false if e and range.exclude_end? and c == 0 + true + else + version_compare(expected, actual).zero? + end + end + + def self.linux?(*ver) + unless defined?(@linux) + @linux = RUBY_PLATFORM.include?('linux') && `uname -r`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @linux + end + private def linux?(*ver) + CoreAssertions.linux?(*ver) + end + + def self.glibc?(*ver) + unless defined?(@glibc) + libc = `/usr/bin/ldd /bin/sh`[/^\s*libc.*=> *\K\S*/] + if libc and /version (\d+)\.(\d+)\.$/ =~ IO.popen([libc], &:read)[] + @glibc = [$1.to_i, $2.to_i] + else + @glibc = false + end + end + version_match? ver, @glibc + end + private def glibc?(*ver) + CoreAssertions.glibc?(*ver) + end + + def self.macos?(*ver) + unless defined?(@macos) + @macos = RUBY_PLATFORM.include?('darwin') && `sw_vers -productVersion`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @macos + end + private def macos?(*ver) + CoreAssertions.macos?(*ver) + end end end end diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb new file mode 100644 index 0000000000..56b420a546 --- /dev/null +++ b/tool/lib/dump.gdb @@ -0,0 +1,17 @@ +set height 0 +set width 0 +set confirm off + +echo \n>>> Threads\n\n +info threads + +echo \n>>> Machine level backtrace\n\n +thread apply all info stack full + +echo \n>>> Dump Ruby level backtrace (if possible)\n\n +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +echo ">>> Finish\n" +detach +quit diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb new file mode 100644 index 0000000000..ed9cb89010 --- /dev/null +++ b/tool/lib/dump.lldb @@ -0,0 +1,13 @@ +script print("\n>>> Threads\n\n") +thread list + +script print("\n>>> Machine level backtrace\n\n") +thread backtrace all + +script print("\n>>> Dump Ruby level backtrace (if possible)\n\n") +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +script print(">>> Finish\n") +detach +quit diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index 728ca7059b..6089605056 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -15,23 +15,22 @@ end module EnvUtil def rubybin if ruby = ENV["RUBY"] - return ruby - end - ruby = "ruby" - exeext = RbConfig::CONFIG["EXEEXT"] - rubyexe = (ruby + exeext if exeext and !exeext.empty?) - 3.times do - if File.exist? ruby and File.executable? ruby and !File.directory? ruby - return File.expand_path(ruby) - end - if rubyexe and File.exist? rubyexe and File.executable? rubyexe - return File.expand_path(rubyexe) - end - ruby = File.join("..", ruby) - end - if defined?(RbConfig.ruby) + ruby + elsif defined?(RbConfig.ruby) RbConfig.ruby else + ruby = "ruby" + exeext = RbConfig::CONFIG["EXEEXT"] + rubyexe = (ruby + exeext if exeext and !exeext.empty?) + 3.times do + if File.exist? ruby and File.executable? ruby and !File.directory? ruby + return File.expand_path(ruby) + end + if rubyexe and File.exist? rubyexe and File.executable? rubyexe + return File.expand_path(rubyexe) + end + ruby = File.join("..", ruby) + end "ruby" end end @@ -53,7 +52,14 @@ module EnvUtil @original_internal_encoding = Encoding.default_internal @original_external_encoding = Encoding.default_external @original_verbose = $VERBOSE - @original_warning = defined?(Warning.[]) ? %i[deprecated experimental].to_h {|i| [i, Warning[i]]} : nil + @original_warning = + if defined?(Warning.categories) + Warning.categories.to_h {|i| [i, Warning[i]]} + elsif defined?(Warning.[]) # 2.7+ + %i[deprecated experimental performance].to_h do |i| + [i, begin Warning[i]; rescue ArgumentError; end] + end.compact + end end end @@ -73,6 +79,72 @@ module EnvUtil end module_function :timeout + class Debugger + @list = [] + + attr_accessor :name + + def self.register(name, &block) + @list << new(name, &block) + end + + def initialize(name, &block) + @name = name + instance_eval(&block) + end + + def usable?; false; end + + def start(pid, *args) end + + def dump(pid, timeout: 60, reprieve: timeout&.div(4)) + dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err) + rescue Errno::ENOENT + return + else + return unless dpid + [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig| + begin + return EnvUtil.timeout(t) {Process.wait(dpid)} + rescue Timeout::Error + Process.kill(sig, dpid) + end + end + true + end + + # sudo -n: --non-interactive + PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM) + + def spawn(*args, **opts) + super(*PRECOMMAND, *args, **opts) + end + + register("gdb") do + class << self + def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end + def start(pid, *args, **opts) + spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts) + end + def command_file(file) "--command=#{file}"; end + end + end + + register("lldb") do + class << self + def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end + def start(pid, *args, **opts) + spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts) + end + def command_file(file) ["--source", file]; end + end + end + + def self.search + @debugger ||= @list.find(&:usable?) + end + end + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) reprieve = apply_timeout_scale(reprieve) if reprieve @@ -88,17 +160,12 @@ module EnvUtil pgroup = pid end - lldb = true if /darwin/ =~ RUBY_PLATFORM - + dumped = false while signal = signals.shift - if lldb and [:ABRT, :KILL].include?(signal) - lldb = false - # sudo -n: --non-interactive - # lldb -p: attach - # -o: run command - system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit]) - true + if !dumped and [:ABRT, :KILL].include?(signal) + Debugger.search&.dump(pid) + dumped = true end begin @@ -155,10 +222,15 @@ module EnvUtil # remain env %w(ASAN_OPTIONS RUBY_ON_BUG).each{|name| - child_env[name] = ENV[name] if ENV[name] + child_env[name] = ENV[name] if !child_env.key?(name) and ENV.key?(name) } args = [args] if args.kind_of?(String) + # use the same parser as current ruby + if (args.none? { |arg| arg.start_with?("--parser=") } and + /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read)) + args = ["--parser=#{current_parser}"] + args + end pid = spawn(child_env, *precommand, rubybin, *args, opt) in_c.close out_c&.close @@ -206,6 +278,12 @@ module EnvUtil end module_function :invoke_ruby + def current_parser + features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}] + features&.split&.include?("+PRISM") ? "prism" : "parse.y" + end + module_function :current_parser + def verbose_warning class << (stderr = "".dup) alias write concat @@ -222,6 +300,21 @@ module EnvUtil end module_function :verbose_warning + if defined?(Warning.[]=) + def deprecation_warning + previous_deprecated = Warning[:deprecated] + Warning[:deprecated] = true + yield + ensure + Warning[:deprecated] = previous_deprecated + end + else + def deprecation_warning + yield + end + end + module_function :deprecation_warning + def default_warning $VERBOSE = false yield @@ -246,7 +339,30 @@ module EnvUtil end module_function :under_gc_stress - def with_default_external(enc) + def under_gc_compact_stress(val = :empty, &block) + raise "compaction doesn't work well on s390x. Omit the test in the caller." if RUBY_PLATFORM =~ /s390x/ # https://github.com/ruby/ruby/pull/5077 + + if GC.respond_to?(:auto_compact) + auto_compact = GC.auto_compact + GC.auto_compact = val + end + + under_gc_stress(&block) + ensure + GC.auto_compact = auto_compact if GC.respond_to?(:auto_compact) + end + module_function :under_gc_compact_stress + + def without_gc + prev_disabled = GC.disable + yield + ensure + GC.enable unless prev_disabled + end + module_function :without_gc + + def with_default_external(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_external = enc } yield ensure @@ -254,7 +370,8 @@ module EnvUtil end module_function :with_default_external - def with_default_internal(enc) + def with_default_internal(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_internal = enc } yield ensure diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb new file mode 100644 index 0000000000..1893e07657 --- /dev/null +++ b/tool/lib/gem_env.rb @@ -0,0 +1 @@ +ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__) diff --git a/tool/lib/iseq_loader_checker.rb b/tool/lib/iseq_loader_checker.rb index 3f07b3a999..73784f8450 100644 --- a/tool/lib/iseq_loader_checker.rb +++ b/tool/lib/iseq_loader_checker.rb @@ -76,6 +76,15 @@ class RubyVM::InstructionSequence # return value i2_bin if CHECK_TO_BINARY end if CHECK_TO_A || CHECK_TO_BINARY + + if opt == "prism" + # If RUBY_ISEQ_DUMP_DEBUG is "prism", we'll set up + # InstructionSequence.load_iseq to intercept loading filepaths to compile + # using prism. + def self.load_iseq(filepath) + RubyVM::InstructionSequence.compile_file_prism(filepath) + end + end end #require_relative 'x'; exit(1) diff --git a/tool/lib/launchable.rb b/tool/lib/launchable.rb new file mode 100644 index 0000000000..38f4fe92b3 --- /dev/null +++ b/tool/lib/launchable.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +require 'json' +require 'uri' + +module Launchable + ## + # JsonStreamWriter writes a JSON file using a stream. + # By utilizing a stream, we can minimize memory usage, especially for large files. + class JsonStreamWriter + def initialize(path) + @file = File.open(path, "w") + @file.write("{") + @indent_level = 0 + @is_first_key_val = true + @is_first_obj = true + write_new_line + end + + def write_object obj + if @is_first_obj + @is_first_obj = false + else + write_comma + write_new_line + end + @indent_level += 1 + @file.write(to_json_str(obj)) + @indent_level -= 1 + @is_first_key_val = true + # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified. + # { + # "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds", + # "status": "TEST_PASSED", + # "duration": 2.7e-05, + # "createdAt": "2024-02-09 12:21:07 +0000", + # "stderr": null, + # "stdout": null + # }: null <- here + # }, + # To prevent this, IO#flush is called here. + @file.flush + end + + def write_array(key) + @indent_level += 1 + @file.write(to_json_str(key)) + write_colon + @file.write(" ", "[") + write_new_line + end + + def close + return if @file.closed? + close_array + @indent_level -= 1 + write_new_line + @file.write("}", "\n") + @file.flush + @file.close + end + + private + def to_json_str(obj) + json = JSON.pretty_generate(obj) + json.gsub(/^/, ' ' * (2 * @indent_level)) + end + + def write_indent + @file.write(" " * 2 * @indent_level) + end + + def write_new_line + @file.write("\n") + end + + def write_comma + @file.write(',') + end + + def write_colon + @file.write(":") + end + + def close_array + write_new_line + write_indent + @file.write("]") + @indent_level -= 1 + end + end +end diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb index 1966145588..69df9a64b8 100644 --- a/tool/lib/leakchecker.rb +++ b/tool/lib/leakchecker.rb @@ -112,7 +112,7 @@ class LeakChecker } unless fd_leaked.empty? unless @@try_lsof == false - @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + @@try_lsof |= system(*%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) end end h.each {|fd, list| @@ -136,14 +136,14 @@ class LeakChecker attr_accessor :count end - def new(data) + def new(...) LeakChecker::TempfileCounter.count += 1 - super(data) + super end } LeakChecker.const_set(:TempfileCounter, m) - class << Tempfile::Remover + class << Tempfile prepend LeakChecker::TempfileCounter end end @@ -155,8 +155,8 @@ class LeakChecker if prev_count == count [prev_count, []] else - tempfiles = ObjectSpace.each_object(Tempfile).find_all {|t| - t.instance_variable_defined?(:@tmpfile) and t.path + tempfiles = ObjectSpace.each_object(Tempfile).reject {|t| + t.instance_variables.empty? || t.closed? } [count, tempfiles] end @@ -233,9 +233,12 @@ class LeakChecker old_env = @env_info new_env = find_env return false if old_env == new_env + if defined?(Bundler::EnvironmentPreserver) + bundler_prefix = Bundler::EnvironmentPreserver::BUNDLER_PREFIX + end (old_env.keys | new_env.keys).sort.each {|k| # Don't report changed environment variables caused by Bundler's backups - next if k.start_with?(Bundler::EnvironmentPreserver::BUNDLER_PREFIX) + next if bundler_prefix and k.start_with?(bundler_prefix) if old_env.has_key?(k) if new_env.has_key?(k) diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb index 5e9e80a68a..429e5f6a1d 100644 --- a/tool/lib/memory_status.rb +++ b/tool/lib/memory_status.rb @@ -12,7 +12,7 @@ module Memory PROC_FILE = procfile VM_PAT = pat def self.read_status - IO.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| + File.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| yield($1.downcase.intern, $2.to_i * 1024) if VM_PAT =~ l end end @@ -20,48 +20,68 @@ module Memory data.scan(pat) {|k, v| keys << k.downcase.intern} when /mswin|mingw/ =~ RUBY_PLATFORM - require 'fiddle/import' - require 'fiddle/types' - - module Win32 - extend Fiddle::Importer - dlload "kernel32.dll", "psapi.dll" - include Fiddle::Win32Types - typealias "SIZE_T", "size_t" - - PROCESS_MEMORY_COUNTERS = struct [ - "DWORD cb", - "DWORD PageFaultCount", - "SIZE_T PeakWorkingSetSize", - "SIZE_T WorkingSetSize", - "SIZE_T QuotaPeakPagedPoolUsage", - "SIZE_T QuotaPagedPoolUsage", - "SIZE_T QuotaPeakNonPagedPoolUsage", - "SIZE_T QuotaNonPagedPoolUsage", - "SIZE_T PagefileUsage", - "SIZE_T PeakPagefileUsage", - ] - - typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" - - extern "HANDLE GetCurrentProcess()", :stdcall - extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall - - module_function - def memory_info - size = PROCESS_MEMORY_COUNTERS.size - data = PROCESS_MEMORY_COUNTERS.malloc - data.cb = size - data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + keys.push(:size, :rss, :peak) + + begin + require 'fiddle/import' + require 'fiddle/types' + rescue LoadError + # Fallback to PowerShell command to get memory information for current process + def self.read_status + cmd = [ + "powershell.exe", "-NoProfile", "-Command", + "Get-Process -Id #{$$} | " \ + "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }" + ] + + IO.popen(cmd, "r", err: [:child, :out]) do |out| + if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read + yield :size, $1.to_i + yield :rss, $2.to_i + yield :peak, $3.to_i + end + end + end + else + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end end - end - keys.push(:size, :rss, :peak) - def self.read_status - if info = Win32.memory_info - yield :size, info.PagefileUsage - yield :rss, info.WorkingSetSize - yield :peak, info.PeakWorkingSetSize + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end end end when (require_relative 'find_executable' diff --git a/tool/lib/output.rb b/tool/lib/output.rb index 51e3d32010..8cb426ae4a 100644 --- a/tool/lib/output.rb +++ b/tool/lib/output.rb @@ -4,21 +4,34 @@ require_relative 'colorize' class Output attr_reader :path, :vpath - def initialize - @path = @timestamp = @ifchange = @color = nil - @vpath = VPath.new + def initialize(path: nil, timestamp: nil, ifchange: nil, color: nil, + overwrite: false, create_only: false, vpath: VPath.new) + @path = path + @timestamp = timestamp + @ifchange = ifchange + @color = color + @overwrite = overwrite + @create_only = create_only + @vpath = vpath end + COLOR_WHEN = { + 'always' => true, 'auto' => nil, 'never' => false, + nil => true, false => false, + } + def def_options(opt) opt.separator(" Output common options:") opt.on('-o', '--output=PATH') {|v| @path = v} opt.on('-t', '--timestamp[=PATH]') {|v| @timestamp = v || true} opt.on('-c', '--[no-]if-change') {|v| @ifchange = v} - opt.on('--color') {@color = true} + opt.on('--[no-]color=[WHEN]', COLOR_WHEN.keys) {|v| @color = COLOR_WHEN[v]} + opt.on('--[no-]create-only') {|v| @create_only = v} + opt.on('--[no-]overwrite') {|v| @overwrite = v} @vpath.def_options(opt) end - def write(data, overwrite: false, create_only: false) + def write(data, overwrite: @overwrite, create_only: @create_only) unless @path $stdout.print data return true diff --git a/tool/lib/path.rb b/tool/lib/path.rb new file mode 100644 index 0000000000..f16a164338 --- /dev/null +++ b/tool/lib/path.rb @@ -0,0 +1,101 @@ +module Path + module_function + + def clean(path) + path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') + nil while path.sub!(/[^\/]+\/\.\.\//, '') + path + end + + def relative(path, base) + path = clean(path) + base = clean(base) + path, base = [path, base].map{|s|s.split("/")} + until path.empty? or base.empty? or path[0] != base[0] + path.shift + base.shift + end + path, base = [path, base].map{|s|s.join("/")} + if base.empty? + path + elsif base.start_with?("../") or File.absolute_path?(base) + File.expand_path(path) + else + base.gsub!(/[^\/]+/, '..') + File.join(base, path) + end + end + + def clean_link(src, dest) + begin + link = File.readlink(dest) + rescue + else + return if link == src + File.unlink(dest) + end + yield src, dest + end + + # Extensions to FileUtils + + module Mswin + def ln_safe(src, dest, real_src, *opt) + cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] + cmd[1, 0] = opt + return if system("cmd", "/c", *cmd) + # TODO: use RUNAS or something + puts cmd.join(" ") + end + + def ln_dir_safe(src, dest, real_src) + ln_safe(src, dest, "/d") + end + end + + module HardlinkExcutable + def ln_exe(relative_src, dest, src) + ln(src, dest, force: true) + end + end + + def ln_safe(src, dest, real_src) + ln_sf(src, dest) + rescue Errno::ENOENT + # Windows disallows to create broken symboic links, probably because + # it is a kind of reparse points. + raise if File.exist?(real_src) + end + + alias ln_dir_safe ln_safe + alias ln_exe ln_safe + + def ln_relative(src, dest, executable = false) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + if executable + return (ln_exe(relative(src, parent), dest, src) if File.exist?(src)) + end + clean_link(relative(src, parent), dest) {|s, d| ln_safe(s, d, src)} + end + + def ln_dir_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative(src, parent), dest) {|s, d| ln_dir_safe(s, d, src)} + end + + case (CROSS_COMPILING || RUBY_PLATFORM) + when /linux|darwin|solaris/ + prepend HardlinkExcutable + extend HardlinkExcutable + when /mingw|mswin/ + unless File.respond_to?(:symlink) + prepend Mswin + extend Mswin + end + else + end +end diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb new file mode 100644 index 0000000000..7b889163b0 --- /dev/null +++ b/tool/lib/test/jobserver.rb @@ -0,0 +1,47 @@ +module Test + module JobServer + end +end + +class << Test::JobServer + def connect(makeflags = ENV["MAKEFLAGS"]) + return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags + begin + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end + rescue + r&.close + nil + else + return r, w + end + end + + def acquire_possible(r, w, max) + return unless tokens = r.read_nonblock(max - 1, exception: false) + if (jobs = tokens.size) > 0 + jobserver, w = w, nil + at_exit do + jobserver.print(tokens) + jobserver.close + end + end + return jobs + 1 + rescue Errno::EBADF + ensure + r&.close + w&.close + end + + def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"]) + if max > 1 and (r, w = connect(makeflags)) + acquire_possible(r, w, max) + end + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 0fa7b4c5cd..2663b7b76a 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -19,47 +19,12 @@ require_relative '../envutil' require_relative '../colorize' require_relative '../leakchecker' require_relative '../test/unit/testcase' +require_relative '../test/jobserver' require 'optparse' # See Test::Unit module Test - class << self - ## - # Filter object for backtraces. - - attr_accessor :backtrace_filter - end - - class BacktraceFilter # :nodoc: - def filter bt - return ["No backtrace"] unless bt - - new_bt = [] - pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] - - unless $DEBUG then - bt.each do |line| - break if pattern.match?(line) - new_bt << line - end - - new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? - new_bt = bt.dup if new_bt.empty? - else - new_bt = bt.dup - end - - new_bt - end - end - - self.backtrace_filter = BacktraceFilter.new - - def self.filter_backtrace bt # :nodoc: - backtrace_filter.filter bt - end - ## # Test::Unit is an implementation of the xUnit testing framework for Ruby. module Unit @@ -73,6 +38,26 @@ module Test class PendedError < AssertionFailedError; end + class << self + ## + # Extract the location where the last assertion method was + # called. Returns "<empty>" if _e_ does not have backtrace, or + # an empty string if no assertion method location was found. + + def location e + last_before_assertion = nil + + return '<empty>' unless e&.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /:in \W(?:.*\#)?(?:assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + return "" unless last_before_assertion + /:in / =~ last_before_assertion ? $` : last_before_assertion + end + end + module Order class NoSort def initialize(seed) @@ -89,17 +74,7 @@ module Test end end - module MJITFirst - def group(list) - # MJIT first - mjit, others = list.partition {|e| /test_mjit/ =~ e} - mjit + others - end - end - class Alpha < NoSort - include MJITFirst - def sort_by_name(list) list.sort_by(&:name) end @@ -112,8 +87,6 @@ module Test # shuffle test suites based on CRC32 of their names Shuffle = Struct.new(:seed, :salt) do - include MJITFirst - def initialize(seed) self.class::CRC_TBL ||= (0..255).map {|i| (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } @@ -131,6 +104,10 @@ module Test list.sort_by {|e| randomize_key(e)} end + def group(list) + list + end + private def crc32(str, crc32 = 0xffffffff) @@ -273,6 +250,8 @@ module Test end module Parallel # :nodoc: all + attr_accessor :prefix + def process_args(args = []) return @options if @options options = super @@ -284,29 +263,10 @@ module Test def non_options(files, options) @jobserver = nil - makeflags = ENV.delete("MAKEFLAGS") - if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags - begin - if fifo = $3 - fifo.gsub!(/\\(?=.)/, '') - r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) - w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) - else - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) - end - rescue - r.close if r - nil - else - r.close_on_exec = true - w.close_on_exec = true - @jobserver = [r, w] - options[:parallel] ||= 1 - end + if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS")) + options[:parallel] ||= 256 # number of tokens to acquire first end - @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 180) + @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) super end @@ -322,7 +282,7 @@ module Test opts.separator "parallel test options:" - options[:retry] = true + options[:retry] = false opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| options[:testing] = true & t # For testing @@ -346,7 +306,8 @@ module Test options[:retry] = false end - opts.on '--ruby VAL', "Path to ruby which is used at -j option" do |a| + opts.on '--ruby VAL', "Path to ruby which is used at -j option", + "Also used as EnvUtil.rubybin by some assertion methods" do |a| options[:ruby] = a.split(/ /).reject(&:empty?) end @@ -393,8 +354,12 @@ module Test @io.puts(*args) end - def run(task,type) - @file = File.basename(task, ".rb") + def run(task, type, base = nil) + if base + @file = task.delete_prefix(base).chomp(".rb") + else + @file = File.basename(task, ".rb") + end @real_file = task begin puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" @@ -429,16 +394,19 @@ module Test rescue IOError end - def quit + def quit(reason = :normal) return if @io.closed? @quit_called = true - @io.puts "quit" + @io.puts "quit #{reason}" rescue Errno::EPIPE => e warn "#{@pid}:#{@status.to_s.ljust(7)}:#{@file}: #{e.message}" end def kill - Process.kill(:KILL, @pid) + EnvUtil::Debugger.search&.dump(@pid) + signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV + Process.kill(signal, @pid) + warn "worker #{to_s} does not respond; #{signal} is sent" rescue Errno::ESRCH end @@ -556,15 +524,15 @@ module Test @workers.reject! do |worker| next unless cond&.call(worker) begin - Timeout.timeout(1) do - worker.quit + Timeout.timeout(5) do + worker.quit(cond ? :timeout : :normal) end rescue Errno::EPIPE rescue Timeout::Error end closed&.push worker begin - Timeout.timeout(0.2) do + Timeout.timeout(1) do worker.close end rescue Timeout::Error @@ -577,7 +545,7 @@ module Test return if (closed ||= @workers).empty? pids = closed.map(&:pid) begin - Timeout.timeout(0.2 * closed.size) do + Timeout.timeout(1 * closed.size) do Process.waitall end rescue Timeout::Error @@ -618,7 +586,7 @@ module Test worker.quit worker = launch_worker end - worker.run(task, type) + worker.run(task, type, (@prefix unless @options[:job_status] == :replace)) @test_count += 1 jobs_status(worker) @@ -696,10 +664,18 @@ module Test @ios = [] # Array of worker IOs @job_tokens = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver begin - [@tasks.size, @options[:parallel]].min.times {launch_worker} - while true - timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0) + @worker_timeout, 1].max + newjobs = [@tasks.size, @options[:parallel]].min - @workers.size + if newjobs > 0 + if @jobserver + t = @jobserver[0].read_nonblock(newjobs, exception: false) + @job_tokens << t if String === t + newjobs = @job_tokens.size + 1 - @workers.size + end + newjobs.times {launch_worker} + end + + timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0), 0].max + @worker_timeout if !(_io = IO.select(@ios, nil, nil, timeout)) timeout = Time.now - @worker_timeout @@ -713,15 +689,9 @@ module Test } break end - break if @tasks.empty? and @workers.empty? - if @jobserver and @job_tokens and !@tasks.empty? and - ((newjobs = [@tasks.size, @options[:parallel]].min) > @workers.size or - !@workers.any? {|x| x.status == :ready}) - t = @jobserver[0].read_nonblock(newjobs, exception: false) - if String === t - @job_tokens << t - t.size.times {launch_worker} - end + if @tasks.empty? + break if @workers.empty? + next # wait for all workers to finish end end rescue Interrupt => ex @@ -757,7 +727,15 @@ module Test del_status_line or puts error, suites = suites.partition {|r| r[:error]} unless suites.empty? - puts "\n""Retrying..." + puts "\n" + @failed_output.puts "Failed tests:" + suites.each {|r| + r[:report].each {|c, m, e| + @failed_output.puts "#{c}##{m}: #{e&.class}: #{e&.message&.slice(/\A.*/)}" + } + } + @failed_output.puts "\n" + puts "Retrying..." @verbose = options[:verbose] suites.map! {|r| ::Object.const_get(r[:testcase])} _run_suites(suites, type) @@ -813,6 +791,7 @@ module Test warn "" @warnings.uniq! {|w| w[1].message} @warnings.each do |w| + @errors += 1 warn "#{w[0]}: #{w[1].message} (#{w[1].class})" end warn "" @@ -882,7 +861,7 @@ module Test end end - def record(suite, method, assertions, time, error) + def record(suite, method, assertions, time, error, source_location = nil) if @options.values_at(:longest, :most_asserted).any? @tops ||= {} rec = [suite.name, method, assertions, time, error] @@ -980,7 +959,7 @@ module Test end def _prepare_run(suites, type) - options[:job_status] ||= :replace if @tty && !@verbose + options[:job_status] ||= @tty ? :replace : :normal unless @verbose case options[:color] when :always color = true @@ -996,11 +975,14 @@ module Test @output = Output.new(self) unless @options[:testing] filter = options[:filter] type = "#{type}_methods" - total = if filter - suites.inject(0) {|n, suite| n + suite.send(type).grep(filter).size} - else - suites.inject(0) {|n, suite| n + suite.send(type).size} - end + total = suites.sum {|suite| + methods = suite.send(type) + if filter + methods.count {|method| filter === "#{suite}##{method}"} + else + methods.size + end + } @test_count = 0 @total_tests = total.to_s(10) end @@ -1094,7 +1076,7 @@ module Test runner.add_status(" = #$1") when /\A\.+\z/ runner.succeed - when /\A\.*[EFS][EFS.]*\z/ + when /\A\.*[EFST][EFST.]*\z/ runner.failed(s) else $stdout.print(s) @@ -1282,8 +1264,13 @@ module Test puts "#{f}: #{$!}" end } + @load_failed = errors.size.nonzero? result end + + def run(*) + super or @load_failed + end end module RepeatOption # :nodoc: all @@ -1294,10 +1281,15 @@ module Test parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| options[:repeat_count] = n end + options[:keep_repeating] = false + parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n| + options[:keep_repeating] = true + end end def _run_anything(type) @repeat_count = @options[:repeat_count] + @keep_repeating = @options[:keep_repeating] super end end @@ -1385,6 +1377,95 @@ module Test end end + module LaunchableOption + module Nothing + private + def setup_options(opts, options) + super + opts.define_tail 'Launchable options:' + # This is expected to be called by Test::Unit::Worker. + opts.on_tail '--launchable-test-reports=PATH', String, 'Do nothing' + end + end + + def record(suite, method, assertions, time, error, source_location = nil) + if writer = @options[:launchable_test_reports] + if loc = (source_location || suite.instance_method(method).source_location) + path, lineno = loc + # Launchable JSON schema is defined at + # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code. + e = case error + when nil + status = 'TEST_PASSED' + nil + when Test::Unit::PendedError + status = 'TEST_SKIPPED' + "Skipped:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Test::Unit::AssertionFailedError + status = 'TEST_FAILED' + "Failure:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Timeout::Error + status = 'TEST_FAILED' + "Timeout:\n#{suite.name}##{method}\n" + else + status = 'TEST_FAILED' + bt = Test::filter_backtrace(error.backtrace).join "\n " + "Error:\n#{suite.name}##{method}:\n#{error.class}: #{error.message.b}\n #{bt}\n" + end + repo_path = File.expand_path("#{__dir__}/../../../") + relative_path = path.delete_prefix("#{repo_path}/") + # The test path is a URL-encoded representation. + # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18 + test_path = {file: relative_path, class: suite.name, testcase: method}.map{|key, val| + "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}" + }.join('#') + end + end + super + ensure + if writer && test_path && status + # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified. + # In such cases, we proceed to execute the operation here. + writer.write_object( + { + testPath: test_path, + status: status, + duration: time, + createdAt: Time.now.to_s, + stderr: e, + stdout: nil, + data: { + lineNumber: lineno + } + } + ) + end + end + + private + def setup_options(opts, options) + super + opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path| + require_relative '../launchable' + options[:launchable_test_reports] = writer = Launchable::JsonStreamWriter.new(path) + writer.write_array('testCases') + main_pid = Process.pid + at_exit { + # This block is executed when the fork block in a test is completed. + # Therefore, we need to verify whether all tests have been completed. + stack = caller + if stack.size == 0 && main_pid == Process.pid && $!.is_a?(SystemExit) + writer.close + end + } + end + + def encode_test_path_component component + component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26') + end + end + end + class Runner # :nodoc: all attr_accessor :report, :failures, :errors, :skips # :nodoc: @@ -1530,7 +1611,7 @@ module Test [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, t, @test_count.fdiv(t), @assertion_count.fdiv(t)] end while @repeat_count && @@current_repeat_count < @repeat_count && - report.empty? && failures.zero? && errors.zero? + (@keep_repeating || report.empty? && failures.zero? && errors.zero?) output.sync = old_sync if sync @@ -1574,7 +1655,7 @@ module Test _start_method(inst) inst._assertions = 0 - print "#{suite}##{method} = " if @verbose + print "#{suite}##{method.inspect.sub(/\A:/, '')} = " if @verbose start_time = Time.now if @verbose result = @@ -1589,9 +1670,7 @@ module Test puts if @verbose $stdout.flush - unless defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # compiler process is wrongly considered as leak - leakchecker.check("#{inst.class}\##{inst.__name__}") - end + leakchecker.check("#{inst.class}\##{inst.__name__}") _end_method(inst) @@ -1622,19 +1701,11 @@ module Test # failure or error in teardown, it will be sent again with the # error or failure. - def record suite, method, assertions, time, error + def record suite, method, assertions, time, error, source_location = nil end def location e # :nodoc: - last_before_assertion = "" - - return '<empty>' unless e.backtrace # SystemStackError can return nil. - - e.backtrace.reverse_each do |s| - break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ - last_before_assertion = s - end - last_before_assertion.sub(/:in .*$/, '') + Test::Unit.location e end ## @@ -1680,7 +1751,7 @@ module Test break unless report.empty? end - return failures + errors if self.test_count > 0 # or return nil... + return (failures + errors).nonzero? # or return nil... rescue Interrupt abort 'Interrupted' end @@ -1713,6 +1784,7 @@ module Test prepend Test::Unit::ExcludesOption prepend Test::Unit::TimeoutOption prepend Test::Unit::RunCount + prepend Test::Unit::LaunchableOption::Nothing ## # Begins the full test run. Delegates to +runner+'s #_run method. @@ -1769,6 +1841,7 @@ module Test class AutoRunner # :nodoc: all class Runner < Test::Unit::Runner include Test::Unit::RequireFiles + include Test::Unit::LaunchableOption end attr_accessor :to_run, :options @@ -1777,6 +1850,7 @@ module Test @force_standalone = force_standalone @runner = Runner.new do |files, options| base = options[:base_directory] ||= default_dir + @runner.prefix = base ? (base + "/") : nil files << default_dir if files.empty? and default_dir @to_run = files yield self if block_given? diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index b4f1dbc176..19581fc3ab 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -522,7 +522,7 @@ module Test # Skips the current test. Gets listed at the end of the run but # doesn't cause a failure exit code. - def pend msg = nil, bt = caller + def pend msg = nil, bt = caller, &_ msg ||= "Skipped, no message given" @skip = true raise Test::Unit::PendedError, msg, bt @@ -768,7 +768,14 @@ EOT e = assert_raise(SyntaxError, mesg) do syntax_check(src, fname, line) end - assert_match(error, e.message, mesg) + + # Prism adds ANSI escape sequences to syntax error messages to + # colorize and format them. We strip them out here to make them easier + # to match against in tests. + message = e.message + message.gsub!(/\e\[.*?m/, "") + + assert_match(error, message, mesg) e end end @@ -791,33 +798,37 @@ EOT MIN_MEASURABLE = 1.0 / MIN_HZ def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) - require 'benchmark' - wait = EnvUtil.apply_timeout_scale(wait) if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" end - tms = Benchmark.measure(msg || '') do - if stop - th = Thread.start {sleep wait; stop.call} - yield - th.join - else - begin - Timeout.timeout(wait) {yield} - rescue Timeout::Error - end + + t0, r0 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error end end - max = pct * tms.real + t1, r1 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + total = t1.utime - t0.utime + t1.stime - t0.stime + t1.cutime - t0.cutime + t1.cstime - t0.cstime + real = r1 - r0 + + max = pct * real min_measurable = MIN_MEASURABLE min_measurable *= 1.30 # add a little (30%) to account for misc. overheads if max < min_measurable max = min_measurable end - assert_operator tms.total, :<=, max, msg + assert_operator total, :<=, max, msg end def assert_is_minus_zero(f) diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb index b3a8957f26..188a0d1a19 100644 --- a/tool/lib/test/unit/parallel.rb +++ b/tool/lib/test/unit/parallel.rb @@ -1,12 +1,6 @@ # frozen_string_literal: true -$LOAD_PATH.unshift "#{__dir__}/../.." -require_relative '../../test/unit' -require_relative '../../profile_test_all' if ENV.key?('RUBY_TEST_ALL_PROFILE') -require_relative '../../tracepointchecker' -require_relative '../../zombie_hunter' -require_relative '../../iseq_loader_checker' -require_relative '../../gc_checker' +require_relative "../../../test/init" module Test module Unit @@ -133,7 +127,18 @@ module Test else _report "ready" end - when /^quit$/ + when /^quit (.+?)$/, "quit" + if $1 == "timeout" + err = ["", "!!! worker #{$$} killed due to timeout:"] + Thread.list.each do |th| + err << "#{ th.inspect }:" + th.backtrace.each do |s| + err << " #{ s }" + end + end + err << "" + STDERR.puts err.join("\n") + end _report "bye" exit end @@ -186,7 +191,7 @@ module Test else error = ProxyError.new(error) end - _report "record", Marshal.dump([suite.name, method, assertions, time, error]) + _report "record", Marshal.dump([suite.name, method, assertions, time, error, suite.instance_method(method).source_location]) super end end @@ -208,5 +213,9 @@ if $0 == __FILE__ end end require 'rubygems' + begin + require 'rake' + rescue LoadError + end Test::Unit::Worker.new.run(ARGV) end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb index 44d9ba7fdb..51ffff37eb 100644 --- a/tool/lib/test/unit/testcase.rb +++ b/tool/lib/test/unit/testcase.rb @@ -137,6 +137,9 @@ module Test attr_reader :__name__ # :nodoc: + # Method name of this test. + alias method_name __name__ + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, Interrupt, SystemExit] # :nodoc: @@ -144,8 +147,7 @@ module Test # Runs the tests reporting the status to +runner+ def run runner - @options = runner.options - + @__runner_options__ = runner.options trap "INFO" do runner.report.each_with_index do |msg, i| warn "\n%3d) %s" % [i + 1, msg] @@ -161,7 +163,7 @@ module Test result = "" begin - @passed = nil + @__passed__ = nil self.before_setup self.setup self.after_setup @@ -169,11 +171,11 @@ module Test result = "." unless io? time = Time.now - start_time runner.record self.class, self.__name__, self._assertions, time, nil - @passed = true + @__passed__ = true rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - @passed = Test::Unit::PendedError === e + @__passed__ = Test::Unit::PendedError === e time = Time.now - start_time runner.record self.class, self.__name__, self._assertions, time, e result = runner.puke self.class, self.__name__, e @@ -184,7 +186,7 @@ module Test rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - @passed = false + @__passed__ = false runner.record self.class, self.__name__, self._assertions, time, e result = runner.puke self.class, self.__name__, e end @@ -206,12 +208,12 @@ module Test def initialize name # :nodoc: @__name__ = name @__io__ = nil - @passed = nil - @@current = self # FIX: make thread local + @__passed__ = nil + @@__current__ = self # FIX: make thread local end def self.current # :nodoc: - @@current # FIX: make thread local + @@__current__ # FIX: make thread local end ## @@ -263,7 +265,7 @@ module Test # Returns true if the test passed. def passed? - @passed + @__passed__ end ## diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb index 857b7928a4..26c9763c13 100644 --- a/tool/lib/vcs.rb +++ b/tool/lib/vcs.rb @@ -51,23 +51,9 @@ module DebugPOpen end using DebugPOpen module DebugSystem - def system(*args) + def system(*args, exception: true, **opts) VCS.dump(args, "args: ") if $DEBUG - exception = false - opts = Hash.try_convert(args[-1]) - if RUBY_VERSION >= "2.6" - unless opts - opts = {} - args << opts - end - exception = opts.fetch(:exception) {opts[:exception] = true} - elsif opts - exception = opts.delete(:exception) {true} - args.pop if opts.empty? - end - ret = super(*args) - raise "Command failed with status (#$?): #{args[0]}" if exception and !ret - ret + super(*args, exception: exception, **opts) end end @@ -154,8 +140,7 @@ class VCS alias dryrun? dryrun alias debug? debug - NullDevice = defined?(IO::NULL) ? IO::NULL : - %w[/dev/null NUL NIL: NL:].find {|dev| File.exist?(dev)} + NullDevice = IO::NULL # returns # * the last revision of the current branch @@ -195,6 +180,7 @@ class VCS rescue ArgumentError modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 end + modified = modified.getlocal(@zone) end return last, changed, modified, *rest end @@ -226,6 +212,7 @@ class VCS def after_export(dir) FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + FileUtils.rm_rf(Dir.glob("#{dir}/.mailmap")) end def revision_handler(rev) @@ -270,155 +257,6 @@ class VCS code end - class SVN < self - register(".svn") - COMMAND = ENV['SVN'] || 'svn' - - def self.revision_name(rev) - "r#{rev}" - end - - def _get_revisions(path, srcdir = nil) - if srcdir and self.class.local_path?(path) - path = File.join(srcdir, path) - end - if srcdir - info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") - info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s - end - info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") - _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) - modified = info_xml[/<date>([^<>]*)/, 1] - branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] - [Integer(last), Integer(changed), modified, branch] - end - - def self.search_root(path) - return unless local_path?(path) - parent = File.realpath(path) - begin - parent = File.dirname(wkdir = parent) - return wkdir if File.directory?(wkdir + "/.svn") - end until parent == wkdir - end - - def get_info - @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") - end - - def url - @url ||= begin - url = get_info[/<root>(.*)<\/root>/, 1] - @url = URI.parse(url+"/") if url - end - end - - def wcroot - @wcroot ||= begin - info = get_info - @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] - @wcroot ||= self.class.search_root(@srcdir) - end - end - - def branch(name) - return trunk if name == "trunk" - url + "branches/#{name}" - end - - def tag(name) - url + "tags/#{name}" - end - - def trunk - url + "trunk" - end - alias master trunk - - def branch_list(pat) - IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| - f.each do |line| - line.chomp! - line.chomp!('/') - yield(line) if File.fnmatch?(pat, line) - end - end - end - - def grep(pat, tag, *files, &block) - cmd = %W"#{COMMAND} cat" - files.map! {|n| File.join(tag, n)} if tag - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files]) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end - - def export(revision, url, dir, keep_temp = false) - if @srcdir and (rootdir = wcroot) - srcdir = File.realpath(@srcdir) - rootdir << "/" - if srcdir.start_with?(rootdir) - subdir = srcdir[rootdir.size..-1] - subdir = nil if subdir.empty? - FileUtils.mkdir_p(svndir = dir+"/.svn") - FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) - system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false - FileUtils.rm_rf(svndir) unless keep_temp - if subdir - tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") - File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") - FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) - begin - Dir.rmdir("#{dir}/#{subdir}") - end until (subdir = File.dirname(subdir)) == '.' - FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) - Dir.rmdir(tmpdir) - end - return self - end - end - IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| - pipe.each {|line| /^A/ =~ line or yield line} - end - self if $?.success? - end - - def after_export(dir) - super - FileUtils.rm_rf(dir+"/.svn") - end - - def branch_beginning(url) - # `--limit` of svn-log is useless in this case, because it is - # applied before `--search`. - rev = IO.pread(%W[ #{COMMAND} log --xml - --search=matz --search-and=has\ started - -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] - rev.to_i if rev - end - - def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path) - range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') - IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -r#{range} #{url}") do |r| - IO.copy_stream(r, path) - end - end - - def commit - args = %W"#{COMMAND} commit" - if dryrun? - VCS.dump(args, "commit: ") - return true - end - system(*args) - end - end - class GIT < self register(".git") do |path, dir| SAFE_DIRECTORIES ||= @@ -478,7 +316,7 @@ class VCS last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref, err: w]]).rstrip w.close unless r.eof? - raise "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" + raise VCS::NotFoundError, "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" end end log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) @@ -532,15 +370,6 @@ class VCS rev[0, 10] end - def revision_handler(rev) - case rev - when Integer - SVN - else - super - end - end - def without_gitconfig envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| h[v] = ENV.delete(v) @@ -615,32 +444,26 @@ class VCS def export(revision, url, dir, keep_temp = false) system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return - (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir)) + GIT.new(File.expand_path(dir)) end def branch_beginning(url) - cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H + year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i + cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25 --author=matz --committer=matz --grep=started\\.$ - #{url.to_str} -- version.h include/ruby/version.h]) + #{url} -- version.h include/ruby/version.h])[/.*/] end - def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil) - svn = nil + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true) from, to = [from, to].map do |rev| rev or next - if Integer === rev - svn = true - rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -n1 --format=format:%H" << - "--grep=^ *git-svn-id: .*@#{rev} ") - end rev unless rev.empty? end - unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from)) + unless from&.match?(/./) or (from = branch_beginning(url))&.match?(/./) warn "no starting commit found", uplevel: 1 from = nil end - if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", chdir: @srcdir, exception: false) system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", chdir: @srcdir, exception: false) @@ -653,22 +476,18 @@ class VCS else arg = ["--since=25 Dec 00:00:00", to] end - writer = - if svn - format_changelog_as_svn(path, arg) - else - if base_url == true - remote, = upstream - if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) - remote.chomp! - # hack to redirect git.r-l.o to github - remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') - remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') - end - base_url = remote - end - format_changelog(path, arg, base_url) + if base_url == true + env = CHANGELOG_ENV + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') end + base_url = remote + end + writer = changelog_formatter(path, arg, base_url) if !path or path == '-' writer[$stdout] else @@ -677,10 +496,14 @@ class VCS end LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' - - def format_changelog(path, arg, base_url = nil) - env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} - cmd = %W"#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges" + CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + + def changelog_formatter(path, arg, base_url = nil) + env = CHANGELOG_ENV + cmd = %W[#{COMMAND} log + --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges + --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] + ] date = "--date=iso-local" unless system(env, *cmd, date, "-1", chdir: @srcdir, out: NullDevice, exception: false) date = "--date=iso" @@ -688,25 +511,69 @@ class VCS cmd << date cmd.concat(arg) proc do |w| - w.print "-*- coding: utf-8 -*-\n\n" - w.print "base-url = #{base_url}\n\n" if base_url + w.print "-*- coding: utf-8 -*-\n" + w.print "\n""base-url = #{base_url}\n" if base_url + + begin + ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true) + .grep_v(/^ *(?:#|$)/) + .to_h {|v| [v, true]} + ignore_revs = nil if ignore_revs.empty? + rescue Errno::ENOENT + end + cmd_pipe(env, cmd, chdir: @srcdir) do |r| - while s = r.gets("\ncommit ") + r.gets(sep = "commit ") + sep = "\n" + sep + while s = r.gets(sep, chomp: true) h, s = s.split(/^$/, 2) + if ignore_revs&.key?(h[/\A\h{40}/]) + next + end + + next if /^Author: *dependabot\[bot\]/ =~ h + h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') fix = $1 + next if /\A *skip\Z/ =~ fix s = s.lines fix.each_line do |x| + next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') + b = ($2&.to_i || (s.size - 1 + $3.to_i)) + sp = $1 + if x.sub!(/^,(?:(\d+)|\$(?:-\d+)?)/, '') + range = b..($1&.to_i || (s.size - 1 + $2.to_i)) + else + range = b..b + end case x - when %r[^ +(\d+)s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\2(.*)\2]o - n = $1.to_i - wrong = $3 - correct = $4 - begin - s[n][wrong] = correct - rescue IndexError - message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #$1\n"] + when %r[^s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\1(.*)\1([gr]+)?]o + wrong = $2 + correct = $3 + if opt = $4 and opt.include?("r") # regexp + wrong = Regexp.new(wrong) + correct.gsub!(/(?<!\\)(?:\\\\)*\K(?:\\n)+/) {"\n" * ($&.size / 2)} + sub = opt.include?("g") ? :gsub! : :sub! + else + sub = false + end + range.each do |n| + if sub + ss = s[n].sub(/^#{sp}/, "") # un-indent for /^/ + if ss.__send__(sub, wrong, correct) + s[n, 1] = ss.lines.map {|l| "#{sp}#{l}"} + next + end + else + begin + s[n][wrong] = correct + rescue IndexError + else + next + end + end + message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] from = [1, n-2].max to = [s.size-1, n+2].min s.each_with_index do |e, i| @@ -716,17 +583,22 @@ class VCS end raise message.join('') end - when %r[^( +)(\d+)i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\3]o - s[$2.to_i, 0] = "#{$1}#{$4}\n" - when %r[^ +(\d+)(?:,(\d+))?d] - n = $1.to_i - e = $2 - s[n..(e ? e.to_i : n)] = [] + when %r[^i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\1]o + insert = "#{sp}#{$2}\n" + range.reverse_each do |n| + s[n, 0] = insert + end + when %r[^d] + s[range] = [] end end s = s.join('') end + s.gsub!(%r[(?!<\w)([-\w]+/[-\w]+)(?:@(\h{8,40})|#(\d{5,}))\b]) do + path = defined?($2) ? "commit/#{$2}" : "pull/#{$3}" + "[#$&](https://github.com/#{$1}/#{path})" + end if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s issue = "#{$1}pull/" s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} @@ -734,28 +606,7 @@ class VCS s.gsub!(/ +\n/, "\n") s.sub!(/^Notes:/, ' \&') - w.print h, s - end - end - end - end - - def format_changelog_as_svn(path, arg) - cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B" - cmd.concat(arg) - proc do |w| - sep = "-"*72 + "\n" - w.print sep - cmd_pipe(cmd) do |r| - while s = r.gets("\0") - s.chomp!("\0") - author, time, s = s.split("\n", 3) - s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '') - rev = $1 - time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)") - lines = s.count("\n") + 1 - lines = "#{lines} line#{lines == 1 ? '' : 's'}" - w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep + w.print sep, h, s end end end @@ -792,46 +643,6 @@ class VCS end end - class GITSVN < GIT - def self.revision_name(rev) - SVN.revision_name(rev) - end - - def last_changed_revision - rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] - com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp - return rev, com - end - - def commit(opts = {}) - rev, com = last_changed_revision - head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp - - commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") - commits.each_with_index do |l, i| - r, a, c = l.split(' ') - dcommit = [COMMAND, "svn", "dcommit"] - dcommit.insert(-2, "-n") if dryrun? - dcommit << "--add-author-from" unless a == c - dcommit << r - system(*dcommit) or return false - system(COMMAND, "checkout", head) or return false - system(COMMAND, "rebase") or return false - end - - if rev - old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] - old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] - 3.times do - sleep 2 - system(*%W"#{COMMAND} pull --no-edit --rebase") - break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) - end - end - true - end - end - class Null < self def get_revisions(path, srcdir = nil) @modified ||= Time.now - 10 diff --git a/tool/lib/webrick.rb b/tool/lib/webrick.rb deleted file mode 100644 index b854b68db4..0000000000 --- a/tool/lib/webrick.rb +++ /dev/null @@ -1,232 +0,0 @@ -# frozen_string_literal: false -## -# = WEB server toolkit. -# -# WEBrick is an HTTP server toolkit that can be configured as an HTTPS server, -# a proxy server, and a virtual-host server. WEBrick features complete -# logging of both server operations and HTTP access. WEBrick supports both -# basic and digest authentication in addition to algorithms not in RFC 2617. -# -# A WEBrick server can be composed of multiple WEBrick servers or servlets to -# provide differing behavior on a per-host or per-path basis. WEBrick -# includes servlets for handling CGI scripts, ERB pages, Ruby blocks and -# directory listings. -# -# WEBrick also includes tools for daemonizing a process and starting a process -# at a higher privilege level and dropping permissions. -# -# == Security -# -# *Warning:* WEBrick is not recommended for production. It only implements -# basic security checks. -# -# == Starting an HTTP server -# -# To create a new WEBrick::HTTPServer that will listen to connections on port -# 8000 and serve documents from the current user's public_html folder: -# -# require 'webrick' -# -# root = File.expand_path '~/public_html' -# server = WEBrick::HTTPServer.new :Port => 8000, :DocumentRoot => root -# -# To run the server you will need to provide a suitable shutdown hook as -# starting the server blocks the current thread: -# -# trap 'INT' do server.shutdown end -# -# server.start -# -# == Custom Behavior -# -# The easiest way to have a server perform custom operations is through -# WEBrick::HTTPServer#mount_proc. The block given will be called with a -# WEBrick::HTTPRequest with request info and a WEBrick::HTTPResponse which -# must be filled in appropriately: -# -# server.mount_proc '/' do |req, res| -# res.body = 'Hello, world!' -# end -# -# Remember that +server.mount_proc+ must precede +server.start+. -# -# == Servlets -# -# Advanced custom behavior can be obtained through mounting a subclass of -# WEBrick::HTTPServlet::AbstractServlet. Servlets provide more modularity -# when writing an HTTP server than mount_proc allows. Here is a simple -# servlet: -# -# class Simple < WEBrick::HTTPServlet::AbstractServlet -# def do_GET request, response -# status, content_type, body = do_stuff_with request -# -# response.status = 200 -# response['Content-Type'] = 'text/plain' -# response.body = 'Hello, World!' -# end -# end -# -# To initialize the servlet you mount it on the server: -# -# server.mount '/simple', Simple -# -# See WEBrick::HTTPServlet::AbstractServlet for more details. -# -# == Virtual Hosts -# -# A server can act as a virtual host for multiple host names. After creating -# the listening host, additional hosts that do not listen can be created and -# attached as virtual hosts: -# -# server = WEBrick::HTTPServer.new # ... -# -# vhost = WEBrick::HTTPServer.new :ServerName => 'vhost.example', -# :DoNotListen => true, # ... -# vhost.mount '/', ... -# -# server.virtual_host vhost -# -# If no +:DocumentRoot+ is provided and no servlets or procs are mounted on the -# main server it will return 404 for all URLs. -# -# == HTTPS -# -# To create an HTTPS server you only need to enable SSL and provide an SSL -# certificate name: -# -# require 'webrick' -# require 'webrick/https' -# -# cert_name = [ -# %w[CN localhost], -# ] -# -# server = WEBrick::HTTPServer.new(:Port => 8000, -# :SSLEnable => true, -# :SSLCertName => cert_name) -# -# This will start the server with a self-generated self-signed certificate. -# The certificate will be changed every time the server is restarted. -# -# To create a server with a pre-determined key and certificate you can provide -# them: -# -# require 'webrick' -# require 'webrick/https' -# require 'openssl' -# -# cert = OpenSSL::X509::Certificate.new File.read '/path/to/cert.pem' -# pkey = OpenSSL::PKey::RSA.new File.read '/path/to/pkey.pem' -# -# server = WEBrick::HTTPServer.new(:Port => 8000, -# :SSLEnable => true, -# :SSLCertificate => cert, -# :SSLPrivateKey => pkey) -# -# == Proxy Server -# -# WEBrick can act as a proxy server: -# -# require 'webrick' -# require 'webrick/httpproxy' -# -# proxy = WEBrick::HTTPProxyServer.new :Port => 8000 -# -# trap 'INT' do proxy.shutdown end -# -# See WEBrick::HTTPProxy for further details including modifying proxied -# responses. -# -# == Basic and Digest authentication -# -# WEBrick provides both Basic and Digest authentication for regular and proxy -# servers. See WEBrick::HTTPAuth, WEBrick::HTTPAuth::BasicAuth and -# WEBrick::HTTPAuth::DigestAuth. -# -# == WEBrick as a daemonized Web Server -# -# WEBrick can be run as a daemonized server for small loads. -# -# === Daemonizing -# -# To start a WEBrick server as a daemon simple run WEBrick::Daemon.start -# before starting the server. -# -# === Dropping Permissions -# -# WEBrick can be started as one user to gain permission to bind to port 80 or -# 443 for serving HTTP or HTTPS traffic then can drop these permissions for -# regular operation. To listen on all interfaces for HTTP traffic: -# -# sockets = WEBrick::Utils.create_listeners nil, 80 -# -# Then drop privileges: -# -# WEBrick::Utils.su 'www' -# -# Then create a server that does not listen by default: -# -# server = WEBrick::HTTPServer.new :DoNotListen => true, # ... -# -# Then overwrite the listening sockets with the port 80 sockets: -# -# server.listeners.replace sockets -# -# === Logging -# -# WEBrick can separately log server operations and end-user access. For -# server operations: -# -# log_file = File.open '/var/log/webrick.log', 'a+' -# log = WEBrick::Log.new log_file -# -# For user access logging: -# -# access_log = [ -# [log_file, WEBrick::AccessLog::COMBINED_LOG_FORMAT], -# ] -# -# server = WEBrick::HTTPServer.new :Logger => log, :AccessLog => access_log -# -# See WEBrick::AccessLog for further log formats. -# -# === Log Rotation -# -# To rotate logs in WEBrick on a HUP signal (like syslogd can send), open the -# log file in 'a+' mode (as above) and trap 'HUP' to reopen the log file: -# -# trap 'HUP' do log_file.reopen '/path/to/webrick.log', 'a+' -# -# == Copyright -# -# Author: IPR -- Internet Programming with Ruby -- writers -# -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -#-- -# $IPR: webrick.rb,v 1.12 2002/10/01 17:16:31 gotoyuzo Exp $ - -module WEBrick -end - -require 'webrick/compat.rb' - -require 'webrick/version.rb' -require 'webrick/config.rb' -require 'webrick/log.rb' -require 'webrick/server.rb' -require_relative 'webrick/utils.rb' -require 'webrick/accesslog' - -require 'webrick/htmlutils.rb' -require 'webrick/httputils.rb' -require 'webrick/cookie.rb' -require 'webrick/httpversion.rb' -require 'webrick/httpstatus.rb' -require 'webrick/httprequest.rb' -require 'webrick/httpresponse.rb' -require 'webrick/httpserver.rb' -require 'webrick/httpservlet.rb' -require 'webrick/httpauth.rb' diff --git a/tool/lib/webrick/.document b/tool/lib/webrick/.document deleted file mode 100644 index c62f89083b..0000000000 --- a/tool/lib/webrick/.document +++ /dev/null @@ -1,6 +0,0 @@ -# Add files to this as they become documented - -*.rb - -httpauth -httpservlet diff --git a/tool/lib/webrick/accesslog.rb b/tool/lib/webrick/accesslog.rb deleted file mode 100644 index e4849637f3..0000000000 --- a/tool/lib/webrick/accesslog.rb +++ /dev/null @@ -1,157 +0,0 @@ -# frozen_string_literal: false -#-- -# accesslog.rb -- Access log handling utilities -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 keita yamaguchi -# Copyright (c) 2002 Internet Programming with Ruby writers -# -# $IPR: accesslog.rb,v 1.1 2002/10/01 17:16:32 gotoyuzo Exp $ - -module WEBrick - - ## - # AccessLog provides logging to various files in various formats. - # - # Multiple logs may be written to at the same time: - # - # access_log = [ - # [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT], - # [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT], - # ] - # - # server = WEBrick::HTTPServer.new :AccessLog => access_log - # - # Custom log formats may be defined. WEBrick::AccessLog provides a subset - # of the formatting from Apache's mod_log_config - # http://httpd.apache.org/docs/mod/mod_log_config.html#formats. See - # AccessLog::setup_params for a list of supported options - - module AccessLog - - ## - # Raised if a parameter such as %e, %i, %o or %n is used without fetching - # a specific field. - - class AccessLogError < StandardError; end - - ## - # The Common Log Format's time format - - CLF_TIME_FORMAT = "[%d/%b/%Y:%H:%M:%S %Z]" - - ## - # Common Log Format - - COMMON_LOG_FORMAT = "%h %l %u %t \"%r\" %s %b" - - ## - # Short alias for Common Log Format - - CLF = COMMON_LOG_FORMAT - - ## - # Referer Log Format - - REFERER_LOG_FORMAT = "%{Referer}i -> %U" - - ## - # User-Agent Log Format - - AGENT_LOG_FORMAT = "%{User-Agent}i" - - ## - # Combined Log Format - - COMBINED_LOG_FORMAT = "#{CLF} \"%{Referer}i\" \"%{User-agent}i\"" - - module_function - - # This format specification is a subset of mod_log_config of Apache: - # - # %a:: Remote IP address - # %b:: Total response size - # %e{variable}:: Given variable in ENV - # %f:: Response filename - # %h:: Remote host name - # %{header}i:: Given request header - # %l:: Remote logname, always "-" - # %m:: Request method - # %{attr}n:: Given request attribute from <tt>req.attributes</tt> - # %{header}o:: Given response header - # %p:: Server's request port - # %{format}p:: The canonical port of the server serving the request or the - # actual port or the client's actual port. Valid formats are - # canonical, local or remote. - # %q:: Request query string - # %r:: First line of the request - # %s:: Request status - # %t:: Time the request was received - # %T:: Time taken to process the request - # %u:: Remote user from auth - # %U:: Unparsed URI - # %%:: Literal % - - def setup_params(config, req, res) - params = Hash.new("") - params["a"] = req.peeraddr[3] - params["b"] = res.sent_size - params["e"] = ENV - params["f"] = res.filename || "" - params["h"] = req.peeraddr[2] - params["i"] = req - params["l"] = "-" - params["m"] = req.request_method - params["n"] = req.attributes - params["o"] = res - params["p"] = req.port - params["q"] = req.query_string - params["r"] = req.request_line.sub(/\x0d?\x0a\z/o, '') - params["s"] = res.status # won't support "%>s" - params["t"] = req.request_time - params["T"] = Time.now - req.request_time - params["u"] = req.user || "-" - params["U"] = req.unparsed_uri - params["v"] = config[:ServerName] - params - end - - ## - # Formats +params+ according to +format_string+ which is described in - # setup_params. - - def format(format_string, params) - format_string.gsub(/\%(?:\{(.*?)\})?>?([a-zA-Z%])/){ - param, spec = $1, $2 - case spec[0] - when ?e, ?i, ?n, ?o - raise AccessLogError, - "parameter is required for \"#{spec}\"" unless param - (param = params[spec][param]) ? escape(param) : "-" - when ?t - params[spec].strftime(param || CLF_TIME_FORMAT) - when ?p - case param - when 'remote' - escape(params["i"].peeraddr[1].to_s) - else - escape(params["p"].to_s) - end - when ?% - "%" - else - escape(params[spec].to_s) - end - } - end - - ## - # Escapes control characters in +data+ - - def escape(data) - data = data.gsub(/[[:cntrl:]\\]+/) {$&.dump[1...-1]} - data.untaint if RUBY_VERSION < '2.7' - data - end - end -end diff --git a/tool/lib/webrick/cgi.rb b/tool/lib/webrick/cgi.rb deleted file mode 100644 index bb0ae2fc84..0000000000 --- a/tool/lib/webrick/cgi.rb +++ /dev/null @@ -1,313 +0,0 @@ -# frozen_string_literal: false -# -# cgi.rb -- Yet another CGI library -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $Id$ - -require_relative "httprequest" -require_relative "httpresponse" -require_relative "config" -require "stringio" - -module WEBrick - - # A CGI library using WEBrick requests and responses. - # - # Example: - # - # class MyCGI < WEBrick::CGI - # def do_GET req, res - # res.body = 'it worked!' - # res.status = 200 - # end - # end - # - # MyCGI.new.start - - class CGI - - # The CGI error exception class - - CGIError = Class.new(StandardError) - - ## - # The CGI configuration. This is based on WEBrick::Config::HTTP - - attr_reader :config - - ## - # The CGI logger - - attr_reader :logger - - ## - # Creates a new CGI interface. - # - # The first argument in +args+ is a configuration hash which would update - # WEBrick::Config::HTTP. - # - # Any remaining arguments are stored in the <code>@options</code> instance - # variable for use by a subclass. - - def initialize(*args) - if defined?(MOD_RUBY) - unless ENV.has_key?("GATEWAY_INTERFACE") - Apache.request.setup_cgi_env - end - end - if %r{HTTP/(\d+\.\d+)} =~ ENV["SERVER_PROTOCOL"] - httpv = $1 - end - @config = WEBrick::Config::HTTP.dup.update( - :ServerSoftware => ENV["SERVER_SOFTWARE"] || "null", - :HTTPVersion => HTTPVersion.new(httpv || "1.0"), - :RunOnCGI => true, # to detect if it runs on CGI. - :NPH => false # set true to run as NPH script. - ) - if config = args.shift - @config.update(config) - end - @config[:Logger] ||= WEBrick::BasicLog.new($stderr) - @logger = @config[:Logger] - @options = args - end - - ## - # Reads +key+ from the configuration - - def [](key) - @config[key] - end - - ## - # Starts the CGI process with the given environment +env+ and standard - # input and output +stdin+ and +stdout+. - - def start(env=ENV, stdin=$stdin, stdout=$stdout) - sock = WEBrick::CGI::Socket.new(@config, env, stdin, stdout) - req = HTTPRequest.new(@config) - res = HTTPResponse.new(@config) - unless @config[:NPH] or defined?(MOD_RUBY) - def res.setup_header - unless @header["status"] - phrase = HTTPStatus::reason_phrase(@status) - @header["status"] = "#{@status} #{phrase}" - end - super - end - def res.status_line - "" - end - end - - begin - req.parse(sock) - req.script_name = (env["SCRIPT_NAME"] || File.expand_path($0)).dup - req.path_info = (env["PATH_INFO"] || "").dup - req.query_string = env["QUERY_STRING"] - req.user = env["REMOTE_USER"] - res.request_method = req.request_method - res.request_uri = req.request_uri - res.request_http_version = req.http_version - res.keep_alive = req.keep_alive? - self.service(req, res) - rescue HTTPStatus::Error => ex - res.set_error(ex) - rescue HTTPStatus::Status => ex - res.status = ex.code - rescue Exception => ex - @logger.error(ex) - res.set_error(ex, true) - ensure - req.fixup - if defined?(MOD_RUBY) - res.setup_header - Apache.request.status_line = "#{res.status} #{res.reason_phrase}" - Apache.request.status = res.status - table = Apache.request.headers_out - res.header.each{|key, val| - case key - when /^content-encoding$/i - Apache::request.content_encoding = val - when /^content-type$/i - Apache::request.content_type = val - else - table[key] = val.to_s - end - } - res.cookies.each{|cookie| - table.add("Set-Cookie", cookie.to_s) - } - Apache.request.send_http_header - res.send_body(sock) - else - res.send_response(sock) - end - end - end - - ## - # Services the request +req+ which will fill in the response +res+. See - # WEBrick::HTTPServlet::AbstractServlet#service for details. - - def service(req, res) - method_name = "do_" + req.request_method.gsub(/-/, "_") - if respond_to?(method_name) - __send__(method_name, req, res) - else - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - end - end - - ## - # Provides HTTP socket emulation from the CGI environment - - class Socket # :nodoc: - include Enumerable - - private - - def initialize(config, env, stdin, stdout) - @config = config - @env = env - @header_part = StringIO.new - @body_part = stdin - @out_port = stdout - @out_port.binmode - - @server_addr = @env["SERVER_ADDR"] || "0.0.0.0" - @server_name = @env["SERVER_NAME"] - @server_port = @env["SERVER_PORT"] - @remote_addr = @env["REMOTE_ADDR"] - @remote_host = @env["REMOTE_HOST"] || @remote_addr - @remote_port = @env["REMOTE_PORT"] || 0 - - begin - @header_part << request_line << CRLF - setup_header - @header_part << CRLF - @header_part.rewind - rescue Exception - raise CGIError, "invalid CGI environment" - end - end - - def request_line - meth = @env["REQUEST_METHOD"] || "GET" - unless url = @env["REQUEST_URI"] - url = (@env["SCRIPT_NAME"] || File.expand_path($0)).dup - url << @env["PATH_INFO"].to_s - url = WEBrick::HTTPUtils.escape_path(url) - if query_string = @env["QUERY_STRING"] - unless query_string.empty? - url << "?" << query_string - end - end - end - # we cannot get real HTTP version of client ;) - httpv = @config[:HTTPVersion] - return "#{meth} #{url} HTTP/#{httpv}" - end - - def setup_header - @env.each{|key, value| - case key - when "CONTENT_TYPE", "CONTENT_LENGTH" - add_header(key.gsub(/_/, "-"), value) - when /^HTTP_(.*)/ - add_header($1.gsub(/_/, "-"), value) - end - } - end - - def add_header(hdrname, value) - unless value.empty? - @header_part << hdrname << ": " << value << CRLF - end - end - - def input - @header_part.eof? ? @body_part : @header_part - end - - public - - def peeraddr - [nil, @remote_port, @remote_host, @remote_addr] - end - - def addr - [nil, @server_port, @server_name, @server_addr] - end - - def gets(eol=LF, size=nil) - input.gets(eol, size) - end - - def read(size=nil) - input.read(size) - end - - def each - input.each{|line| yield(line) } - end - - def eof? - input.eof? - end - - def <<(data) - @out_port << data - end - - def write(data) - @out_port.write(data) - end - - def cert - return nil unless defined?(OpenSSL) - if pem = @env["SSL_SERVER_CERT"] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - - def peer_cert - return nil unless defined?(OpenSSL) - if pem = @env["SSL_CLIENT_CERT"] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - - def peer_cert_chain - return nil unless defined?(OpenSSL) - if @env["SSL_CLIENT_CERT_CHAIN_0"] - keys = @env.keys - certs = keys.sort.collect{|k| - if /^SSL_CLIENT_CERT_CHAIN_\d+$/ =~ k - if pem = @env[k] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - } - certs.compact - end - end - - def cipher - return nil unless defined?(OpenSSL) - if cipher = @env["SSL_CIPHER"] - ret = [ cipher ] - ret << @env["SSL_PROTOCOL"] - ret << @env["SSL_CIPHER_USEKEYSIZE"] - ret << @env["SSL_CIPHER_ALGKEYSIZE"] - ret - end - end - end - end -end diff --git a/tool/lib/webrick/compat.rb b/tool/lib/webrick/compat.rb deleted file mode 100644 index c497a1933c..0000000000 --- a/tool/lib/webrick/compat.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: false -# -# compat.rb -- cross platform compatibility -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: compat.rb,v 1.6 2002/10/01 17:16:32 gotoyuzo Exp $ - -## -# System call error module used by webrick for cross platform compatibility. -# -# EPROTO:: protocol error -# ECONNRESET:: remote host reset the connection request -# ECONNABORTED:: Client sent TCP reset (RST) before server has accepted the -# connection requested by client. -# -module Errno - ## - # Protocol error. - - class EPROTO < SystemCallError; end - - ## - # Remote host reset the connection request. - - class ECONNRESET < SystemCallError; end - - ## - # Client sent TCP reset (RST) before server has accepted the connection - # requested by client. - - class ECONNABORTED < SystemCallError; end -end diff --git a/tool/lib/webrick/config.rb b/tool/lib/webrick/config.rb deleted file mode 100644 index 9f2ab44f49..0000000000 --- a/tool/lib/webrick/config.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: false -# -# config.rb -- Default configurations. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: config.rb,v 1.52 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'version' -require_relative 'httpversion' -require_relative 'httputils' -require_relative 'utils' -require_relative 'log' - -module WEBrick - module Config - LIBDIR = File::dirname(__FILE__) # :nodoc: - - # for GenericServer - General = Hash.new { |hash, key| - case key - when :ServerName - hash[key] = Utils.getservername - else - nil - end - }.update( - :BindAddress => nil, # "0.0.0.0" or "::" or nil - :Port => nil, # users MUST specify this!! - :MaxClients => 100, # maximum number of the concurrent connections - :ServerType => nil, # default: WEBrick::SimpleServer - :Logger => nil, # default: WEBrick::Log.new - :ServerSoftware => "WEBrick/#{WEBrick::VERSION} " + - "(Ruby/#{RUBY_VERSION}/#{RUBY_RELEASE_DATE})", - :TempDir => ENV['TMPDIR']||ENV['TMP']||ENV['TEMP']||'/tmp', - :DoNotListen => false, - :StartCallback => nil, - :StopCallback => nil, - :AcceptCallback => nil, - :DoNotReverseLookup => true, - :ShutdownSocketWithoutClose => false, - ) - - # for HTTPServer, HTTPRequest, HTTPResponse ... - HTTP = General.dup.update( - :Port => 80, - :RequestTimeout => 30, - :HTTPVersion => HTTPVersion.new("1.1"), - :AccessLog => nil, - :MimeTypes => HTTPUtils::DefaultMimeTypes, - :DirectoryIndex => ["index.html","index.htm","index.cgi","index.rhtml"], - :DocumentRoot => nil, - :DocumentRootOptions => { :FancyIndexing => true }, - :RequestCallback => nil, - :ServerAlias => nil, - :InputBufferSize => 65536, # input buffer size in reading request body - :OutputBufferSize => 65536, # output buffer size in sending File or IO - - # for HTTPProxyServer - :ProxyAuthProc => nil, - :ProxyContentHandler => nil, - :ProxyVia => true, - :ProxyTimeout => true, - :ProxyURI => nil, - - :CGIInterpreter => nil, - :CGIPathEnv => nil, - - # workaround: if Request-URIs contain 8bit chars, - # they should be escaped before calling of URI::parse(). - :Escape8bitURI => false - ) - - ## - # Default configuration for WEBrick::HTTPServlet::FileHandler - # - # :AcceptableLanguages:: - # Array of languages allowed for accept-language. There is no default - # :DirectoryCallback:: - # Allows preprocessing of directory requests. There is no default - # callback. - # :FancyIndexing:: - # If true, show an index for directories. The default is true. - # :FileCallback:: - # Allows preprocessing of file requests. There is no default callback. - # :HandlerCallback:: - # Allows preprocessing of requests. There is no default callback. - # :HandlerTable:: - # Maps file suffixes to file handlers. DefaultFileHandler is used by - # default but any servlet can be used. - # :NondisclosureName:: - # Do not show files matching this array of globs. .ht* and *~ are - # excluded by default. - # :UserDir:: - # Directory inside ~user to serve content from for /~user requests. - # Only works if mounted on /. Disabled by default. - - FileHandler = { - :NondisclosureName => [".ht*", "*~"], - :FancyIndexing => false, - :HandlerTable => {}, - :HandlerCallback => nil, - :DirectoryCallback => nil, - :FileCallback => nil, - :UserDir => nil, # e.g. "public_html" - :AcceptableLanguages => [] # ["en", "ja", ... ] - } - - ## - # Default configuration for WEBrick::HTTPAuth::BasicAuth - # - # :AutoReloadUserDB:: Reload the user database provided by :UserDB - # automatically? - - BasicAuth = { - :AutoReloadUserDB => true, - } - - ## - # Default configuration for WEBrick::HTTPAuth::DigestAuth. - # - # :Algorithm:: MD5, MD5-sess (default), SHA1, SHA1-sess - # :Domain:: An Array of URIs that define the protected space - # :Qop:: 'auth' for authentication, 'auth-int' for integrity protection or - # both - # :UseOpaque:: Should the server send opaque values to the client? This - # helps prevent replay attacks. - # :CheckNc:: Should the server check the nonce count? This helps the - # server detect replay attacks. - # :UseAuthenticationInfoHeader:: Should the server send an - # AuthenticationInfo header? - # :AutoReloadUserDB:: Reload the user database provided by :UserDB - # automatically? - # :NonceExpirePeriod:: How long should we store used nonces? Default is - # 30 minutes. - # :NonceExpireDelta:: How long is a nonce valid? Default is 1 minute - # :InternetExplorerHack:: Hack which allows Internet Explorer to work. - # :OperaHack:: Hack which allows Opera to work. - - DigestAuth = { - :Algorithm => 'MD5-sess', # or 'MD5' - :Domain => nil, # an array includes domain names. - :Qop => [ 'auth' ], # 'auth' or 'auth-int' or both. - :UseOpaque => true, - :UseNextNonce => false, - :CheckNc => false, - :UseAuthenticationInfoHeader => true, - :AutoReloadUserDB => true, - :NonceExpirePeriod => 30*60, - :NonceExpireDelta => 60, - :InternetExplorerHack => true, - :OperaHack => true, - } - end -end diff --git a/tool/lib/webrick/cookie.rb b/tool/lib/webrick/cookie.rb deleted file mode 100644 index 5fd3bfb228..0000000000 --- a/tool/lib/webrick/cookie.rb +++ /dev/null @@ -1,172 +0,0 @@ -# frozen_string_literal: false -# -# cookie.rb -- Cookie class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cookie.rb,v 1.16 2002/09/21 12:23:35 gotoyuzo Exp $ - -require 'time' -require_relative 'httputils' - -module WEBrick - - ## - # Processes HTTP cookies - - class Cookie - - ## - # The cookie name - - attr_reader :name - - ## - # The cookie value - - attr_accessor :value - - ## - # The cookie version - - attr_accessor :version - - ## - # The cookie domain - attr_accessor :domain - - ## - # The cookie path - - attr_accessor :path - - ## - # Is this a secure cookie? - - attr_accessor :secure - - ## - # The cookie comment - - attr_accessor :comment - - ## - # The maximum age of the cookie - - attr_accessor :max_age - - #attr_accessor :comment_url, :discard, :port - - ## - # Creates a new cookie with the given +name+ and +value+ - - def initialize(name, value) - @name = name - @value = value - @version = 0 # Netscape Cookie - - @domain = @path = @secure = @comment = @max_age = - @expires = @comment_url = @discard = @port = nil - end - - ## - # Sets the cookie expiration to the time +t+. The expiration time may be - # a false value to disable expiration or a Time or HTTP format time string - # to set the expiration date. - - def expires=(t) - @expires = t && (t.is_a?(Time) ? t.httpdate : t.to_s) - end - - ## - # Retrieves the expiration time as a Time - - def expires - @expires && Time.parse(@expires) - end - - ## - # The cookie string suitable for use in an HTTP header - - def to_s - ret = "" - ret << @name << "=" << @value - ret << "; " << "Version=" << @version.to_s if @version > 0 - ret << "; " << "Domain=" << @domain if @domain - ret << "; " << "Expires=" << @expires if @expires - ret << "; " << "Max-Age=" << @max_age.to_s if @max_age - ret << "; " << "Comment=" << @comment if @comment - ret << "; " << "Path=" << @path if @path - ret << "; " << "Secure" if @secure - ret - end - - ## - # Parses a Cookie field sent from the user-agent. Returns an array of - # cookies. - - def self.parse(str) - if str - ret = [] - cookie = nil - ver = 0 - str.split(/;\s+/).each{|x| - key, val = x.split(/=/,2) - val = val ? HTTPUtils::dequote(val) : "" - case key - when "$Version"; ver = val.to_i - when "$Path"; cookie.path = val - when "$Domain"; cookie.domain = val - when "$Port"; cookie.port = val - else - ret << cookie if cookie - cookie = self.new(key, val) - cookie.version = ver - end - } - ret << cookie if cookie - ret - end - end - - ## - # Parses the cookie in +str+ - - def self.parse_set_cookie(str) - cookie_elem = str.split(/;/) - first_elem = cookie_elem.shift - first_elem.strip! - key, value = first_elem.split(/=/, 2) - cookie = new(key, HTTPUtils.dequote(value)) - cookie_elem.each{|pair| - pair.strip! - key, value = pair.split(/=/, 2) - if value - value = HTTPUtils.dequote(value.strip) - end - case key.downcase - when "domain" then cookie.domain = value - when "path" then cookie.path = value - when "expires" then cookie.expires = value - when "max-age" then cookie.max_age = Integer(value) - when "comment" then cookie.comment = value - when "version" then cookie.version = Integer(value) - when "secure" then cookie.secure = true - end - } - return cookie - end - - ## - # Parses the cookies in +str+ - - def self.parse_set_cookies(str) - return str.split(/,(?=[^;,]*=)|,$/).collect{|c| - parse_set_cookie(c) - } - end - end -end diff --git a/tool/lib/webrick/htmlutils.rb b/tool/lib/webrick/htmlutils.rb deleted file mode 100644 index ed9f4ac0d3..0000000000 --- a/tool/lib/webrick/htmlutils.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: false -#-- -# htmlutils.rb -- HTMLUtils Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htmlutils.rb,v 1.7 2002/09/21 12:23:35 gotoyuzo Exp $ - -module WEBrick - module HTMLUtils - - ## - # Escapes &, ", > and < in +string+ - - def escape(string) - return "" unless string - str = string.b - str.gsub!(/&/n, '&') - str.gsub!(/\"/n, '"') - str.gsub!(/>/n, '>') - str.gsub!(/</n, '<') - str.force_encoding(string.encoding) - end - module_function :escape - - end -end diff --git a/tool/lib/webrick/httpauth.rb b/tool/lib/webrick/httpauth.rb deleted file mode 100644 index f8bf09a6f1..0000000000 --- a/tool/lib/webrick/httpauth.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: false -# -# httpauth.rb -- HTTP access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpauth.rb,v 1.14 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'httpauth/basicauth' -require_relative 'httpauth/digestauth' -require_relative 'httpauth/htpasswd' -require_relative 'httpauth/htdigest' -require_relative 'httpauth/htgroup' - -module WEBrick - - ## - # HTTPAuth provides both basic and digest authentication. - # - # To enable authentication for requests in WEBrick you will need a user - # database and an authenticator. To start, here's an Htpasswd database for - # use with a DigestAuth authenticator: - # - # config = { :Realm => 'DigestAuth example realm' } - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' - # htpasswd.auth_type = WEBrick::HTTPAuth::DigestAuth - # htpasswd.set_passwd config[:Realm], 'username', 'password' - # htpasswd.flush - # - # The +:Realm+ is used to provide different access to different groups - # across several resources on a server. Typically you'll need only one - # realm for a server. - # - # This database can be used to create an authenticator: - # - # config[:UserDB] = htpasswd - # - # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config - # - # To authenticate a request call #authenticate with a request and response - # object in a servlet: - # - # def do_GET req, res - # @authenticator.authenticate req, res - # end - # - # For digest authentication the authenticator must not be created every - # request, it must be passed in as an option via WEBrick::HTTPServer#mount. - - module HTTPAuth - module_function - - def _basic_auth(req, res, realm, req_field, res_field, err_type, - block) # :nodoc: - user = pass = nil - if /^Basic\s+(.*)/o =~ req[req_field] - userpass = $1 - user, pass = userpass.unpack("m*")[0].split(":", 2) - end - if block.call(user, pass) - req.user = user - return - end - res[res_field] = "Basic realm=\"#{realm}\"" - raise err_type - end - - ## - # Simple wrapper for providing basic authentication for a request. When - # called with a request +req+, response +res+, authentication +realm+ and - # +block+ the block will be called with a +username+ and +password+. If - # the block returns true the request is allowed to continue, otherwise an - # HTTPStatus::Unauthorized error is raised. - - def basic_auth(req, res, realm, &block) # :yield: username, password - _basic_auth(req, res, realm, "Authorization", "WWW-Authenticate", - HTTPStatus::Unauthorized, block) - end - - ## - # Simple wrapper for providing basic authentication for a proxied request. - # When called with a request +req+, response +res+, authentication +realm+ - # and +block+ the block will be called with a +username+ and +password+. - # If the block returns true the request is allowed to continue, otherwise - # an HTTPStatus::ProxyAuthenticationRequired error is raised. - - def proxy_basic_auth(req, res, realm, &block) # :yield: username, password - _basic_auth(req, res, realm, "Proxy-Authorization", "Proxy-Authenticate", - HTTPStatus::ProxyAuthenticationRequired, block) - end - end -end diff --git a/tool/lib/webrick/httpauth/authenticator.rb b/tool/lib/webrick/httpauth/authenticator.rb deleted file mode 100644 index 8f0eaa3aca..0000000000 --- a/tool/lib/webrick/httpauth/authenticator.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: false -#-- -# httpauth/authenticator.rb -- Authenticator mix-in module. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: authenticator.rb,v 1.3 2003/02/20 07:15:47 gotoyuzo Exp $ - -module WEBrick - module HTTPAuth - - ## - # Module providing generic support for both Digest and Basic - # authentication schemes. - - module Authenticator - - RequestField = "Authorization" # :nodoc: - ResponseField = "WWW-Authenticate" # :nodoc: - ResponseInfoField = "Authentication-Info" # :nodoc: - AuthException = HTTPStatus::Unauthorized # :nodoc: - - ## - # Method of authentication, must be overridden by the including class - - AuthScheme = nil - - ## - # The realm this authenticator covers - - attr_reader :realm - - ## - # The user database for this authenticator - - attr_reader :userdb - - ## - # The logger for this authenticator - - attr_reader :logger - - private - - # :stopdoc: - - ## - # Initializes the authenticator from +config+ - - def check_init(config) - [:UserDB, :Realm].each{|sym| - unless config[sym] - raise ArgumentError, "Argument #{sym.inspect} missing." - end - } - @realm = config[:Realm] - @userdb = config[:UserDB] - @logger = config[:Logger] || Log::new($stderr) - @reload_db = config[:AutoReloadUserDB] - @request_field = self::class::RequestField - @response_field = self::class::ResponseField - @resp_info_field = self::class::ResponseInfoField - @auth_exception = self::class::AuthException - @auth_scheme = self::class::AuthScheme - end - - ## - # Ensures +req+ has credentials that can be authenticated. - - def check_scheme(req) - unless credentials = req[@request_field] - error("no credentials in the request.") - return nil - end - unless match = /^#{@auth_scheme}\s+/i.match(credentials) - error("invalid scheme in %s.", credentials) - info("%s: %s", @request_field, credentials) if $DEBUG - return nil - end - return match.post_match - end - - def log(meth, fmt, *args) - msg = format("%s %s: ", @auth_scheme, @realm) - msg << fmt % args - @logger.__send__(meth, msg) - end - - def error(fmt, *args) - if @logger.error? - log(:error, fmt, *args) - end - end - - def info(fmt, *args) - if @logger.info? - log(:info, fmt, *args) - end - end - - # :startdoc: - end - - ## - # Module providing generic support for both Digest and Basic - # authentication schemes for proxies. - - module ProxyAuthenticator - RequestField = "Proxy-Authorization" # :nodoc: - ResponseField = "Proxy-Authenticate" # :nodoc: - InfoField = "Proxy-Authentication-Info" # :nodoc: - AuthException = HTTPStatus::ProxyAuthenticationRequired # :nodoc: - end - end -end diff --git a/tool/lib/webrick/httpauth/basicauth.rb b/tool/lib/webrick/httpauth/basicauth.rb deleted file mode 100644 index 7d0a9cfc8f..0000000000 --- a/tool/lib/webrick/httpauth/basicauth.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/basicauth.rb -- HTTP basic access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: basicauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ - -require_relative '../config' -require_relative '../httpstatus' -require_relative 'authenticator' - -module WEBrick - module HTTPAuth - - ## - # Basic Authentication for WEBrick - # - # Use this class to add basic authentication to a WEBrick servlet. - # - # Here is an example of how to set up a BasicAuth: - # - # config = { :Realm => 'BasicAuth example realm' } - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file', password_hash: :bcrypt - # htpasswd.set_passwd config[:Realm], 'username', 'password' - # htpasswd.flush - # - # config[:UserDB] = htpasswd - # - # basic_auth = WEBrick::HTTPAuth::BasicAuth.new config - - class BasicAuth - include Authenticator - - AuthScheme = "Basic" # :nodoc: - - ## - # Used by UserDB to create a basic password entry - - def self.make_passwd(realm, user, pass) - pass ||= "" - pass.crypt(Utils::random_string(2)) - end - - attr_reader :realm, :userdb, :logger - - ## - # Creates a new BasicAuth instance. - # - # See WEBrick::Config::BasicAuth for default configuration entries - # - # You must supply the following configuration entries: - # - # :Realm:: The name of the realm being protected. - # :UserDB:: A database of usernames and passwords. - # A WEBrick::HTTPAuth::Htpasswd instance should be used. - - def initialize(config, default=Config::BasicAuth) - check_init(config) - @config = default.dup.update(config) - end - - ## - # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if - # the authentication was not correct. - - def authenticate(req, res) - unless basic_credentials = check_scheme(req) - challenge(req, res) - end - userid, password = basic_credentials.unpack("m*")[0].split(":", 2) - password ||= "" - if userid.empty? - error("user id was not given.") - challenge(req, res) - end - unless encpass = @userdb.get_passwd(@realm, userid, @reload_db) - error("%s: the user is not allowed.", userid) - challenge(req, res) - end - - case encpass - when /\A\$2[aby]\$/ - password_matches = BCrypt::Password.new(encpass.sub(/\A\$2[aby]\$/, '$2a$')) == password - else - password_matches = password.crypt(encpass) == encpass - end - - unless password_matches - error("%s: password unmatch.", userid) - challenge(req, res) - end - info("%s: authentication succeeded.", userid) - req.user = userid - end - - ## - # Returns a challenge response which asks for authentication information - - def challenge(req, res) - res[@response_field] = "#{@auth_scheme} realm=\"#{@realm}\"" - raise @auth_exception - end - end - - ## - # Basic authentication for proxy servers. See BasicAuth for details. - - class ProxyBasicAuth < BasicAuth - include ProxyAuthenticator - end - end -end diff --git a/tool/lib/webrick/httpauth/digestauth.rb b/tool/lib/webrick/httpauth/digestauth.rb deleted file mode 100644 index 3cf12899d2..0000000000 --- a/tool/lib/webrick/httpauth/digestauth.rb +++ /dev/null @@ -1,395 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/digestauth.rb -- HTTP digest access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. -# Copyright (c) 2003 H.M. -# -# The original implementation is provided by H.M. -# URL: http://rwiki.jin.gr.jp/cgi-bin/rw-cgi.rb?cmd=view;name= -# %C7%A7%BE%DA%B5%A1%C7%BD%A4%F2%B2%FE%C2%A4%A4%B7%A4%C6%A4%DF%A4%EB -# -# $IPR: digestauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ - -require_relative '../config' -require_relative '../httpstatus' -require_relative 'authenticator' -require 'digest/md5' -require 'digest/sha1' - -module WEBrick - module HTTPAuth - - ## - # RFC 2617 Digest Access Authentication for WEBrick - # - # Use this class to add digest authentication to a WEBrick servlet. - # - # Here is an example of how to set up DigestAuth: - # - # config = { :Realm => 'DigestAuth example realm' } - # - # htdigest = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' - # htdigest.set_passwd config[:Realm], 'username', 'password' - # htdigest.flush - # - # config[:UserDB] = htdigest - # - # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config - # - # When using this as with a servlet be sure not to create a new DigestAuth - # object in the servlet's #initialize. By default WEBrick creates a new - # servlet instance for every request and the DigestAuth object must be - # used across requests. - - class DigestAuth - include Authenticator - - AuthScheme = "Digest" # :nodoc: - - ## - # Struct containing the opaque portion of the digest authentication - - OpaqueInfo = Struct.new(:time, :nonce, :nc) # :nodoc: - - ## - # Digest authentication algorithm - - attr_reader :algorithm - - ## - # Quality of protection. RFC 2617 defines "auth" and "auth-int" - - attr_reader :qop - - ## - # Used by UserDB to create a digest password entry - - def self.make_passwd(realm, user, pass) - pass ||= "" - Digest::MD5::hexdigest([user, realm, pass].join(":")) - end - - ## - # Creates a new DigestAuth instance. Be sure to use the same DigestAuth - # instance for multiple requests as it saves state between requests in - # order to perform authentication. - # - # See WEBrick::Config::DigestAuth for default configuration entries - # - # You must supply the following configuration entries: - # - # :Realm:: The name of the realm being protected. - # :UserDB:: A database of usernames and passwords. - # A WEBrick::HTTPAuth::Htdigest instance should be used. - - def initialize(config, default=Config::DigestAuth) - check_init(config) - @config = default.dup.update(config) - @algorithm = @config[:Algorithm] - @domain = @config[:Domain] - @qop = @config[:Qop] - @use_opaque = @config[:UseOpaque] - @use_next_nonce = @config[:UseNextNonce] - @check_nc = @config[:CheckNc] - @use_auth_info_header = @config[:UseAuthenticationInfoHeader] - @nonce_expire_period = @config[:NonceExpirePeriod] - @nonce_expire_delta = @config[:NonceExpireDelta] - @internet_explorer_hack = @config[:InternetExplorerHack] - - case @algorithm - when 'MD5','MD5-sess' - @h = Digest::MD5 - when 'SHA1','SHA1-sess' # it is a bonus feature :-) - @h = Digest::SHA1 - else - msg = format('Algorithm "%s" is not supported.', @algorithm) - raise ArgumentError.new(msg) - end - - @instance_key = hexdigest(self.__id__, Time.now.to_i, Process.pid) - @opaques = {} - @last_nonce_expire = Time.now - @mutex = Thread::Mutex.new - end - - ## - # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if - # the authentication was not correct. - - def authenticate(req, res) - unless result = @mutex.synchronize{ _authenticate(req, res) } - challenge(req, res) - end - if result == :nonce_is_stale - challenge(req, res, true) - end - return true - end - - ## - # Returns a challenge response which asks for authentication information - - def challenge(req, res, stale=false) - nonce = generate_next_nonce(req) - if @use_opaque - opaque = generate_opaque(req) - @opaques[opaque].nonce = nonce - end - - param = Hash.new - param["realm"] = HTTPUtils::quote(@realm) - param["domain"] = HTTPUtils::quote(@domain.to_a.join(" ")) if @domain - param["nonce"] = HTTPUtils::quote(nonce) - param["opaque"] = HTTPUtils::quote(opaque) if opaque - param["stale"] = stale.to_s - param["algorithm"] = @algorithm - param["qop"] = HTTPUtils::quote(@qop.to_a.join(",")) if @qop - - res[@response_field] = - "#{@auth_scheme} " + param.map{|k,v| "#{k}=#{v}" }.join(", ") - info("%s: %s", @response_field, res[@response_field]) if $DEBUG - raise @auth_exception - end - - private - - # :stopdoc: - - MustParams = ['username','realm','nonce','uri','response'] - MustParamsAuth = ['cnonce','nc'] - - def _authenticate(req, res) - unless digest_credentials = check_scheme(req) - return false - end - - auth_req = split_param_value(digest_credentials) - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - req_params = MustParams + MustParamsAuth - else - req_params = MustParams - end - req_params.each{|key| - unless auth_req.has_key?(key) - error('%s: parameter missing. "%s"', auth_req['username'], key) - raise HTTPStatus::BadRequest - end - } - - if !check_uri(req, auth_req) - raise HTTPStatus::BadRequest - end - - if auth_req['realm'] != @realm - error('%s: realm unmatch. "%s" for "%s"', - auth_req['username'], auth_req['realm'], @realm) - return false - end - - auth_req['algorithm'] ||= 'MD5' - if auth_req['algorithm'].upcase != @algorithm.upcase - error('%s: algorithm unmatch. "%s" for "%s"', - auth_req['username'], auth_req['algorithm'], @algorithm) - return false - end - - if (@qop.nil? && auth_req.has_key?('qop')) || - (@qop && (! @qop.member?(auth_req['qop']))) - error('%s: the qop is not allowed. "%s"', - auth_req['username'], auth_req['qop']) - return false - end - - password = @userdb.get_passwd(@realm, auth_req['username'], @reload_db) - unless password - error('%s: the user is not allowed.', auth_req['username']) - return false - end - - nonce_is_invalid = false - if @use_opaque - info("@opaque = %s", @opaque.inspect) if $DEBUG - if !(opaque = auth_req['opaque']) - error('%s: opaque is not given.', auth_req['username']) - nonce_is_invalid = true - elsif !(opaque_struct = @opaques[opaque]) - error('%s: invalid opaque is given.', auth_req['username']) - nonce_is_invalid = true - elsif !check_opaque(opaque_struct, req, auth_req) - @opaques.delete(auth_req['opaque']) - nonce_is_invalid = true - end - elsif !check_nonce(req, auth_req) - nonce_is_invalid = true - end - - if /-sess$/i =~ auth_req['algorithm'] - ha1 = hexdigest(password, auth_req['nonce'], auth_req['cnonce']) - else - ha1 = password - end - - if auth_req['qop'] == "auth" || auth_req['qop'] == nil - ha2 = hexdigest(req.request_method, auth_req['uri']) - ha2_res = hexdigest("", auth_req['uri']) - elsif auth_req['qop'] == "auth-int" - body_digest = @h.new - req.body { |chunk| body_digest.update(chunk) } - body_digest = body_digest.hexdigest - ha2 = hexdigest(req.request_method, auth_req['uri'], body_digest) - ha2_res = hexdigest("", auth_req['uri'], body_digest) - end - - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - param2 = ['nonce', 'nc', 'cnonce', 'qop'].map{|key| - auth_req[key] - }.join(':') - digest = hexdigest(ha1, param2, ha2) - digest_res = hexdigest(ha1, param2, ha2_res) - else - digest = hexdigest(ha1, auth_req['nonce'], ha2) - digest_res = hexdigest(ha1, auth_req['nonce'], ha2_res) - end - - if digest != auth_req['response'] - error("%s: digest unmatch.", auth_req['username']) - return false - elsif nonce_is_invalid - error('%s: digest is valid, but nonce is not valid.', - auth_req['username']) - return :nonce_is_stale - elsif @use_auth_info_header - auth_info = { - 'nextnonce' => generate_next_nonce(req), - 'rspauth' => digest_res - } - if @use_opaque - opaque_struct.time = req.request_time - opaque_struct.nonce = auth_info['nextnonce'] - opaque_struct.nc = "%08x" % (auth_req['nc'].hex + 1) - end - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - ['qop','cnonce','nc'].each{|key| - auth_info[key] = auth_req[key] - } - end - res[@resp_info_field] = auth_info.keys.map{|key| - if key == 'nc' - key + '=' + auth_info[key] - else - key + "=" + HTTPUtils::quote(auth_info[key]) - end - }.join(', ') - end - info('%s: authentication succeeded.', auth_req['username']) - req.user = auth_req['username'] - return true - end - - def split_param_value(string) - ret = {} - string.scan(/\G\s*([\w\-.*%!]+)=\s*(?:\"((?>\\.|[^\"])*)\"|([^,\"]*))\s*,?/) do - ret[$1] = $3 || $2.gsub(/\\(.)/, "\\1") - end - ret - end - - def generate_next_nonce(req) - now = "%012d" % req.request_time.to_i - pk = hexdigest(now, @instance_key)[0,32] - nonce = [now + ":" + pk].pack("m0") # it has 60 length of chars. - nonce - end - - def check_nonce(req, auth_req) - username = auth_req['username'] - nonce = auth_req['nonce'] - - pub_time, pk = nonce.unpack("m*")[0].split(":", 2) - if (!pub_time || !pk) - error("%s: empty nonce is given", username) - return false - elsif (hexdigest(pub_time, @instance_key)[0,32] != pk) - error("%s: invalid private-key: %s for %s", - username, hexdigest(pub_time, @instance_key)[0,32], pk) - return false - end - - diff_time = req.request_time.to_i - pub_time.to_i - if (diff_time < 0) - error("%s: difference of time-stamp is negative.", username) - return false - elsif diff_time > @nonce_expire_period - error("%s: nonce is expired.", username) - return false - end - - return true - end - - def generate_opaque(req) - @mutex.synchronize{ - now = req.request_time - if now - @last_nonce_expire > @nonce_expire_delta - @opaques.delete_if{|key,val| - (now - val.time) > @nonce_expire_period - } - @last_nonce_expire = now - end - begin - opaque = Utils::random_string(16) - end while @opaques[opaque] - @opaques[opaque] = OpaqueInfo.new(now, nil, '00000001') - opaque - } - end - - def check_opaque(opaque_struct, req, auth_req) - if (@use_next_nonce && auth_req['nonce'] != opaque_struct.nonce) - error('%s: nonce unmatched. "%s" for "%s"', - auth_req['username'], auth_req['nonce'], opaque_struct.nonce) - return false - elsif !check_nonce(req, auth_req) - return false - end - if (@check_nc && auth_req['nc'] != opaque_struct.nc) - error('%s: nc unmatched."%s" for "%s"', - auth_req['username'], auth_req['nc'], opaque_struct.nc) - return false - end - true - end - - def check_uri(req, auth_req) - uri = auth_req['uri'] - if uri != req.request_uri.to_s && uri != req.unparsed_uri && - (@internet_explorer_hack && uri != req.path) - error('%s: uri unmatch. "%s" for "%s"', auth_req['username'], - auth_req['uri'], req.request_uri.to_s) - return false - end - true - end - - def hexdigest(*args) - @h.hexdigest(args.join(":")) - end - - # :startdoc: - end - - ## - # Digest authentication for proxy servers. See DigestAuth for details. - - class ProxyDigestAuth < DigestAuth - include ProxyAuthenticator - - private - def check_uri(req, auth_req) # :nodoc: - return true - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htdigest.rb b/tool/lib/webrick/httpauth/htdigest.rb deleted file mode 100644 index 93b18e2c75..0000000000 --- a/tool/lib/webrick/httpauth/htdigest.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htdigest.rb -- Apache compatible htdigest file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htdigest.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ - -require_relative 'userdb' -require_relative 'digestauth' -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htdigest accesses apache-compatible digest password files. Passwords are - # matched to a realm where they are valid. For security, the path for a - # digest password database should be stored outside of the paths available - # to the HTTP server. - # - # Htdigest is intended for use with WEBrick::HTTPAuth::DigestAuth and - # stores passwords using cryptographic hashes. - # - # htpasswd = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' - # htpasswd.set_passwd 'my realm', 'username', 'password' - # htpasswd.flush - - class Htdigest - include UserDB - - ## - # Open a digest password database at +path+ - - def initialize(path) - @path = path - @mtime = Time.at(0) - @digest = Hash.new - @mutex = Thread::Mutex::new - @auth_type = DigestAuth - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reloads passwords from the database - - def reload - mtime = File::mtime(@path) - if mtime > @mtime - @digest.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - user, realm, pass = line.split(/:/, 3) - unless @digest[realm] - @digest[realm] = Hash.new - end - @digest[realm][user] = pass - end - } - @mtime = mtime - end - end - - ## - # Flush the password database. If +output+ is given the database will - # be written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htpasswd", File::dirname(output)) - renamed = false - begin - each{|item| tmp.puts(item.join(":")) } - tmp.close - File::rename(tmp.path, output) - renamed = true - ensure - tmp.close - File.unlink(tmp.path) if !renamed - end - end - - ## - # Retrieves a password from the database for +user+ in +realm+. If - # +reload_db+ is true the database will be reloaded first. - - def get_passwd(realm, user, reload_db) - reload() if reload_db - if hash = @digest[realm] - hash[user] - end - end - - ## - # Sets a password in the database for +user+ in +realm+ to +pass+. - - def set_passwd(realm, user, pass) - @mutex.synchronize{ - unless @digest[realm] - @digest[realm] = Hash.new - end - @digest[realm][user] = make_passwd(realm, user, pass) - } - end - - ## - # Removes a password from the database for +user+ in +realm+. - - def delete_passwd(realm, user) - if hash = @digest[realm] - hash.delete(user) - end - end - - ## - # Iterate passwords in the database. - - def each # :yields: [user, realm, password_hash] - @digest.keys.sort.each{|realm| - hash = @digest[realm] - hash.keys.sort.each{|user| - yield([user, realm, hash[user]]) - } - } - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htgroup.rb b/tool/lib/webrick/httpauth/htgroup.rb deleted file mode 100644 index e06c441b18..0000000000 --- a/tool/lib/webrick/httpauth/htgroup.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htgroup.rb -- Apache compatible htgroup file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htgroup.rb,v 1.1 2003/02/16 22:22:56 gotoyuzo Exp $ - -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htgroup accesses apache-compatible group files. Htgroup can be used to - # provide group-based authentication for users. Currently Htgroup is not - # directly integrated with any authenticators in WEBrick. For security, - # the path for a digest password database should be stored outside of the - # paths available to the HTTP server. - # - # Example: - # - # htgroup = WEBrick::HTTPAuth::Htgroup.new 'my_group_file' - # htgroup.add 'superheroes', %w[spiderman batman] - # - # htgroup.members('superheroes').include? 'magneto' # => false - - class Htgroup - - ## - # Open a group database at +path+ - - def initialize(path) - @path = path - @mtime = Time.at(0) - @group = Hash.new - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reload groups from the database - - def reload - if (mtime = File::mtime(@path)) > @mtime - @group.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - group, members = line.split(/:\s*/) - @group[group] = members.split(/\s+/) - end - } - @mtime = mtime - end - end - - ## - # Flush the group database. If +output+ is given the database will be - # written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htgroup", File::dirname(output)) - begin - @group.keys.sort.each{|group| - tmp.puts(format("%s: %s", group, self.members(group).join(" "))) - } - ensure - tmp.close - if $! - File.unlink(tmp.path) - else - return File.rename(tmp.path, output) - end - end - end - - ## - # Retrieve the list of members from +group+ - - def members(group) - reload - @group[group] || [] - end - - ## - # Add an Array of +members+ to +group+ - - def add(group, members) - @group[group] = members(group) | members - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htpasswd.rb b/tool/lib/webrick/httpauth/htpasswd.rb deleted file mode 100644 index abca30532e..0000000000 --- a/tool/lib/webrick/httpauth/htpasswd.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htpasswd -- Apache compatible htpasswd file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htpasswd.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ - -require_relative 'userdb' -require_relative 'basicauth' -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htpasswd accesses apache-compatible password files. Passwords are - # matched to a realm where they are valid. For security, the path for a - # password database should be stored outside of the paths available to the - # HTTP server. - # - # Htpasswd is intended for use with WEBrick::HTTPAuth::BasicAuth. - # - # To create an Htpasswd database with a single user: - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' - # htpasswd.set_passwd 'my realm', 'username', 'password' - # htpasswd.flush - - class Htpasswd - include UserDB - - ## - # Open a password database at +path+ - - def initialize(path, password_hash: nil) - @path = path - @mtime = Time.at(0) - @passwd = Hash.new - @auth_type = BasicAuth - @password_hash = password_hash - - case @password_hash - when nil - # begin - # require "string/crypt" - # rescue LoadError - # warn("Unable to load string/crypt, proceeding with deprecated use of String#crypt, consider using password_hash: :bcrypt") - # end - @password_hash = :crypt - when :crypt - # require "string/crypt" - when :bcrypt - require "bcrypt" - else - raise ArgumentError, "only :crypt and :bcrypt are supported for password_hash keyword argument" - end - - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reload passwords from the database - - def reload - mtime = File::mtime(@path) - if mtime > @mtime - @passwd.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - case line - when %r!\A[^:]+:[a-zA-Z0-9./]{13}\z! - if @password_hash == :bcrypt - raise StandardError, ".htpasswd file contains crypt password, only bcrypt passwords supported" - end - user, pass = line.split(":") - when %r!\A[^:]+:\$2[aby]\$\d{2}\$.{53}\z! - if @password_hash == :crypt - raise StandardError, ".htpasswd file contains bcrypt password, only crypt passwords supported" - end - user, pass = line.split(":") - when /:\$/, /:{SHA}/ - raise NotImplementedError, - 'MD5, SHA1 .htpasswd file not supported' - else - raise StandardError, 'bad .htpasswd file' - end - @passwd[user] = pass - end - } - @mtime = mtime - end - end - - ## - # Flush the password database. If +output+ is given the database will - # be written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htpasswd", File::dirname(output)) - renamed = false - begin - each{|item| tmp.puts(item.join(":")) } - tmp.close - File::rename(tmp.path, output) - renamed = true - ensure - tmp.close - File.unlink(tmp.path) if !renamed - end - end - - ## - # Retrieves a password from the database for +user+ in +realm+. If - # +reload_db+ is true the database will be reloaded first. - - def get_passwd(realm, user, reload_db) - reload() if reload_db - @passwd[user] - end - - ## - # Sets a password in the database for +user+ in +realm+ to +pass+. - - def set_passwd(realm, user, pass) - if @password_hash == :bcrypt - # Cost of 5 to match Apache default, and because the - # bcrypt default of 10 will introduce significant delays - # for every request. - @passwd[user] = BCrypt::Password.create(pass, :cost=>5) - else - @passwd[user] = make_passwd(realm, user, pass) - end - end - - ## - # Removes a password from the database for +user+ in +realm+. - - def delete_passwd(realm, user) - @passwd.delete(user) - end - - ## - # Iterate passwords in the database. - - def each # :yields: [user, password] - @passwd.keys.sort.each{|user| - yield([user, @passwd[user]]) - } - end - end - end -end diff --git a/tool/lib/webrick/httpauth/userdb.rb b/tool/lib/webrick/httpauth/userdb.rb deleted file mode 100644 index 7a17715cdf..0000000000 --- a/tool/lib/webrick/httpauth/userdb.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: false -#-- -# httpauth/userdb.rb -- UserDB mix-in module. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: userdb.rb,v 1.2 2003/02/20 07:15:48 gotoyuzo Exp $ - -module WEBrick - module HTTPAuth - - ## - # User database mixin for HTTPAuth. This mixin dispatches user record - # access to the underlying auth_type for this database. - - module UserDB - - ## - # The authentication type. - # - # WEBrick::HTTPAuth::BasicAuth or WEBrick::HTTPAuth::DigestAuth are - # built-in. - - attr_accessor :auth_type - - ## - # Creates an obscured password in +realm+ with +user+ and +password+ - # using the auth_type of this database. - - def make_passwd(realm, user, pass) - @auth_type::make_passwd(realm, user, pass) - end - - ## - # Sets a password in +realm+ with +user+ and +password+ for the - # auth_type of this database. - - def set_passwd(realm, user, pass) - self[user] = pass - end - - ## - # Retrieves a password in +realm+ for +user+ for the auth_type of this - # database. +reload_db+ is a dummy value. - - def get_passwd(realm, user, reload_db=false) - make_passwd(realm, user, self[user]) - end - end - end -end diff --git a/tool/lib/webrick/httpproxy.rb b/tool/lib/webrick/httpproxy.rb deleted file mode 100644 index 7607c3df88..0000000000 --- a/tool/lib/webrick/httpproxy.rb +++ /dev/null @@ -1,354 +0,0 @@ -# frozen_string_literal: false -# -# httpproxy.rb -- HTTPProxy Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 GOTO Kentaro -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpproxy.rb,v 1.18 2003/03/08 18:58:10 gotoyuzo Exp $ -# $kNotwork: straw.rb,v 1.3 2002/02/12 15:13:07 gotoken Exp $ - -require_relative "httpserver" -require "net/http" - -module WEBrick - - NullReader = Object.new # :nodoc: - class << NullReader # :nodoc: - def read(*args) - nil - end - alias gets read - end - - FakeProxyURI = Object.new # :nodoc: - class << FakeProxyURI # :nodoc: - def method_missing(meth, *args) - if %w(scheme host port path query userinfo).member?(meth.to_s) - return nil - end - super - end - end - - # :startdoc: - - ## - # An HTTP Proxy server which proxies GET, HEAD and POST requests. - # - # To create a simple proxy server: - # - # require 'webrick' - # require 'webrick/httpproxy' - # - # proxy = WEBrick::HTTPProxyServer.new Port: 8000 - # - # trap 'INT' do proxy.shutdown end - # trap 'TERM' do proxy.shutdown end - # - # proxy.start - # - # See ::new for proxy-specific configuration items. - # - # == Modifying proxied responses - # - # To modify content the proxy server returns use the +:ProxyContentHandler+ - # option: - # - # handler = proc do |req, res| - # if res['content-type'] == 'text/plain' then - # res.body << "\nThis content was proxied!\n" - # end - # end - # - # proxy = - # WEBrick::HTTPProxyServer.new Port: 8000, ProxyContentHandler: handler - - class HTTPProxyServer < HTTPServer - - ## - # Proxy server configurations. The proxy server handles the following - # configuration items in addition to those supported by HTTPServer: - # - # :ProxyAuthProc:: Called with a request and response to authorize a - # request - # :ProxyVia:: Appended to the via header - # :ProxyURI:: The proxy server's URI - # :ProxyContentHandler:: Called with a request and response and allows - # modification of the response - # :ProxyTimeout:: Sets the proxy timeouts to 30 seconds for open and 60 - # seconds for read operations - - def initialize(config={}, default=Config::HTTP) - super(config, default) - c = @config - @via = "#{c[:HTTPVersion]} #{c[:ServerName]}:#{c[:Port]}" - end - - # :stopdoc: - def service(req, res) - if req.request_method == "CONNECT" - do_CONNECT(req, res) - elsif req.unparsed_uri =~ %r!^http://! - proxy_service(req, res) - else - super(req, res) - end - end - - def proxy_auth(req, res) - if proc = @config[:ProxyAuthProc] - proc.call(req, res) - end - req.header.delete("proxy-authorization") - end - - def proxy_uri(req, res) - # should return upstream proxy server's URI - return @config[:ProxyURI] - end - - def proxy_service(req, res) - # Proxy Authentication - proxy_auth(req, res) - - begin - public_send("do_#{req.request_method}", req, res) - rescue NoMethodError - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - rescue => err - logger.debug("#{err.class}: #{err.message}") - raise HTTPStatus::ServiceUnavailable, err.message - end - - # Process contents - if handler = @config[:ProxyContentHandler] - handler.call(req, res) - end - end - - def do_CONNECT(req, res) - # Proxy Authentication - proxy_auth(req, res) - - ua = Thread.current[:WEBrickSocket] # User-Agent - raise HTTPStatus::InternalServerError, - "[BUG] cannot get socket" unless ua - - host, port = req.unparsed_uri.split(":", 2) - # Proxy authentication for upstream proxy server - if proxy = proxy_uri(req, res) - proxy_request_line = "CONNECT #{host}:#{port} HTTP/1.0" - if proxy.userinfo - credentials = "Basic " + [proxy.userinfo].pack("m0") - end - host, port = proxy.host, proxy.port - end - - begin - @logger.debug("CONNECT: upstream proxy is `#{host}:#{port}'.") - os = TCPSocket.new(host, port) # origin server - - if proxy - @logger.debug("CONNECT: sending a Request-Line") - os << proxy_request_line << CRLF - @logger.debug("CONNECT: > #{proxy_request_line}") - if credentials - @logger.debug("CONNECT: sending credentials") - os << "Proxy-Authorization: " << credentials << CRLF - end - os << CRLF - proxy_status_line = os.gets(LF) - @logger.debug("CONNECT: read Status-Line from the upstream server") - @logger.debug("CONNECT: < #{proxy_status_line}") - if %r{^HTTP/\d+\.\d+\s+200\s*} =~ proxy_status_line - while line = os.gets(LF) - break if /\A(#{CRLF}|#{LF})\z/om =~ line - end - else - raise HTTPStatus::BadGateway - end - end - @logger.debug("CONNECT #{host}:#{port}: succeeded") - res.status = HTTPStatus::RC_OK - rescue => ex - @logger.debug("CONNECT #{host}:#{port}: failed `#{ex.message}'") - res.set_error(ex) - raise HTTPStatus::EOFError - ensure - if handler = @config[:ProxyContentHandler] - handler.call(req, res) - end - res.send_response(ua) - access_log(@config, req, res) - - # Should clear request-line not to send the response twice. - # see: HTTPServer#run - req.parse(NullReader) rescue nil - end - - begin - while fds = IO::select([ua, os]) - if fds[0].member?(ua) - buf = ua.readpartial(1024); - @logger.debug("CONNECT: #{buf.bytesize} byte from User-Agent") - os.write(buf) - elsif fds[0].member?(os) - buf = os.readpartial(1024); - @logger.debug("CONNECT: #{buf.bytesize} byte from #{host}:#{port}") - ua.write(buf) - end - end - rescue - os.close - @logger.debug("CONNECT #{host}:#{port}: closed") - end - - raise HTTPStatus::EOFError - end - - def do_GET(req, res) - perform_proxy_request(req, res, Net::HTTP::Get) - end - - def do_HEAD(req, res) - perform_proxy_request(req, res, Net::HTTP::Head) - end - - def do_POST(req, res) - perform_proxy_request(req, res, Net::HTTP::Post, req.body_reader) - end - - def do_OPTIONS(req, res) - res['allow'] = "GET,HEAD,POST,OPTIONS,CONNECT" - end - - private - - # Some header fields should not be transferred. - HopByHop = %w( connection keep-alive proxy-authenticate upgrade - proxy-authorization te trailers transfer-encoding ) - ShouldNotTransfer = %w( set-cookie proxy-connection ) - def split_field(f) f ? f.split(/,\s+/).collect{|i| i.downcase } : [] end - - def choose_header(src, dst) - connections = split_field(src['connection']) - src.each{|key, value| - key = key.downcase - if HopByHop.member?(key) || # RFC2616: 13.5.1 - connections.member?(key) || # RFC2616: 14.10 - ShouldNotTransfer.member?(key) # pragmatics - @logger.debug("choose_header: `#{key}: #{value}'") - next - end - dst[key] = value - } - end - - # Net::HTTP is stupid about the multiple header fields. - # Here is workaround: - def set_cookie(src, dst) - if str = src['set-cookie'] - cookies = [] - str.split(/,\s*/).each{|token| - if /^[^=]+;/o =~ token - cookies[-1] << ", " << token - elsif /=/o =~ token - cookies << token - else - cookies[-1] << ", " << token - end - } - dst.cookies.replace(cookies) - end - end - - def set_via(h) - if @config[:ProxyVia] - if h['via'] - h['via'] << ", " << @via - else - h['via'] = @via - end - end - end - - def setup_proxy_header(req, res) - # Choose header fields to transfer - header = Hash.new - choose_header(req, header) - set_via(header) - return header - end - - def setup_upstream_proxy_authentication(req, res, header) - if upstream = proxy_uri(req, res) - if upstream.userinfo - header['proxy-authorization'] = - "Basic " + [upstream.userinfo].pack("m0") - end - return upstream - end - return FakeProxyURI - end - - def create_net_http(uri, upstream) - Net::HTTP.new(uri.host, uri.port, upstream.host, upstream.port) - end - - def perform_proxy_request(req, res, req_class, body_stream = nil) - uri = req.request_uri - path = uri.path.dup - path << "?" << uri.query if uri.query - header = setup_proxy_header(req, res) - upstream = setup_upstream_proxy_authentication(req, res, header) - - body_tmp = [] - http = create_net_http(uri, upstream) - req_fib = Fiber.new do - http.start do - if @config[:ProxyTimeout] - ################################## these issues are - http.open_timeout = 30 # secs # necessary (maybe because - http.read_timeout = 60 # secs # Ruby's bug, but why?) - ################################## - end - if body_stream && req['transfer-encoding'] =~ /\bchunked\b/i - header['Transfer-Encoding'] = 'chunked' - end - http_req = req_class.new(path, header) - http_req.body_stream = body_stream if body_stream - http.request(http_req) do |response| - # Persistent connection requirements are mysterious for me. - # So I will close the connection in every response. - res['proxy-connection'] = "close" - res['connection'] = "close" - - # stream Net::HTTP::HTTPResponse to WEBrick::HTTPResponse - res.status = response.code.to_i - res.chunked = response.chunked? - choose_header(response, res) - set_cookie(response, res) - set_via(res) - response.read_body do |buf| - body_tmp << buf - Fiber.yield # wait for res.body Proc#call - end - end # http.request - end - end - req_fib.resume # read HTTP response headers and first chunk of the body - res.body = ->(socket) do - while buf = body_tmp.shift - socket.write(buf) - buf.clear - req_fib.resume # continue response.read_body - end - end - end - # :stopdoc: - end -end diff --git a/tool/lib/webrick/httprequest.rb b/tool/lib/webrick/httprequest.rb deleted file mode 100644 index d34eac7ecf..0000000000 --- a/tool/lib/webrick/httprequest.rb +++ /dev/null @@ -1,636 +0,0 @@ -# frozen_string_literal: false -# -# httprequest.rb -- HTTPRequest Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httprequest.rb,v 1.64 2003/07/13 17:18:22 gotoyuzo Exp $ - -require 'fiber' -require 'uri' -require_relative 'httpversion' -require_relative 'httpstatus' -require_relative 'httputils' -require_relative 'cookie' - -module WEBrick - - ## - # An HTTP request. This is consumed by service and do_* methods in - # WEBrick servlets - - class HTTPRequest - - BODY_CONTAINABLE_METHODS = [ "POST", "PUT" ] # :nodoc: - - # :section: Request line - - ## - # The complete request line such as: - # - # GET / HTTP/1.1 - - attr_reader :request_line - - ## - # The request method, GET, POST, PUT, etc. - - attr_reader :request_method - - ## - # The unparsed URI of the request - - attr_reader :unparsed_uri - - ## - # The HTTP version of the request - - attr_reader :http_version - - # :section: Request-URI - - ## - # The parsed URI of the request - - attr_reader :request_uri - - ## - # The request path - - attr_reader :path - - ## - # The script name (CGI variable) - - attr_accessor :script_name - - ## - # The path info (CGI variable) - - attr_accessor :path_info - - ## - # The query from the URI of the request - - attr_accessor :query_string - - # :section: Header and entity body - - ## - # The raw header of the request - - attr_reader :raw_header - - ## - # The parsed header of the request - - attr_reader :header - - ## - # The parsed request cookies - - attr_reader :cookies - - ## - # The Accept header value - - attr_reader :accept - - ## - # The Accept-Charset header value - - attr_reader :accept_charset - - ## - # The Accept-Encoding header value - - attr_reader :accept_encoding - - ## - # The Accept-Language header value - - attr_reader :accept_language - - # :section: - - ## - # The remote user (CGI variable) - - attr_accessor :user - - ## - # The socket address of the server - - attr_reader :addr - - ## - # The socket address of the client - - attr_reader :peeraddr - - ## - # Hash of request attributes - - attr_reader :attributes - - ## - # Is this a keep-alive connection? - - attr_reader :keep_alive - - ## - # The local time this request was received - - attr_reader :request_time - - ## - # Creates a new HTTP request. WEBrick::Config::HTTP is the default - # configuration. - - def initialize(config) - @config = config - @buffer_size = @config[:InputBufferSize] - @logger = config[:Logger] - - @request_line = @request_method = - @unparsed_uri = @http_version = nil - - @request_uri = @host = @port = @path = nil - @script_name = @path_info = nil - @query_string = nil - @query = nil - @form_data = nil - - @raw_header = Array.new - @header = nil - @cookies = [] - @accept = [] - @accept_charset = [] - @accept_encoding = [] - @accept_language = [] - @body = "" - - @addr = @peeraddr = nil - @attributes = {} - @user = nil - @keep_alive = false - @request_time = nil - - @remaining_size = nil - @socket = nil - - @forwarded_proto = @forwarded_host = @forwarded_port = - @forwarded_server = @forwarded_for = nil - end - - ## - # Parses a request from +socket+. This is called internally by - # WEBrick::HTTPServer. - - def parse(socket=nil) - @socket = socket - begin - @peeraddr = socket.respond_to?(:peeraddr) ? socket.peeraddr : [] - @addr = socket.respond_to?(:addr) ? socket.addr : [] - rescue Errno::ENOTCONN - raise HTTPStatus::EOFError - end - - read_request_line(socket) - if @http_version.major > 0 - read_header(socket) - @header['cookie'].each{|cookie| - @cookies += Cookie::parse(cookie) - } - @accept = HTTPUtils.parse_qvalues(self['accept']) - @accept_charset = HTTPUtils.parse_qvalues(self['accept-charset']) - @accept_encoding = HTTPUtils.parse_qvalues(self['accept-encoding']) - @accept_language = HTTPUtils.parse_qvalues(self['accept-language']) - end - return if @request_method == "CONNECT" - return if @unparsed_uri == "*" - - begin - setup_forwarded_info - @request_uri = parse_uri(@unparsed_uri) - @path = HTTPUtils::unescape(@request_uri.path) - @path = HTTPUtils::normalize_path(@path) - @host = @request_uri.host - @port = @request_uri.port - @query_string = @request_uri.query - @script_name = "" - @path_info = @path.dup - rescue - raise HTTPStatus::BadRequest, "bad URI `#{@unparsed_uri}'." - end - - if /\Aclose\z/io =~ self["connection"] - @keep_alive = false - elsif /\Akeep-alive\z/io =~ self["connection"] - @keep_alive = true - elsif @http_version < "1.1" - @keep_alive = false - else - @keep_alive = true - end - end - - ## - # Generate HTTP/1.1 100 continue response if the client expects it, - # otherwise does nothing. - - def continue # :nodoc: - if self['expect'] == '100-continue' && @config[:HTTPVersion] >= "1.1" - @socket << "HTTP/#{@config[:HTTPVersion]} 100 continue#{CRLF}#{CRLF}" - @header.delete('expect') - end - end - - ## - # Returns the request body. - - def body(&block) # :yields: body_chunk - block ||= Proc.new{|chunk| @body << chunk } - read_body(@socket, block) - @body.empty? ? nil : @body - end - - ## - # Prepares the HTTPRequest object for use as the - # source for IO.copy_stream - - def body_reader - @body_tmp = [] - @body_rd = Fiber.new do - body do |buf| - @body_tmp << buf - Fiber.yield - end - end - @body_rd.resume # grab the first chunk and yield - self - end - - # for IO.copy_stream. - def readpartial(size, buf = ''.b) # :nodoc - res = @body_tmp.shift or raise EOFError, 'end of file reached' - if res.length > size - @body_tmp.unshift(res[size..-1]) - res = res[0..size - 1] - end - buf.replace(res) - res.clear - # get more chunks - check alive? because we can take a partial chunk - @body_rd.resume if @body_rd.alive? - buf - end - - ## - # Request query as a Hash - - def query - unless @query - parse_query() - end - @query - end - - ## - # The content-length header - - def content_length - return Integer(self['content-length']) - end - - ## - # The content-type header - - def content_type - return self['content-type'] - end - - ## - # Retrieves +header_name+ - - def [](header_name) - if @header - value = @header[header_name.downcase] - value.empty? ? nil : value.join(", ") - end - end - - ## - # Iterates over the request headers - - def each - if @header - @header.each{|k, v| - value = @header[k] - yield(k, value.empty? ? nil : value.join(", ")) - } - end - end - - ## - # The host this request is for - - def host - return @forwarded_host || @host - end - - ## - # The port this request is for - - def port - return @forwarded_port || @port - end - - ## - # The server name this request is for - - def server_name - return @forwarded_server || @config[:ServerName] - end - - ## - # The client's IP address - - def remote_ip - return self["client-ip"] || @forwarded_for || @peeraddr[3] - end - - ## - # Is this an SSL request? - - def ssl? - return @request_uri.scheme == "https" - end - - ## - # Should the connection this request was made on be kept alive? - - def keep_alive? - @keep_alive - end - - def to_s # :nodoc: - ret = @request_line.dup - @raw_header.each{|line| ret << line } - ret << CRLF - ret << body if body - ret - end - - ## - # Consumes any remaining body and updates keep-alive status - - def fixup() # :nodoc: - begin - body{|chunk| } # read remaining body - rescue HTTPStatus::Error => ex - @logger.error("HTTPRequest#fixup: #{ex.class} occurred.") - @keep_alive = false - rescue => ex - @logger.error(ex) - @keep_alive = false - end - end - - # This method provides the metavariables defined by the revision 3 - # of "The WWW Common Gateway Interface Version 1.1" - # To browse the current document of CGI Version 1.1, see below: - # http://tools.ietf.org/html/rfc3875 - - def meta_vars - meta = Hash.new - - cl = self["Content-Length"] - ct = self["Content-Type"] - meta["CONTENT_LENGTH"] = cl if cl.to_i > 0 - meta["CONTENT_TYPE"] = ct.dup if ct - meta["GATEWAY_INTERFACE"] = "CGI/1.1" - meta["PATH_INFO"] = @path_info ? @path_info.dup : "" - #meta["PATH_TRANSLATED"] = nil # no plan to be provided - meta["QUERY_STRING"] = @query_string ? @query_string.dup : "" - meta["REMOTE_ADDR"] = @peeraddr[3] - meta["REMOTE_HOST"] = @peeraddr[2] - #meta["REMOTE_IDENT"] = nil # no plan to be provided - meta["REMOTE_USER"] = @user - meta["REQUEST_METHOD"] = @request_method.dup - meta["REQUEST_URI"] = @request_uri.to_s - meta["SCRIPT_NAME"] = @script_name.dup - meta["SERVER_NAME"] = @host - meta["SERVER_PORT"] = @port.to_s - meta["SERVER_PROTOCOL"] = "HTTP/" + @config[:HTTPVersion].to_s - meta["SERVER_SOFTWARE"] = @config[:ServerSoftware].dup - - self.each{|key, val| - next if /^content-type$/i =~ key - next if /^content-length$/i =~ key - name = "HTTP_" + key - name.gsub!(/-/o, "_") - name.upcase! - meta[name] = val - } - - meta - end - - private - - # :stopdoc: - - MAX_URI_LENGTH = 2083 # :nodoc: - - # same as Mongrel, Thin and Puma - MAX_HEADER_LENGTH = (112 * 1024) # :nodoc: - - def read_request_line(socket) - @request_line = read_line(socket, MAX_URI_LENGTH) if socket - raise HTTPStatus::EOFError unless @request_line - - @request_bytes = @request_line.bytesize - if @request_bytes >= MAX_URI_LENGTH and @request_line[-1, 1] != LF - raise HTTPStatus::RequestURITooLarge - end - - @request_time = Time.now - if /^(\S+)\s+(\S++)(?:\s+HTTP\/(\d+\.\d+))?\r?\n/mo =~ @request_line - @request_method = $1 - @unparsed_uri = $2 - @http_version = HTTPVersion.new($3 ? $3 : "0.9") - else - rl = @request_line.sub(/\x0d?\x0a\z/o, '') - raise HTTPStatus::BadRequest, "bad Request-Line `#{rl}'." - end - end - - def read_header(socket) - if socket - while line = read_line(socket) - break if /\A(#{CRLF}|#{LF})\z/om =~ line - if (@request_bytes += line.bytesize) > MAX_HEADER_LENGTH - raise HTTPStatus::RequestEntityTooLarge, 'headers too large' - end - @raw_header << line - end - end - @header = HTTPUtils::parse_header(@raw_header.join) - end - - def parse_uri(str, scheme="http") - if @config[:Escape8bitURI] - str = HTTPUtils::escape8bit(str) - end - str.sub!(%r{\A/+}o, '/') - uri = URI::parse(str) - return uri if uri.absolute? - if @forwarded_host - host, port = @forwarded_host, @forwarded_port - elsif self["host"] - pattern = /\A(#{URI::REGEXP::PATTERN::HOST})(?::(\d+))?\z/n - host, port = *self['host'].scan(pattern)[0] - elsif @addr.size > 0 - host, port = @addr[2], @addr[1] - else - host, port = @config[:ServerName], @config[:Port] - end - uri.scheme = @forwarded_proto || scheme - uri.host = host - uri.port = port ? port.to_i : nil - return URI::parse(uri.to_s) - end - - def read_body(socket, block) - return unless socket - if tc = self['transfer-encoding'] - case tc - when /\Achunked\z/io then read_chunked(socket, block) - else raise HTTPStatus::NotImplemented, "Transfer-Encoding: #{tc}." - end - elsif self['content-length'] || @remaining_size - @remaining_size ||= self['content-length'].to_i - while @remaining_size > 0 - sz = [@buffer_size, @remaining_size].min - break unless buf = read_data(socket, sz) - @remaining_size -= buf.bytesize - block.call(buf) - end - if @remaining_size > 0 && @socket.eof? - raise HTTPStatus::BadRequest, "invalid body size." - end - elsif BODY_CONTAINABLE_METHODS.member?(@request_method) && !@socket.eof - raise HTTPStatus::LengthRequired - end - return @body - end - - def read_chunk_size(socket) - line = read_line(socket) - if /^([0-9a-fA-F]+)(?:;(\S+))?/ =~ line - chunk_size = $1.hex - chunk_ext = $2 - [ chunk_size, chunk_ext ] - else - raise HTTPStatus::BadRequest, "bad chunk `#{line}'." - end - end - - def read_chunked(socket, block) - chunk_size, = read_chunk_size(socket) - while chunk_size > 0 - begin - sz = [ chunk_size, @buffer_size ].min - data = read_data(socket, sz) # read chunk-data - if data.nil? || data.bytesize != sz - raise HTTPStatus::BadRequest, "bad chunk data size." - end - block.call(data) - end while (chunk_size -= sz) > 0 - - read_line(socket) # skip CRLF - chunk_size, = read_chunk_size(socket) - end - read_header(socket) # trailer + CRLF - @header.delete("transfer-encoding") - @remaining_size = 0 - end - - def _read_data(io, method, *arg) - begin - WEBrick::Utils.timeout(@config[:RequestTimeout]){ - return io.__send__(method, *arg) - } - rescue Errno::ECONNRESET - return nil - rescue Timeout::Error - raise HTTPStatus::RequestTimeout - end - end - - def read_line(io, size=4096) - _read_data(io, :gets, LF, size) - end - - def read_data(io, size) - _read_data(io, :read, size) - end - - def parse_query() - begin - if @request_method == "GET" || @request_method == "HEAD" - @query = HTTPUtils::parse_query(@query_string) - elsif self['content-type'] =~ /^application\/x-www-form-urlencoded/ - @query = HTTPUtils::parse_query(body) - elsif self['content-type'] =~ /^multipart\/form-data; boundary=(.+)/ - boundary = HTTPUtils::dequote($1) - @query = HTTPUtils::parse_form_data(body, boundary) - else - @query = Hash.new - end - rescue => ex - raise HTTPStatus::BadRequest, ex.message - end - end - - PrivateNetworkRegexp = / - ^unknown$| - ^((::ffff:)?127.0.0.1|::1)$| - ^(::ffff:)?(10|172\.(1[6-9]|2[0-9]|3[01])|192\.168)\. - /ixo - - # It's said that all X-Forwarded-* headers will contain more than one - # (comma-separated) value if the original request already contained one of - # these headers. Since we could use these values as Host header, we choose - # the initial(first) value. (apr_table_mergen() adds new value after the - # existing value with ", " prefix) - def setup_forwarded_info - if @forwarded_server = self["x-forwarded-server"] - @forwarded_server = @forwarded_server.split(",", 2).first - end - if @forwarded_proto = self["x-forwarded-proto"] - @forwarded_proto = @forwarded_proto.split(",", 2).first - end - if host_port = self["x-forwarded-host"] - host_port = host_port.split(",", 2).first - if host_port =~ /\A(\[[0-9a-fA-F:]+\])(?::(\d+))?\z/ - @forwarded_host = $1 - tmp = $2 - else - @forwarded_host, tmp = host_port.split(":", 2) - end - @forwarded_port = (tmp || (@forwarded_proto == "https" ? 443 : 80)).to_i - end - if addrs = self["x-forwarded-for"] - addrs = addrs.split(",").collect(&:strip) - addrs.reject!{|ip| PrivateNetworkRegexp =~ ip } - @forwarded_for = addrs.first - end - end - - # :startdoc: - end -end diff --git a/tool/lib/webrick/httpresponse.rb b/tool/lib/webrick/httpresponse.rb deleted file mode 100644 index ba4494ab74..0000000000 --- a/tool/lib/webrick/httpresponse.rb +++ /dev/null @@ -1,564 +0,0 @@ -# frozen_string_literal: false -# -# httpresponse.rb -- HTTPResponse Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpresponse.rb,v 1.45 2003/07/11 11:02:25 gotoyuzo Exp $ - -require 'time' -require 'uri' -require_relative 'httpversion' -require_relative 'htmlutils' -require_relative 'httputils' -require_relative 'httpstatus' - -module WEBrick - ## - # An HTTP response. This is filled in by the service or do_* methods of a - # WEBrick HTTP Servlet. - - class HTTPResponse - class InvalidHeader < StandardError - end - - ## - # HTTP Response version - - attr_reader :http_version - - ## - # Response status code (200) - - attr_reader :status - - ## - # Response header - - attr_reader :header - - ## - # Response cookies - - attr_reader :cookies - - ## - # Response reason phrase ("OK") - - attr_accessor :reason_phrase - - ## - # Body may be: - # * a String; - # * an IO-like object that responds to +#read+ and +#readpartial+; - # * a Proc-like object that responds to +#call+. - # - # In the latter case, either #chunked= should be set to +true+, - # or <code>header['content-length']</code> explicitly provided. - # Example: - # - # server.mount_proc '/' do |req, res| - # res.chunked = true - # # or - # # res.header['content-length'] = 10 - # res.body = proc { |out| out.write(Time.now.to_s) } - # end - - attr_accessor :body - - ## - # Request method for this response - - attr_accessor :request_method - - ## - # Request URI for this response - - attr_accessor :request_uri - - ## - # Request HTTP version for this response - - attr_accessor :request_http_version - - ## - # Filename of the static file in this response. Only used by the - # FileHandler servlet. - - attr_accessor :filename - - ## - # Is this a keep-alive response? - - attr_accessor :keep_alive - - ## - # Configuration for this response - - attr_reader :config - - ## - # Bytes sent in this response - - attr_reader :sent_size - - ## - # Creates a new HTTP response object. WEBrick::Config::HTTP is the - # default configuration. - - def initialize(config) - @config = config - @buffer_size = config[:OutputBufferSize] - @logger = config[:Logger] - @header = Hash.new - @status = HTTPStatus::RC_OK - @reason_phrase = nil - @http_version = HTTPVersion::convert(@config[:HTTPVersion]) - @body = '' - @keep_alive = true - @cookies = [] - @request_method = nil - @request_uri = nil - @request_http_version = @http_version # temporary - @chunked = false - @filename = nil - @sent_size = 0 - @bodytempfile = nil - end - - ## - # The response's HTTP status line - - def status_line - "HTTP/#@http_version #@status #@reason_phrase".rstrip << CRLF - end - - ## - # Sets the response's status to the +status+ code - - def status=(status) - @status = status - @reason_phrase = HTTPStatus::reason_phrase(status) - end - - ## - # Retrieves the response header +field+ - - def [](field) - @header[field.downcase] - end - - ## - # Sets the response header +field+ to +value+ - - def []=(field, value) - @chunked = value.to_s.downcase == 'chunked' if field.downcase == 'transfer-encoding' - @header[field.downcase] = value.to_s - end - - ## - # The content-length header - - def content_length - if len = self['content-length'] - return Integer(len) - end - end - - ## - # Sets the content-length header to +len+ - - def content_length=(len) - self['content-length'] = len.to_s - end - - ## - # The content-type header - - def content_type - self['content-type'] - end - - ## - # Sets the content-type header to +type+ - - def content_type=(type) - self['content-type'] = type - end - - ## - # Iterates over each header in the response - - def each - @header.each{|field, value| yield(field, value) } - end - - ## - # Will this response body be returned using chunked transfer-encoding? - - def chunked? - @chunked - end - - ## - # Enables chunked transfer encoding. - - def chunked=(val) - @chunked = val ? true : false - end - - ## - # Will this response's connection be kept alive? - - def keep_alive? - @keep_alive - end - - ## - # Sends the response on +socket+ - - def send_response(socket) # :nodoc: - begin - setup_header() - send_header(socket) - send_body(socket) - rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ENOTCONN => ex - @logger.debug(ex) - @keep_alive = false - rescue Exception => ex - @logger.error(ex) - @keep_alive = false - end - end - - ## - # Sets up the headers for sending - - def setup_header() # :nodoc: - @reason_phrase ||= HTTPStatus::reason_phrase(@status) - @header['server'] ||= @config[:ServerSoftware] - @header['date'] ||= Time.now.httpdate - - # HTTP/0.9 features - if @request_http_version < "1.0" - @http_version = HTTPVersion.new("0.9") - @keep_alive = false - end - - # HTTP/1.0 features - if @request_http_version < "1.1" - if chunked? - @chunked = false - ver = @request_http_version.to_s - msg = "chunked is set for an HTTP/#{ver} request. (ignored)" - @logger.warn(msg) - end - end - - # Determine the message length (RFC2616 -- 4.4 Message Length) - if @status == 304 || @status == 204 || HTTPStatus::info?(@status) - @header.delete('content-length') - @body = "" - elsif chunked? - @header["transfer-encoding"] = "chunked" - @header.delete('content-length') - elsif %r{^multipart/byteranges} =~ @header['content-type'] - @header.delete('content-length') - elsif @header['content-length'].nil? - if @body.respond_to? :readpartial - elsif @body.respond_to? :call - make_body_tempfile - else - @header['content-length'] = (@body ? @body.bytesize : 0).to_s - end - end - - # Keep-Alive connection. - if @header['connection'] == "close" - @keep_alive = false - elsif keep_alive? - if chunked? || @header['content-length'] || @status == 304 || @status == 204 || HTTPStatus.info?(@status) - @header['connection'] = "Keep-Alive" - else - msg = "Could not determine content-length of response body. Set content-length of the response or set Response#chunked = true" - @logger.warn(msg) - @header['connection'] = "close" - @keep_alive = false - end - else - @header['connection'] = "close" - end - - # Location is a single absoluteURI. - if location = @header['location'] - if @request_uri - @header['location'] = @request_uri.merge(location).to_s - end - end - end - - def make_body_tempfile # :nodoc: - return if @bodytempfile - bodytempfile = Tempfile.create("webrick") - if @body.nil? - # nothing - elsif @body.respond_to? :readpartial - IO.copy_stream(@body, bodytempfile) - @body.close - elsif @body.respond_to? :call - @body.call(bodytempfile) - else - bodytempfile.write @body - end - bodytempfile.rewind - @body = @bodytempfile = bodytempfile - @header['content-length'] = bodytempfile.stat.size.to_s - end - - def remove_body_tempfile # :nodoc: - if @bodytempfile - @bodytempfile.close - File.unlink @bodytempfile.path - @bodytempfile = nil - end - end - - - ## - # Sends the headers on +socket+ - - def send_header(socket) # :nodoc: - if @http_version.major > 0 - data = status_line() - @header.each{|key, value| - tmp = key.gsub(/\bwww|^te$|\b\w/){ $&.upcase } - data << "#{tmp}: #{check_header(value)}" << CRLF - } - @cookies.each{|cookie| - data << "Set-Cookie: " << check_header(cookie.to_s) << CRLF - } - data << CRLF - socket.write(data) - end - rescue InvalidHeader => e - @header.clear - @cookies.clear - set_error e - retry - end - - ## - # Sends the body on +socket+ - - def send_body(socket) # :nodoc: - if @body.respond_to? :readpartial then - send_body_io(socket) - elsif @body.respond_to?(:call) then - send_body_proc(socket) - else - send_body_string(socket) - end - end - - ## - # Redirects to +url+ with a WEBrick::HTTPStatus::Redirect +status+. - # - # Example: - # - # res.set_redirect WEBrick::HTTPStatus::TemporaryRedirect - - def set_redirect(status, url) - url = URI(url).to_s - @body = "<HTML><A HREF=\"#{url}\">#{url}</A>.</HTML>\n" - @header['location'] = url - raise status - end - - ## - # Creates an error page for exception +ex+ with an optional +backtrace+ - - def set_error(ex, backtrace=false) - case ex - when HTTPStatus::Status - @keep_alive = false if HTTPStatus::error?(ex.code) - self.status = ex.code - else - @keep_alive = false - self.status = HTTPStatus::RC_INTERNAL_SERVER_ERROR - end - @header['content-type'] = "text/html; charset=ISO-8859-1" - - if respond_to?(:create_error_page) - create_error_page() - return - end - - if @request_uri - host, port = @request_uri.host, @request_uri.port - else - host, port = @config[:ServerName], @config[:Port] - end - - error_body(backtrace, ex, host, port) - end - - private - - def check_header(header_value) - header_value = header_value.to_s - if /[\r\n]/ =~ header_value - raise InvalidHeader - else - header_value - end - end - - # :stopdoc: - - def error_body(backtrace, ex, host, port) - @body = '' - @body << <<-_end_of_html_ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN"> -<HTML> - <HEAD><TITLE>#{HTMLUtils::escape(@reason_phrase)}</TITLE></HEAD> - <BODY> - <H1>#{HTMLUtils::escape(@reason_phrase)}</H1> - #{HTMLUtils::escape(ex.message)} - <HR> - _end_of_html_ - - if backtrace && $DEBUG - @body << "backtrace of `#{HTMLUtils::escape(ex.class.to_s)}' " - @body << "#{HTMLUtils::escape(ex.message)}" - @body << "<PRE>" - ex.backtrace.each{|line| @body << "\t#{line}\n"} - @body << "</PRE><HR>" - end - - @body << <<-_end_of_html_ - <ADDRESS> - #{HTMLUtils::escape(@config[:ServerSoftware])} at - #{host}:#{port} - </ADDRESS> - </BODY> -</HTML> - _end_of_html_ - end - - def send_body_io(socket) - begin - if @request_method == "HEAD" - # do nothing - elsif chunked? - buf = '' - begin - @body.readpartial(@buffer_size, buf) - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - socket.write(data) - data.clear - @sent_size += size - rescue EOFError - break - end while true - buf.clear - socket.write("0#{CRLF}#{CRLF}") - else - if %r{\Abytes (\d+)-(\d+)/\d+\z} =~ @header['content-range'] - offset = $1.to_i - size = $2.to_i - offset + 1 - else - offset = nil - size = @header['content-length'] - size = size.to_i if size - end - begin - @sent_size = IO.copy_stream(@body, socket, size, offset) - rescue NotImplementedError - @body.seek(offset, IO::SEEK_SET) - @sent_size = IO.copy_stream(@body, socket, size) - end - end - ensure - @body.close - end - remove_body_tempfile - end - - def send_body_string(socket) - if @request_method == "HEAD" - # do nothing - elsif chunked? - body ? @body.bytesize : 0 - while buf = @body[@sent_size, @buffer_size] - break if buf.empty? - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - buf.clear - socket.write(data) - @sent_size += size - end - socket.write("0#{CRLF}#{CRLF}") - else - if @body && @body.bytesize > 0 - socket.write(@body) - @sent_size = @body.bytesize - end - end - end - - def send_body_proc(socket) - if @request_method == "HEAD" - # do nothing - elsif chunked? - @body.call(ChunkedWrapper.new(socket, self)) - socket.write("0#{CRLF}#{CRLF}") - else - size = @header['content-length'].to_i - if @bodytempfile - @bodytempfile.rewind - IO.copy_stream(@bodytempfile, socket) - else - @body.call(socket) - end - @sent_size = size - end - end - - class ChunkedWrapper - def initialize(socket, resp) - @socket = socket - @resp = resp - end - - def write(buf) - return 0 if buf.empty? - socket = @socket - @resp.instance_eval { - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - socket.write(data) - data.clear - @sent_size += size - size - } - end - - def <<(*buf) - write(buf) - self - end - end - - # preserved for compatibility with some 3rd-party handlers - def _write_data(socket, data) - socket << data - end - - # :startdoc: - end - -end diff --git a/tool/lib/webrick/https.rb b/tool/lib/webrick/https.rb deleted file mode 100644 index b0a49bc40b..0000000000 --- a/tool/lib/webrick/https.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: false -# -# https.rb -- SSL/TLS enhancement for HTTPServer -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: https.rb,v 1.15 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'ssl' -require_relative 'httpserver' - -module WEBrick - module Config - HTTP.update(SSL) - end - - ## - #-- - # Adds SSL functionality to WEBrick::HTTPRequest - - class HTTPRequest - - ## - # HTTP request SSL cipher - - attr_reader :cipher - - ## - # HTTP request server certificate - - attr_reader :server_cert - - ## - # HTTP request client certificate - - attr_reader :client_cert - - # :stopdoc: - - alias orig_parse parse - - def parse(socket=nil) - if socket.respond_to?(:cert) - @server_cert = socket.cert || @config[:SSLCertificate] - @client_cert = socket.peer_cert - @client_cert_chain = socket.peer_cert_chain - @cipher = socket.cipher - end - orig_parse(socket) - end - - alias orig_parse_uri parse_uri - - def parse_uri(str, scheme="https") - if server_cert - return orig_parse_uri(str, scheme) - end - return orig_parse_uri(str) - end - private :parse_uri - - alias orig_meta_vars meta_vars - - def meta_vars - meta = orig_meta_vars - if server_cert - meta["HTTPS"] = "on" - meta["SSL_SERVER_CERT"] = @server_cert.to_pem - meta["SSL_CLIENT_CERT"] = @client_cert ? @client_cert.to_pem : "" - if @client_cert_chain - @client_cert_chain.each_with_index{|cert, i| - meta["SSL_CLIENT_CERT_CHAIN_#{i}"] = cert.to_pem - } - end - meta["SSL_CIPHER"] = @cipher[0] - meta["SSL_PROTOCOL"] = @cipher[1] - meta["SSL_CIPHER_USEKEYSIZE"] = @cipher[2].to_s - meta["SSL_CIPHER_ALGKEYSIZE"] = @cipher[3].to_s - end - meta - end - - # :startdoc: - end - - ## - #-- - # Fake WEBrick::HTTPRequest for lookup_server - - class SNIRequest - - ## - # The SNI hostname - - attr_reader :host - - ## - # The socket address of the server - - attr_reader :addr - - ## - # The port this request is for - - attr_reader :port - - ## - # Creates a new SNIRequest. - - def initialize(sslsocket, hostname) - @host = hostname - @addr = sslsocket.addr - @port = @addr[1] - end - end - - - ## - #-- - # Adds SSL functionality to WEBrick::HTTPServer - - class HTTPServer < ::WEBrick::GenericServer - ## - # ServerNameIndication callback - - def ssl_servername_callback(sslsocket, hostname = nil) - req = SNIRequest.new(sslsocket, hostname) - server = lookup_server(req) - server ? server.ssl_context : nil - end - - # :stopdoc: - - ## - # Check whether +server+ is also SSL server. - # Also +server+'s SSL context will be created. - - alias orig_virtual_host virtual_host - - def virtual_host(server) - if @config[:SSLEnable] && !server.ssl_context - raise ArgumentError, "virtual host must set SSLEnable to true" - end - orig_virtual_host(server) - end - - # :startdoc: - end -end diff --git a/tool/lib/webrick/httpserver.rb b/tool/lib/webrick/httpserver.rb deleted file mode 100644 index f3f948da3b..0000000000 --- a/tool/lib/webrick/httpserver.rb +++ /dev/null @@ -1,293 +0,0 @@ -# frozen_string_literal: false -# -# httpserver.rb -- HTTPServer Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpserver.rb,v 1.63 2002/10/01 17:16:32 gotoyuzo Exp $ - -require_relative 'server' -require_relative 'httputils' -require_relative 'httpstatus' -require_relative 'httprequest' -require_relative 'httpresponse' -require_relative 'httpservlet' -require_relative 'accesslog' - -module WEBrick - class HTTPServerError < ServerError; end - - ## - # An HTTP Server - - class HTTPServer < ::WEBrick::GenericServer - ## - # Creates a new HTTP server according to +config+ - # - # An HTTP server uses the following attributes: - # - # :AccessLog:: An array of access logs. See WEBrick::AccessLog - # :BindAddress:: Local address for the server to bind to - # :DocumentRoot:: Root path to serve files from - # :DocumentRootOptions:: Options for the default HTTPServlet::FileHandler - # :HTTPVersion:: The HTTP version of this server - # :Port:: Port to listen on - # :RequestCallback:: Called with a request and response before each - # request is serviced. - # :RequestTimeout:: Maximum time to wait between requests - # :ServerAlias:: Array of alternate names for this server for virtual - # hosting - # :ServerName:: Name for this server for virtual hosting - - def initialize(config={}, default=Config::HTTP) - super(config, default) - @http_version = HTTPVersion::convert(@config[:HTTPVersion]) - - @mount_tab = MountTable.new - if @config[:DocumentRoot] - mount("/", HTTPServlet::FileHandler, @config[:DocumentRoot], - @config[:DocumentRootOptions]) - end - - unless @config[:AccessLog] - @config[:AccessLog] = [ - [ $stderr, AccessLog::COMMON_LOG_FORMAT ], - [ $stderr, AccessLog::REFERER_LOG_FORMAT ] - ] - end - - @virtual_hosts = Array.new - end - - ## - # Processes requests on +sock+ - - def run(sock) - while true - req = create_request(@config) - res = create_response(@config) - server = self - begin - timeout = @config[:RequestTimeout] - while timeout > 0 - break if sock.to_io.wait_readable(0.5) - break if @status != :Running - timeout -= 0.5 - end - raise HTTPStatus::EOFError if timeout <= 0 || @status != :Running - raise HTTPStatus::EOFError if sock.eof? - req.parse(sock) - res.request_method = req.request_method - res.request_uri = req.request_uri - res.request_http_version = req.http_version - res.keep_alive = req.keep_alive? - server = lookup_server(req) || self - if callback = server[:RequestCallback] - callback.call(req, res) - elsif callback = server[:RequestHandler] - msg = ":RequestHandler is deprecated, please use :RequestCallback" - @logger.warn(msg) - callback.call(req, res) - end - server.service(req, res) - rescue HTTPStatus::EOFError, HTTPStatus::RequestTimeout => ex - res.set_error(ex) - rescue HTTPStatus::Error => ex - @logger.error(ex.message) - res.set_error(ex) - rescue HTTPStatus::Status => ex - res.status = ex.code - rescue StandardError => ex - @logger.error(ex) - res.set_error(ex, true) - ensure - if req.request_line - if req.keep_alive? && res.keep_alive? - req.fixup() - end - res.send_response(sock) - server.access_log(@config, req, res) - end - end - break if @http_version < "1.1" - break unless req.keep_alive? - break unless res.keep_alive? - end - end - - ## - # Services +req+ and fills in +res+ - - def service(req, res) - if req.unparsed_uri == "*" - if req.request_method == "OPTIONS" - do_OPTIONS(req, res) - raise HTTPStatus::OK - end - raise HTTPStatus::NotFound, "`#{req.unparsed_uri}' not found." - end - - servlet, options, script_name, path_info = search_servlet(req.path) - raise HTTPStatus::NotFound, "`#{req.path}' not found." unless servlet - req.script_name = script_name - req.path_info = path_info - si = servlet.get_instance(self, *options) - @logger.debug(format("%s is invoked.", si.class.name)) - si.service(req, res) - end - - ## - # The default OPTIONS request handler says GET, HEAD, POST and OPTIONS - # requests are allowed. - - def do_OPTIONS(req, res) - res["allow"] = "GET,HEAD,POST,OPTIONS" - end - - ## - # Mounts +servlet+ on +dir+ passing +options+ to the servlet at creation - # time - - def mount(dir, servlet, *options) - @logger.debug(sprintf("%s is mounted on %s.", servlet.inspect, dir)) - @mount_tab[dir] = [ servlet, options ] - end - - ## - # Mounts +proc+ or +block+ on +dir+ and calls it with a - # WEBrick::HTTPRequest and WEBrick::HTTPResponse - - def mount_proc(dir, proc=nil, &block) - proc ||= block - raise HTTPServerError, "must pass a proc or block" unless proc - mount(dir, HTTPServlet::ProcHandler.new(proc)) - end - - ## - # Unmounts +dir+ - - def unmount(dir) - @logger.debug(sprintf("unmount %s.", dir)) - @mount_tab.delete(dir) - end - alias umount unmount - - ## - # Finds a servlet for +path+ - - def search_servlet(path) - script_name, path_info = @mount_tab.scan(path) - servlet, options = @mount_tab[script_name] - if servlet - [ servlet, options, script_name, path_info ] - end - end - - ## - # Adds +server+ as a virtual host. - - def virtual_host(server) - @virtual_hosts << server - @virtual_hosts = @virtual_hosts.sort_by{|s| - num = 0 - num -= 4 if s[:BindAddress] - num -= 2 if s[:Port] - num -= 1 if s[:ServerName] - num - } - end - - ## - # Finds the appropriate virtual host to handle +req+ - - def lookup_server(req) - @virtual_hosts.find{|s| - (s[:BindAddress].nil? || req.addr[3] == s[:BindAddress]) && - (s[:Port].nil? || req.port == s[:Port]) && - ((s[:ServerName].nil? || req.host == s[:ServerName]) || - (!s[:ServerAlias].nil? && s[:ServerAlias].find{|h| h === req.host})) - } - end - - ## - # Logs +req+ and +res+ in the access logs. +config+ is used for the - # server name. - - def access_log(config, req, res) - param = AccessLog::setup_params(config, req, res) - @config[:AccessLog].each{|logger, fmt| - logger << AccessLog::format(fmt+"\n", param) - } - end - - ## - # Creates the HTTPRequest used when handling the HTTP - # request. Can be overridden by subclasses. - def create_request(with_webrick_config) - HTTPRequest.new(with_webrick_config) - end - - ## - # Creates the HTTPResponse used when handling the HTTP - # request. Can be overridden by subclasses. - def create_response(with_webrick_config) - HTTPResponse.new(with_webrick_config) - end - - ## - # Mount table for the path a servlet is mounted on in the directory space - # of the server. Users of WEBrick can only access this indirectly via - # WEBrick::HTTPServer#mount, WEBrick::HTTPServer#unmount and - # WEBrick::HTTPServer#search_servlet - - class MountTable # :nodoc: - def initialize - @tab = Hash.new - compile - end - - def [](dir) - dir = normalize(dir) - @tab[dir] - end - - def []=(dir, val) - dir = normalize(dir) - @tab[dir] = val - compile - val - end - - def delete(dir) - dir = normalize(dir) - res = @tab.delete(dir) - compile - res - end - - def scan(path) - @scanner =~ path - [ $&, $' ] - end - - private - - def compile - k = @tab.keys - k.sort! - k.reverse! - k.collect!{|path| Regexp.escape(path) } - @scanner = Regexp.new("\\A(" + k.join("|") +")(?=/|\\z)") - end - - def normalize(dir) - ret = dir ? dir.dup : "" - ret.sub!(%r|/+\z|, "") - ret - end - end - end -end diff --git a/tool/lib/webrick/httpservlet.rb b/tool/lib/webrick/httpservlet.rb deleted file mode 100644 index da49a1405b..0000000000 --- a/tool/lib/webrick/httpservlet.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: false -# -# httpservlet.rb -- HTTPServlet Utility File -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpservlet.rb,v 1.21 2003/02/23 12:24:46 gotoyuzo Exp $ - -require_relative 'httpservlet/abstract' -require_relative 'httpservlet/filehandler' -require_relative 'httpservlet/cgihandler' -require_relative 'httpservlet/erbhandler' -require_relative 'httpservlet/prochandler' - -module WEBrick - module HTTPServlet - FileHandler.add_handler("cgi", CGIHandler) - FileHandler.add_handler("rhtml", ERBHandler) - end -end diff --git a/tool/lib/webrick/httpservlet/abstract.rb b/tool/lib/webrick/httpservlet/abstract.rb deleted file mode 100644 index bccb091861..0000000000 --- a/tool/lib/webrick/httpservlet/abstract.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: false -# -# httpservlet.rb -- HTTPServlet Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: abstract.rb,v 1.24 2003/07/11 11:16:46 gotoyuzo Exp $ - -require_relative '../htmlutils' -require_relative '../httputils' -require_relative '../httpstatus' - -module WEBrick - module HTTPServlet - class HTTPServletError < StandardError; end - - ## - # AbstractServlet allows HTTP server modules to be reused across multiple - # servers and allows encapsulation of functionality. - # - # By default a servlet will respond to GET, HEAD (through an alias to GET) - # and OPTIONS requests. - # - # By default a new servlet is initialized for every request. A servlet - # instance can be reused by overriding ::get_instance in the - # AbstractServlet subclass. - # - # == A Simple Servlet - # - # class Simple < WEBrick::HTTPServlet::AbstractServlet - # def do_GET request, response - # status, content_type, body = do_stuff_with request - # - # response.status = status - # response['Content-Type'] = content_type - # response.body = body - # end - # - # def do_stuff_with request - # return 200, 'text/plain', 'you got a page' - # end - # end - # - # This servlet can be mounted on a server at a given path: - # - # server.mount '/simple', Simple - # - # == Servlet Configuration - # - # Servlets can be configured via initialize. The first argument is the - # HTTP server the servlet is being initialized for. - # - # class Configurable < Simple - # def initialize server, color, size - # super server - # @color = color - # @size = size - # end - # - # def do_stuff_with request - # content = "<p " \ - # %q{style="color: #{@color}; font-size: #{@size}"} \ - # ">Hello, World!" - # - # return 200, "text/html", content - # end - # end - # - # This servlet must be provided two arguments at mount time: - # - # server.mount '/configurable', Configurable, 'red', '2em' - - class AbstractServlet - - ## - # Factory for servlet instances that will handle a request from +server+ - # using +options+ from the mount point. By default a new servlet - # instance is created for every call. - - def self.get_instance(server, *options) - self.new(server, *options) - end - - ## - # Initializes a new servlet for +server+ using +options+ which are - # stored as-is in +@options+. +@logger+ is also provided. - - def initialize(server, *options) - @server = @config = server - @logger = @server[:Logger] - @options = options - end - - ## - # Dispatches to a +do_+ method based on +req+ if such a method is - # available. (+do_GET+ for a GET request). Raises a MethodNotAllowed - # exception if the method is not implemented. - - def service(req, res) - method_name = "do_" + req.request_method.gsub(/-/, "_") - if respond_to?(method_name) - __send__(method_name, req, res) - else - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - end - end - - ## - # Raises a NotFound exception - - def do_GET(req, res) - raise HTTPStatus::NotFound, "not found." - end - - ## - # Dispatches to do_GET - - def do_HEAD(req, res) - do_GET(req, res) - end - - ## - # Returns the allowed HTTP request methods - - def do_OPTIONS(req, res) - m = self.methods.grep(/\Ado_([A-Z]+)\z/) {$1} - m.sort! - res["allow"] = m.join(",") - end - - private - - ## - # Redirects to a path ending in / - - def redirect_to_directory_uri(req, res) - if req.path[-1] != ?/ - location = WEBrick::HTTPUtils.escape_path(req.path + "/") - if req.query_string && req.query_string.bytesize > 0 - location << "?" << req.query_string - end - res.set_redirect(HTTPStatus::MovedPermanently, location) - end - end - end - - end -end diff --git a/tool/lib/webrick/httpservlet/cgi_runner.rb b/tool/lib/webrick/httpservlet/cgi_runner.rb deleted file mode 100644 index 0398c16749..0000000000 --- a/tool/lib/webrick/httpservlet/cgi_runner.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: false -# -# cgi_runner.rb -- CGI launcher. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cgi_runner.rb,v 1.9 2002/09/25 11:33:15 gotoyuzo Exp $ - -def sysread(io, size) - buf = "" - while size > 0 - tmp = io.sysread(size) - buf << tmp - size -= tmp.bytesize - end - return buf -end - -STDIN.binmode - -len = sysread(STDIN, 8).to_i -out = sysread(STDIN, len) -STDOUT.reopen(File.open(out, "w")) - -len = sysread(STDIN, 8).to_i -err = sysread(STDIN, len) -STDERR.reopen(File.open(err, "w")) - -len = sysread(STDIN, 8).to_i -dump = sysread(STDIN, len) -hash = Marshal.restore(dump) -ENV.keys.each{|name| ENV.delete(name) } -hash.each{|k, v| ENV[k] = v if v } - -dir = File::dirname(ENV["SCRIPT_FILENAME"]) -Dir::chdir dir - -if ARGV[0] - argv = ARGV.dup - argv << ENV["SCRIPT_FILENAME"] - exec(*argv) - # NOTREACHED -end -exec ENV["SCRIPT_FILENAME"] diff --git a/tool/lib/webrick/httpservlet/cgihandler.rb b/tool/lib/webrick/httpservlet/cgihandler.rb deleted file mode 100644 index 4457770b7a..0000000000 --- a/tool/lib/webrick/httpservlet/cgihandler.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: false -# -# cgihandler.rb -- CGIHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cgihandler.rb,v 1.27 2003/03/21 19:56:01 gotoyuzo Exp $ - -require 'rbconfig' -require 'tempfile' -require_relative '../config' -require_relative 'abstract' - -module WEBrick - module HTTPServlet - - ## - # Servlet for handling CGI scripts - # - # Example: - # - # server.mount('/cgi/my_script', WEBrick::HTTPServlet::CGIHandler, - # '/path/to/my_script') - - class CGIHandler < AbstractServlet - Ruby = RbConfig.ruby # :nodoc: - CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: - CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb".freeze].freeze # :nodoc: - - ## - # Creates a new CGI script servlet for the script at +name+ - - def initialize(server, name) - super(server, name) - @script_filename = name - @tempdir = server[:TempDir] - interpreter = server[:CGIInterpreter] - if interpreter.is_a?(Array) - @cgicmd = CGIRunnerArray + interpreter - else - @cgicmd = "#{CGIRunner} #{interpreter}" - end - end - - # :stopdoc: - - def do_GET(req, res) - cgi_in = IO::popen(@cgicmd, "wb") - cgi_out = Tempfile.new("webrick.cgiout.", @tempdir, mode: IO::BINARY) - cgi_out.set_encoding("ASCII-8BIT") - cgi_err = Tempfile.new("webrick.cgierr.", @tempdir, mode: IO::BINARY) - cgi_err.set_encoding("ASCII-8BIT") - begin - cgi_in.sync = true - meta = req.meta_vars - meta["SCRIPT_FILENAME"] = @script_filename - meta["PATH"] = @config[:CGIPathEnv] - meta.delete("HTTP_PROXY") - if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM - meta["SystemRoot"] = ENV["SystemRoot"] - end - dump = Marshal.dump(meta) - - cgi_in.write("%8d" % cgi_out.path.bytesize) - cgi_in.write(cgi_out.path) - cgi_in.write("%8d" % cgi_err.path.bytesize) - cgi_in.write(cgi_err.path) - cgi_in.write("%8d" % dump.bytesize) - cgi_in.write(dump) - - req.body { |chunk| cgi_in.write(chunk) } - ensure - cgi_in.close - status = $?.exitstatus - sleep 0.1 if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM - data = cgi_out.read - cgi_out.close(true) - if errmsg = cgi_err.read - if errmsg.bytesize > 0 - @logger.error("CGIHandler: #{@script_filename}:\n" + errmsg) - end - end - cgi_err.close(true) - end - - if status != 0 - @logger.error("CGIHandler: #{@script_filename} exit with #{status}") - end - - data = "" unless data - raw_header, body = data.split(/^[\xd\xa]+/, 2) - raise HTTPStatus::InternalServerError, - "Premature end of script headers: #{@script_filename}" if body.nil? - - begin - header = HTTPUtils::parse_header(raw_header) - if /^(\d+)/ =~ header['status'][0] - res.status = $1.to_i - header.delete('status') - end - if header.has_key?('location') - # RFC 3875 6.2.3, 6.2.4 - res.status = 302 unless (300...400) === res.status - end - if header.has_key?('set-cookie') - header['set-cookie'].each{|k| - res.cookies << Cookie.parse_set_cookie(k) - } - header.delete('set-cookie') - end - header.each{|key, val| res[key] = val.join(", ") } - rescue => ex - raise HTTPStatus::InternalServerError, ex.message - end - res.body = body - end - alias do_POST do_GET - - # :startdoc: - end - - end -end diff --git a/tool/lib/webrick/httpservlet/erbhandler.rb b/tool/lib/webrick/httpservlet/erbhandler.rb deleted file mode 100644 index cd09e5f216..0000000000 --- a/tool/lib/webrick/httpservlet/erbhandler.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: false -# -# erbhandler.rb -- ERBHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: erbhandler.rb,v 1.25 2003/02/24 19:25:31 gotoyuzo Exp $ - -require_relative 'abstract' - -require 'erb' - -module WEBrick - module HTTPServlet - - ## - # ERBHandler evaluates an ERB file and returns the result. This handler - # is automatically used if there are .rhtml files in a directory served by - # the FileHandler. - # - # ERBHandler supports GET and POST methods. - # - # The ERB file is evaluated with the local variables +servlet_request+ and - # +servlet_response+ which are a WEBrick::HTTPRequest and - # WEBrick::HTTPResponse respectively. - # - # Example .rhtml file: - # - # Request to <%= servlet_request.request_uri %> - # - # Query params <%= servlet_request.query.inspect %> - - class ERBHandler < AbstractServlet - - ## - # Creates a new ERBHandler on +server+ that will evaluate and serve the - # ERB file +name+ - - def initialize(server, name) - super(server, name) - @script_filename = name - end - - ## - # Handles GET requests - - def do_GET(req, res) - unless defined?(ERB) - @logger.warn "#{self.class}: ERB not defined." - raise HTTPStatus::Forbidden, "ERBHandler cannot work." - end - begin - data = File.open(@script_filename, &:read) - res.body = evaluate(ERB.new(data), req, res) - res['content-type'] ||= - HTTPUtils::mime_type(@script_filename, @config[:MimeTypes]) - rescue StandardError - raise - rescue Exception => ex - @logger.error(ex) - raise HTTPStatus::InternalServerError, ex.message - end - end - - ## - # Handles POST requests - - alias do_POST do_GET - - private - - ## - # Evaluates +erb+ providing +servlet_request+ and +servlet_response+ as - # local variables. - - def evaluate(erb, servlet_request, servlet_response) - Module.new.module_eval{ - servlet_request.meta_vars - servlet_request.query - erb.result(binding) - } - end - end - end -end diff --git a/tool/lib/webrick/httpservlet/filehandler.rb b/tool/lib/webrick/httpservlet/filehandler.rb deleted file mode 100644 index 010df0e918..0000000000 --- a/tool/lib/webrick/httpservlet/filehandler.rb +++ /dev/null @@ -1,552 +0,0 @@ -# frozen_string_literal: false -# -# filehandler.rb -- FileHandler Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: filehandler.rb,v 1.44 2003/06/07 01:34:51 gotoyuzo Exp $ - -require 'time' - -require_relative '../htmlutils' -require_relative '../httputils' -require_relative '../httpstatus' - -module WEBrick - module HTTPServlet - - ## - # Servlet for serving a single file. You probably want to use the - # FileHandler servlet instead as it handles directories and fancy indexes. - # - # Example: - # - # server.mount('/my_page.txt', WEBrick::HTTPServlet::DefaultFileHandler, - # '/path/to/my_page.txt') - # - # This servlet handles If-Modified-Since and Range requests. - - class DefaultFileHandler < AbstractServlet - - ## - # Creates a DefaultFileHandler instance for the file at +local_path+. - - def initialize(server, local_path) - super(server, local_path) - @local_path = local_path - end - - # :stopdoc: - - def do_GET(req, res) - st = File::stat(@local_path) - mtime = st.mtime - res['etag'] = sprintf("%x-%x-%x", st.ino, st.size, st.mtime.to_i) - - if not_modified?(req, res, mtime, res['etag']) - res.body = '' - raise HTTPStatus::NotModified - elsif req['range'] - make_partial_content(req, res, @local_path, st.size) - raise HTTPStatus::PartialContent - else - mtype = HTTPUtils::mime_type(@local_path, @config[:MimeTypes]) - res['content-type'] = mtype - res['content-length'] = st.size.to_s - res['last-modified'] = mtime.httpdate - res.body = File.open(@local_path, "rb") - end - end - - def not_modified?(req, res, mtime, etag) - if ir = req['if-range'] - begin - if Time.httpdate(ir) >= mtime - return true - end - rescue - if HTTPUtils::split_header_value(ir).member?(res['etag']) - return true - end - end - end - - if (ims = req['if-modified-since']) && Time.parse(ims) >= mtime - return true - end - - if (inm = req['if-none-match']) && - HTTPUtils::split_header_value(inm).member?(res['etag']) - return true - end - - return false - end - - # returns a lambda for webrick/httpresponse.rb send_body_proc - def multipart_body(body, parts, boundary, mtype, filesize) - lambda do |socket| - begin - begin - first = parts.shift - last = parts.shift - socket.write( - "--#{boundary}#{CRLF}" \ - "Content-Type: #{mtype}#{CRLF}" \ - "Content-Range: bytes #{first}-#{last}/#{filesize}#{CRLF}" \ - "#{CRLF}" - ) - - begin - IO.copy_stream(body, socket, last - first + 1, first) - rescue NotImplementedError - body.seek(first, IO::SEEK_SET) - IO.copy_stream(body, socket, last - first + 1) - end - socket.write(CRLF) - end while parts[0] - socket.write("--#{boundary}--#{CRLF}") - ensure - body.close - end - end - end - - def make_partial_content(req, res, filename, filesize) - mtype = HTTPUtils::mime_type(filename, @config[:MimeTypes]) - unless ranges = HTTPUtils::parse_range_header(req['range']) - raise HTTPStatus::BadRequest, - "Unrecognized range-spec: \"#{req['range']}\"" - end - File.open(filename, "rb"){|io| - if ranges.size > 1 - time = Time.now - boundary = "#{time.sec}_#{time.usec}_#{Process::pid}" - parts = [] - ranges.each {|range| - prange = prepare_range(range, filesize) - next if prange[0] < 0 - parts.concat(prange) - } - raise HTTPStatus::RequestRangeNotSatisfiable if parts.empty? - res["content-type"] = "multipart/byteranges; boundary=#{boundary}" - if req.http_version < '1.1' - res['connection'] = 'close' - else - res.chunked = true - end - res.body = multipart_body(io.dup, parts, boundary, mtype, filesize) - elsif range = ranges[0] - first, last = prepare_range(range, filesize) - raise HTTPStatus::RequestRangeNotSatisfiable if first < 0 - res['content-type'] = mtype - res['content-range'] = "bytes #{first}-#{last}/#{filesize}" - res['content-length'] = (last - first + 1).to_s - res.body = io.dup - else - raise HTTPStatus::BadRequest - end - } - end - - def prepare_range(range, filesize) - first = range.first < 0 ? filesize + range.first : range.first - return -1, -1 if first < 0 || first >= filesize - last = range.last < 0 ? filesize + range.last : range.last - last = filesize - 1 if last >= filesize - return first, last - end - - # :startdoc: - end - - ## - # Serves a directory including fancy indexing and a variety of other - # options. - # - # Example: - # - # server.mount('/assets', WEBrick::HTTPServlet::FileHandler, - # '/path/to/assets') - - class FileHandler < AbstractServlet - HandlerTable = Hash.new # :nodoc: - - ## - # Allow custom handling of requests for files with +suffix+ by class - # +handler+ - - def self.add_handler(suffix, handler) - HandlerTable[suffix] = handler - end - - ## - # Remove custom handling of requests for files with +suffix+ - - def self.remove_handler(suffix) - HandlerTable.delete(suffix) - end - - ## - # Creates a FileHandler servlet on +server+ that serves files starting - # at directory +root+ - # - # +options+ may be a Hash containing keys from - # WEBrick::Config::FileHandler or +true+ or +false+. - # - # If +options+ is true or false then +:FancyIndexing+ is enabled or - # disabled respectively. - - def initialize(server, root, options={}, default=Config::FileHandler) - @config = server.config - @logger = @config[:Logger] - @root = File.expand_path(root) - if options == true || options == false - options = { :FancyIndexing => options } - end - @options = default.dup.update(options) - end - - # :stopdoc: - - def set_filesystem_encoding(str) - enc = Encoding.find('filesystem') - if enc == Encoding::US_ASCII - str.b - else - str.dup.force_encoding(enc) - end - end - - def service(req, res) - # if this class is mounted on "/" and /~username is requested. - # we're going to override path information before invoking service. - if defined?(Etc) && @options[:UserDir] && req.script_name.empty? - if %r|^(/~([^/]+))| =~ req.path_info - script_name, user = $1, $2 - path_info = $' - begin - passwd = Etc::getpwnam(user) - @root = File::join(passwd.dir, @options[:UserDir]) - req.script_name = script_name - req.path_info = path_info - rescue - @logger.debug "#{self.class}#do_GET: getpwnam(#{user}) failed" - end - end - end - prevent_directory_traversal(req, res) - super(req, res) - end - - def do_GET(req, res) - unless exec_handler(req, res) - set_dir_list(req, res) - end - end - - def do_POST(req, res) - unless exec_handler(req, res) - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - def do_OPTIONS(req, res) - unless exec_handler(req, res) - super(req, res) - end - end - - # ToDo - # RFC2518: HTTP Extensions for Distributed Authoring -- WEBDAV - # - # PROPFIND PROPPATCH MKCOL DELETE PUT COPY MOVE - # LOCK UNLOCK - - # RFC3253: Versioning Extensions to WebDAV - # (Web Distributed Authoring and Versioning) - # - # VERSION-CONTROL REPORT CHECKOUT CHECK_IN UNCHECKOUT - # MKWORKSPACE UPDATE LABEL MERGE ACTIVITY - - private - - def trailing_pathsep?(path) - # check for trailing path separator: - # File.dirname("/aaaa/bbbb/") #=> "/aaaa") - # File.dirname("/aaaa/bbbb/x") #=> "/aaaa/bbbb") - # File.dirname("/aaaa/bbbb") #=> "/aaaa") - # File.dirname("/aaaa/bbbbx") #=> "/aaaa") - return File.dirname(path) != File.dirname(path+"x") - end - - def prevent_directory_traversal(req, res) - # Preventing directory traversal on Windows platforms; - # Backslashes (0x5c) in path_info are not interpreted as special - # character in URI notation. So the value of path_info should be - # normalize before accessing to the filesystem. - - # dirty hack for filesystem encoding; in nature, File.expand_path - # should not be used for path normalization. [Bug #3345] - path = req.path_info.dup.force_encoding(Encoding.find("filesystem")) - if trailing_pathsep?(req.path_info) - # File.expand_path removes the trailing path separator. - # Adding a character is a workaround to save it. - # File.expand_path("/aaa/") #=> "/aaa" - # File.expand_path("/aaa/" + "x") #=> "/aaa/x" - expanded = File.expand_path(path + "x") - expanded.chop! # remove trailing "x" - else - expanded = File.expand_path(path) - end - expanded.force_encoding(req.path_info.encoding) - req.path_info = expanded - end - - def exec_handler(req, res) - raise HTTPStatus::NotFound, "`#{req.path}' not found." unless @root - if set_filename(req, res) - handler = get_handler(req, res) - call_callback(:HandlerCallback, req, res) - h = handler.get_instance(@config, res.filename) - h.service(req, res) - return true - end - call_callback(:HandlerCallback, req, res) - return false - end - - def get_handler(req, res) - suffix1 = (/\.(\w+)\z/ =~ res.filename) && $1.downcase - if /\.(\w+)\.([\w\-]+)\z/ =~ res.filename - if @options[:AcceptableLanguages].include?($2.downcase) - suffix2 = $1.downcase - end - end - handler_table = @options[:HandlerTable] - return handler_table[suffix1] || handler_table[suffix2] || - HandlerTable[suffix1] || HandlerTable[suffix2] || - DefaultFileHandler - end - - def set_filename(req, res) - res.filename = @root - path_info = req.path_info.scan(%r|/[^/]*|) - - path_info.unshift("") # dummy for checking @root dir - while base = path_info.first - base = set_filesystem_encoding(base) - break if base == "/" - break unless File.directory?(File.expand_path(res.filename + base)) - shift_path_info(req, res, path_info) - call_callback(:DirectoryCallback, req, res) - end - - if base = path_info.first - base = set_filesystem_encoding(base) - if base == "/" - if file = search_index_file(req, res) - shift_path_info(req, res, path_info, file) - call_callback(:FileCallback, req, res) - return true - end - shift_path_info(req, res, path_info) - elsif file = search_file(req, res, base) - shift_path_info(req, res, path_info, file) - call_callback(:FileCallback, req, res) - return true - else - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - return false - end - - def check_filename(req, res, name) - if nondisclosure_name?(name) || windows_ambiguous_name?(name) - @logger.warn("the request refers nondisclosure name `#{name}'.") - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - def shift_path_info(req, res, path_info, base=nil) - tmp = path_info.shift - base = base || set_filesystem_encoding(tmp) - req.path_info = path_info.join - req.script_name << base - res.filename = File.expand_path(res.filename + base) - check_filename(req, res, File.basename(res.filename)) - end - - def search_index_file(req, res) - @config[:DirectoryIndex].each{|index| - if file = search_file(req, res, "/"+index) - return file - end - } - return nil - end - - def search_file(req, res, basename) - langs = @options[:AcceptableLanguages] - path = res.filename + basename - if File.file?(path) - return basename - elsif langs.size > 0 - req.accept_language.each{|lang| - path_with_lang = path + ".#{lang}" - if langs.member?(lang) && File.file?(path_with_lang) - return basename + ".#{lang}" - end - } - (langs - req.accept_language).each{|lang| - path_with_lang = path + ".#{lang}" - if File.file?(path_with_lang) - return basename + ".#{lang}" - end - } - end - return nil - end - - def call_callback(callback_name, req, res) - if cb = @options[callback_name] - cb.call(req, res) - end - end - - def windows_ambiguous_name?(name) - return true if /[. ]+\z/ =~ name - return true if /::\$DATA\z/ =~ name - return false - end - - def nondisclosure_name?(name) - @options[:NondisclosureName].each{|pattern| - if File.fnmatch(pattern, name, File::FNM_CASEFOLD) - return true - end - } - return false - end - - def set_dir_list(req, res) - redirect_to_directory_uri(req, res) - unless @options[:FancyIndexing] - raise HTTPStatus::Forbidden, "no access permission to `#{req.path}'" - end - local_path = res.filename - list = Dir::entries(local_path).collect{|name| - next if name == "." || name == ".." - next if nondisclosure_name?(name) - next if windows_ambiguous_name?(name) - st = (File::stat(File.join(local_path, name)) rescue nil) - if st.nil? - [ name, nil, -1 ] - elsif st.directory? - [ name + "/", st.mtime, -1 ] - else - [ name, st.mtime, st.size ] - end - } - list.compact! - - query = req.query - - d0 = nil - idx = nil - %w[N M S].each_with_index do |q, i| - if d = query.delete(q) - idx ||= i - d0 ||= d - end - end - d0 ||= "A" - idx ||= 0 - d1 = (d0 == "A") ? "D" : "A" - - if d0 == "A" - list.sort!{|a,b| a[idx] <=> b[idx] } - else - list.sort!{|a,b| b[idx] <=> a[idx] } - end - - namewidth = query["NameWidth"] - if namewidth == "*" - namewidth = nil - elsif !namewidth or (namewidth = namewidth.to_i) < 2 - namewidth = 25 - end - query = query.inject('') {|s, (k, v)| s << '&' << HTMLUtils::escape("#{k}=#{v}")} - - type = "text/html" - case enc = Encoding.find('filesystem') - when Encoding::US_ASCII, Encoding::ASCII_8BIT - else - type << "; charset=\"#{enc.name}\"" - end - res['content-type'] = type - - title = "Index of #{HTMLUtils::escape(req.path)}" - res.body = <<-_end_of_html_ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> -<HTML> - <HEAD> - <TITLE>#{title}</TITLE> - <style type="text/css"> - <!-- - .name, .mtime { text-align: left; } - .size { text-align: right; } - td { text-overflow: ellipsis; white-space: nowrap; overflow: hidden; } - table { border-collapse: collapse; } - tr th { border-bottom: 2px groove; } - //--> - </style> - </HEAD> - <BODY> - <H1>#{title}</H1> - _end_of_html_ - - res.body << "<TABLE width=\"100%\"><THEAD><TR>\n" - res.body << "<TH class=\"name\"><A HREF=\"?N=#{d1}#{query}\">Name</A></TH>" - res.body << "<TH class=\"mtime\"><A HREF=\"?M=#{d1}#{query}\">Last modified</A></TH>" - res.body << "<TH class=\"size\"><A HREF=\"?S=#{d1}#{query}\">Size</A></TH>\n" - res.body << "</TR></THEAD>\n" - res.body << "<TBODY>\n" - - query.sub!(/\A&/, '?') - list.unshift [ "..", File::mtime(local_path+"/.."), -1 ] - list.each{ |name, time, size| - if name == ".." - dname = "Parent Directory" - elsif namewidth and name.size > namewidth - dname = name[0...(namewidth - 2)] << '..' - else - dname = name - end - s = "<TR><TD class=\"name\"><A HREF=\"#{HTTPUtils::escape(name)}#{query if name.end_with?('/')}\">#{HTMLUtils::escape(dname)}</A></TD>" - s << "<TD class=\"mtime\">" << (time ? time.strftime("%Y/%m/%d %H:%M") : "") << "</TD>" - s << "<TD class=\"size\">" << (size >= 0 ? size.to_s : "-") << "</TD></TR>\n" - res.body << s - } - res.body << "</TBODY></TABLE>" - res.body << "<HR>" - - res.body << <<-_end_of_html_ - <ADDRESS> - #{HTMLUtils::escape(@config[:ServerSoftware])}<BR> - at #{req.host}:#{req.port} - </ADDRESS> - </BODY> -</HTML> - _end_of_html_ - end - - # :startdoc: - end - end -end diff --git a/tool/lib/webrick/httpservlet/prochandler.rb b/tool/lib/webrick/httpservlet/prochandler.rb deleted file mode 100644 index 599ffc4340..0000000000 --- a/tool/lib/webrick/httpservlet/prochandler.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: false -# -# prochandler.rb -- ProcHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: prochandler.rb,v 1.7 2002/09/21 12:23:42 gotoyuzo Exp $ - -require_relative 'abstract' - -module WEBrick - module HTTPServlet - - ## - # Mounts a proc at a path that accepts a request and response. - # - # Instead of mounting this servlet with WEBrick::HTTPServer#mount use - # WEBrick::HTTPServer#mount_proc: - # - # server.mount_proc '/' do |req, res| - # res.body = 'it worked!' - # res.status = 200 - # end - - class ProcHandler < AbstractServlet - # :stopdoc: - def get_instance(server, *options) - self - end - - def initialize(proc) - @proc = proc - end - - def do_GET(request, response) - @proc.call(request, response) - end - - alias do_POST do_GET - # :startdoc: - end - - end -end diff --git a/tool/lib/webrick/httpstatus.rb b/tool/lib/webrick/httpstatus.rb deleted file mode 100644 index c811f21964..0000000000 --- a/tool/lib/webrick/httpstatus.rb +++ /dev/null @@ -1,194 +0,0 @@ -# frozen_string_literal: false -#-- -# httpstatus.rb -- HTTPStatus Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpstatus.rb,v 1.11 2003/03/24 20:18:55 gotoyuzo Exp $ - -require_relative 'accesslog' - -module WEBrick - - ## - # This module is used to manager HTTP status codes. - # - # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html for more - # information. - module HTTPStatus - - ## - # Root of the HTTP status class hierarchy - class Status < StandardError - class << self - attr_reader :code, :reason_phrase # :nodoc: - end - - # Returns the HTTP status code - def code() self::class::code end - - # Returns the HTTP status description - def reason_phrase() self::class::reason_phrase end - - alias to_i code # :nodoc: - end - - # Root of the HTTP info statuses - class Info < Status; end - # Root of the HTTP success statuses - class Success < Status; end - # Root of the HTTP redirect statuses - class Redirect < Status; end - # Root of the HTTP error statuses - class Error < Status; end - # Root of the HTTP client error statuses - class ClientError < Error; end - # Root of the HTTP server error statuses - class ServerError < Error; end - - class EOFError < StandardError; end - - # HTTP status codes and descriptions - StatusMessage = { # :nodoc: - 100 => 'Continue', - 101 => 'Switching Protocols', - 200 => 'OK', - 201 => 'Created', - 202 => 'Accepted', - 203 => 'Non-Authoritative Information', - 204 => 'No Content', - 205 => 'Reset Content', - 206 => 'Partial Content', - 207 => 'Multi-Status', - 300 => 'Multiple Choices', - 301 => 'Moved Permanently', - 302 => 'Found', - 303 => 'See Other', - 304 => 'Not Modified', - 305 => 'Use Proxy', - 307 => 'Temporary Redirect', - 400 => 'Bad Request', - 401 => 'Unauthorized', - 402 => 'Payment Required', - 403 => 'Forbidden', - 404 => 'Not Found', - 405 => 'Method Not Allowed', - 406 => 'Not Acceptable', - 407 => 'Proxy Authentication Required', - 408 => 'Request Timeout', - 409 => 'Conflict', - 410 => 'Gone', - 411 => 'Length Required', - 412 => 'Precondition Failed', - 413 => 'Request Entity Too Large', - 414 => 'Request-URI Too Large', - 415 => 'Unsupported Media Type', - 416 => 'Request Range Not Satisfiable', - 417 => 'Expectation Failed', - 422 => 'Unprocessable Entity', - 423 => 'Locked', - 424 => 'Failed Dependency', - 426 => 'Upgrade Required', - 428 => 'Precondition Required', - 429 => 'Too Many Requests', - 431 => 'Request Header Fields Too Large', - 451 => 'Unavailable For Legal Reasons', - 500 => 'Internal Server Error', - 501 => 'Not Implemented', - 502 => 'Bad Gateway', - 503 => 'Service Unavailable', - 504 => 'Gateway Timeout', - 505 => 'HTTP Version Not Supported', - 507 => 'Insufficient Storage', - 511 => 'Network Authentication Required', - } - - # Maps a status code to the corresponding Status class - CodeToError = {} # :nodoc: - - # Creates a status or error class for each status code and - # populates the CodeToError map. - StatusMessage.each{|code, message| - message.freeze - var_name = message.gsub(/[ \-]/,'_').upcase - err_name = message.gsub(/[ \-]/,'') - - case code - when 100...200; parent = Info - when 200...300; parent = Success - when 300...400; parent = Redirect - when 400...500; parent = ClientError - when 500...600; parent = ServerError - end - - const_set("RC_#{var_name}", code) - err_class = Class.new(parent) - err_class.instance_variable_set(:@code, code) - err_class.instance_variable_set(:@reason_phrase, message) - const_set(err_name, err_class) - CodeToError[code] = err_class - } - - ## - # Returns the description corresponding to the HTTP status +code+ - # - # WEBrick::HTTPStatus.reason_phrase 404 - # => "Not Found" - def reason_phrase(code) - StatusMessage[code.to_i] - end - - ## - # Is +code+ an informational status? - def info?(code) - code.to_i >= 100 and code.to_i < 200 - end - - ## - # Is +code+ a successful status? - def success?(code) - code.to_i >= 200 and code.to_i < 300 - end - - ## - # Is +code+ a redirection status? - def redirect?(code) - code.to_i >= 300 and code.to_i < 400 - end - - ## - # Is +code+ an error status? - def error?(code) - code.to_i >= 400 and code.to_i < 600 - end - - ## - # Is +code+ a client error status? - def client_error?(code) - code.to_i >= 400 and code.to_i < 500 - end - - ## - # Is +code+ a server error status? - def server_error?(code) - code.to_i >= 500 and code.to_i < 600 - end - - ## - # Returns the status class corresponding to +code+ - # - # WEBrick::HTTPStatus[302] - # => WEBrick::HTTPStatus::NotFound - # - def self.[](code) - CodeToError[code] - end - - module_function :reason_phrase - module_function :info?, :success?, :redirect?, :error? - module_function :client_error?, :server_error? - end -end diff --git a/tool/lib/webrick/httputils.rb b/tool/lib/webrick/httputils.rb deleted file mode 100644 index e21284ee7f..0000000000 --- a/tool/lib/webrick/httputils.rb +++ /dev/null @@ -1,512 +0,0 @@ -# frozen_string_literal: false -# -# httputils.rb -- HTTPUtils Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httputils.rb,v 1.34 2003/06/05 21:34:08 gotoyuzo Exp $ - -require 'socket' -require 'tempfile' - -module WEBrick - CR = "\x0d" # :nodoc: - LF = "\x0a" # :nodoc: - CRLF = "\x0d\x0a" # :nodoc: - - ## - # HTTPUtils provides utility methods for working with the HTTP protocol. - # - # This module is generally used internally by WEBrick - - module HTTPUtils - - ## - # Normalizes a request path. Raises an exception if the path cannot be - # normalized. - - def normalize_path(path) - raise "abnormal path `#{path}'" if path[0] != ?/ - ret = path.dup - - ret.gsub!(%r{/+}o, '/') # // => / - while ret.sub!(%r'/\.(?:/|\Z)', '/'); end # /. => / - while ret.sub!(%r'/(?!\.\./)[^/]+/\.\.(?:/|\Z)', '/'); end # /foo/.. => /foo - - raise "abnormal path `#{path}'" if %r{/\.\.(/|\Z)} =~ ret - ret - end - module_function :normalize_path - - ## - # Default mime types - - DefaultMimeTypes = { - "ai" => "application/postscript", - "asc" => "text/plain", - "avi" => "video/x-msvideo", - "bin" => "application/octet-stream", - "bmp" => "image/bmp", - "class" => "application/octet-stream", - "cer" => "application/pkix-cert", - "crl" => "application/pkix-crl", - "crt" => "application/x-x509-ca-cert", - #"crl" => "application/x-pkcs7-crl", - "css" => "text/css", - "dms" => "application/octet-stream", - "doc" => "application/msword", - "dvi" => "application/x-dvi", - "eps" => "application/postscript", - "etx" => "text/x-setext", - "exe" => "application/octet-stream", - "gif" => "image/gif", - "htm" => "text/html", - "html" => "text/html", - "jpe" => "image/jpeg", - "jpeg" => "image/jpeg", - "jpg" => "image/jpeg", - "js" => "application/javascript", - "json" => "application/json", - "lha" => "application/octet-stream", - "lzh" => "application/octet-stream", - "mjs" => "application/javascript", - "mov" => "video/quicktime", - "mpe" => "video/mpeg", - "mpeg" => "video/mpeg", - "mpg" => "video/mpeg", - "pbm" => "image/x-portable-bitmap", - "pdf" => "application/pdf", - "pgm" => "image/x-portable-graymap", - "png" => "image/png", - "pnm" => "image/x-portable-anymap", - "ppm" => "image/x-portable-pixmap", - "ppt" => "application/vnd.ms-powerpoint", - "ps" => "application/postscript", - "qt" => "video/quicktime", - "ras" => "image/x-cmu-raster", - "rb" => "text/plain", - "rd" => "text/plain", - "rtf" => "application/rtf", - "sgm" => "text/sgml", - "sgml" => "text/sgml", - "svg" => "image/svg+xml", - "tif" => "image/tiff", - "tiff" => "image/tiff", - "txt" => "text/plain", - "wasm" => "application/wasm", - "xbm" => "image/x-xbitmap", - "xhtml" => "text/html", - "xls" => "application/vnd.ms-excel", - "xml" => "text/xml", - "xpm" => "image/x-xpixmap", - "xwd" => "image/x-xwindowdump", - "zip" => "application/zip", - } - - ## - # Loads Apache-compatible mime.types in +file+. - - def load_mime_types(file) - # note: +file+ may be a "| command" for now; some people may - # rely on this, but currently we do not use this method by default. - File.open(file){ |io| - hash = Hash.new - io.each{ |line| - next if /^#/ =~ line - line.chomp! - mimetype, ext0 = line.split(/\s+/, 2) - next unless ext0 - next if ext0.empty? - ext0.split(/\s+/).each{ |ext| hash[ext] = mimetype } - } - hash - } - end - module_function :load_mime_types - - ## - # Returns the mime type of +filename+ from the list in +mime_tab+. If no - # mime type was found application/octet-stream is returned. - - def mime_type(filename, mime_tab) - suffix1 = (/\.(\w+)$/ =~ filename && $1.downcase) - suffix2 = (/\.(\w+)\.[\w\-]+$/ =~ filename && $1.downcase) - mime_tab[suffix1] || mime_tab[suffix2] || "application/octet-stream" - end - module_function :mime_type - - ## - # Parses an HTTP header +raw+ into a hash of header fields with an Array - # of values. - - def parse_header(raw) - header = Hash.new([].freeze) - field = nil - raw.each_line{|line| - case line - when /^([A-Za-z0-9!\#$%&'*+\-.^_`|~]+):\s*(.*?)\s*\z/om - field, value = $1, $2 - field.downcase! - header[field] = [] unless header.has_key?(field) - header[field] << value - when /^\s+(.*?)\s*\z/om - value = $1 - unless field - raise HTTPStatus::BadRequest, "bad header '#{line}'." - end - header[field][-1] << " " << value - else - raise HTTPStatus::BadRequest, "bad header '#{line}'." - end - } - header.each{|key, values| - values.each(&:strip!) - } - header - end - module_function :parse_header - - ## - # Splits a header value +str+ according to HTTP specification. - - def split_header_value(str) - str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",]+)+) - (?:,\s*|\Z)'xn).flatten - end - module_function :split_header_value - - ## - # Parses a Range header value +ranges_specifier+ - - def parse_range_header(ranges_specifier) - if /^bytes=(.*)/ =~ ranges_specifier - byte_range_set = split_header_value($1) - byte_range_set.collect{|range_spec| - case range_spec - when /^(\d+)-(\d+)/ then $1.to_i .. $2.to_i - when /^(\d+)-/ then $1.to_i .. -1 - when /^-(\d+)/ then -($1.to_i) .. -1 - else return nil - end - } - end - end - module_function :parse_range_header - - ## - # Parses q values in +value+ as used in Accept headers. - - def parse_qvalues(value) - tmp = [] - if value - parts = value.split(/,\s*/) - parts.each {|part| - if m = %r{^([^\s,]+?)(?:;\s*q=(\d+(?:\.\d+)?))?$}.match(part) - val = m[1] - q = (m[2] or 1).to_f - tmp.push([val, q]) - end - } - tmp = tmp.sort_by{|val, q| -q} - tmp.collect!{|val, q| val} - end - return tmp - end - module_function :parse_qvalues - - ## - # Removes quotes and escapes from +str+ - - def dequote(str) - ret = (/\A"(.*)"\Z/ =~ str) ? $1 : str.dup - ret.gsub!(/\\(.)/, "\\1") - ret - end - module_function :dequote - - ## - # Quotes and escapes quotes in +str+ - - def quote(str) - '"' << str.gsub(/[\\\"]/o, "\\\1") << '"' - end - module_function :quote - - ## - # Stores multipart form data. FormData objects are created when - # WEBrick::HTTPUtils.parse_form_data is called. - - class FormData < String - EmptyRawHeader = [].freeze # :nodoc: - EmptyHeader = {}.freeze # :nodoc: - - ## - # The name of the form data part - - attr_accessor :name - - ## - # The filename of the form data part - - attr_accessor :filename - - attr_accessor :next_data # :nodoc: - protected :next_data - - ## - # Creates a new FormData object. - # - # +args+ is an Array of form data entries. One FormData will be created - # for each entry. - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you - - def initialize(*args) - @name = @filename = @next_data = nil - if args.empty? - @raw_header = [] - @header = nil - super("") - else - @raw_header = EmptyRawHeader - @header = EmptyHeader - super(args.shift) - unless args.empty? - @next_data = self.class.new(*args) - end - end - end - - ## - # Retrieves the header at the first entry in +key+ - - def [](*key) - begin - @header[key[0].downcase].join(", ") - rescue StandardError, NameError - super - end - end - - ## - # Adds +str+ to this FormData which may be the body, a header or a - # header entry. - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you - - def <<(str) - if @header - super - elsif str == CRLF - @header = HTTPUtils::parse_header(@raw_header.join) - if cd = self['content-disposition'] - if /\s+name="(.*?)"/ =~ cd then @name = $1 end - if /\s+filename="(.*?)"/ =~ cd then @filename = $1 end - end - else - @raw_header << str - end - self - end - - ## - # Adds +data+ at the end of the chain of entries - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you. - - def append_data(data) - tmp = self - while tmp - unless tmp.next_data - tmp.next_data = data - break - end - tmp = tmp.next_data - end - self - end - - ## - # Yields each entry in this FormData - - def each_data - tmp = self - while tmp - next_data = tmp.next_data - yield(tmp) - tmp = next_data - end - end - - ## - # Returns all the FormData as an Array - - def list - ret = [] - each_data{|data| - ret << data.to_s - } - ret - end - - ## - # A FormData will behave like an Array - - alias :to_ary :list - - ## - # This FormData's body - - def to_s - String.new(self) - end - end - - ## - # Parses the query component of a URI in +str+ - - def parse_query(str) - query = Hash.new - if str - str.split(/[&;]/).each{|x| - next if x.empty? - key, val = x.split(/=/,2) - key = unescape_form(key) - val = unescape_form(val.to_s) - val = FormData.new(val) - val.name = key - if query.has_key?(key) - query[key].append_data(val) - next - end - query[key] = val - } - end - query - end - module_function :parse_query - - ## - # Parses form data in +io+ with the given +boundary+ - - def parse_form_data(io, boundary) - boundary_regexp = /\A--#{Regexp.quote(boundary)}(--)?#{CRLF}\z/ - form_data = Hash.new - return form_data unless io - data = nil - io.each_line{|line| - if boundary_regexp =~ line - if data - data.chop! - key = data.name - if form_data.has_key?(key) - form_data[key].append_data(data) - else - form_data[key] = data - end - end - data = FormData.new - next - else - if data - data << line - end - end - } - return form_data - end - module_function :parse_form_data - - ##### - - reserved = ';/?:@&=+$,' - num = '0123456789' - lowalpha = 'abcdefghijklmnopqrstuvwxyz' - upalpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - mark = '-_.!~*\'()' - unreserved = num + lowalpha + upalpha + mark - control = (0x0..0x1f).collect{|c| c.chr }.join + "\x7f" - space = " " - delims = '<>#%"' - unwise = '{}|\\^[]`' - nonascii = (0x80..0xff).collect{|c| c.chr }.join - - module_function - - # :stopdoc: - - def _make_regex(str) /([#{Regexp.escape(str)}])/n end - def _make_regex!(str) /([^#{Regexp.escape(str)}])/n end - def _escape(str, regex) - str = str.b - str.gsub!(regex) {"%%%02X" % $1.ord} - # %-escaped string should contain US-ASCII only - str.force_encoding(Encoding::US_ASCII) - end - def _unescape(str, regex) - str = str.b - str.gsub!(regex) {$1.hex.chr} - # encoding of %-unescaped string is unknown - str - end - - UNESCAPED = _make_regex(control+space+delims+unwise+nonascii) - UNESCAPED_FORM = _make_regex(reserved+control+delims+unwise+nonascii) - NONASCII = _make_regex(nonascii) - ESCAPED = /%([0-9a-fA-F]{2})/ - UNESCAPED_PCHAR = _make_regex!(unreserved+":@&=+$,") - - # :startdoc: - - ## - # Escapes HTTP reserved and unwise characters in +str+ - - def escape(str) - _escape(str, UNESCAPED) - end - - ## - # Unescapes HTTP reserved and unwise characters in +str+ - - def unescape(str) - _unescape(str, ESCAPED) - end - - ## - # Escapes form reserved characters in +str+ - - def escape_form(str) - ret = _escape(str, UNESCAPED_FORM) - ret.gsub!(/ /, "+") - ret - end - - ## - # Unescapes form reserved characters in +str+ - - def unescape_form(str) - _unescape(str.gsub(/\+/, " "), ESCAPED) - end - - ## - # Escapes path +str+ - - def escape_path(str) - result = "" - str.scan(%r{/([^/]*)}).each{|i| - result << "/" << _escape(i[0], UNESCAPED_PCHAR) - } - return result - end - - ## - # Escapes 8 bit characters in +str+ - - def escape8bit(str) - _escape(str, NONASCII) - end - end -end diff --git a/tool/lib/webrick/httpversion.rb b/tool/lib/webrick/httpversion.rb deleted file mode 100644 index 8a251944a2..0000000000 --- a/tool/lib/webrick/httpversion.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: false -#-- -# HTTPVersion.rb -- presentation of HTTP version -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpversion.rb,v 1.5 2002/09/21 12:23:37 gotoyuzo Exp $ - -module WEBrick - - ## - # Represents an HTTP protocol version - - class HTTPVersion - include Comparable - - ## - # The major protocol version number - - attr_accessor :major - - ## - # The minor protocol version number - - attr_accessor :minor - - ## - # Converts +version+ into an HTTPVersion - - def self.convert(version) - version.is_a?(self) ? version : new(version) - end - - ## - # Creates a new HTTPVersion from +version+. - - def initialize(version) - case version - when HTTPVersion - @major, @minor = version.major, version.minor - when String - if /^(\d+)\.(\d+)$/ =~ version - @major, @minor = $1.to_i, $2.to_i - end - end - if @major.nil? || @minor.nil? - raise ArgumentError, - format("cannot convert %s into %s", version.class, self.class) - end - end - - ## - # Compares this version with +other+ according to the HTTP specification - # rules. - - def <=>(other) - unless other.is_a?(self.class) - other = self.class.new(other) - end - if (ret = @major <=> other.major) == 0 - return @minor <=> other.minor - end - return ret - end - - ## - # The HTTP version as show in the HTTP request and response. For example, - # "1.1" - - def to_s - format("%d.%d", @major, @minor) - end - end -end diff --git a/tool/lib/webrick/log.rb b/tool/lib/webrick/log.rb deleted file mode 100644 index 2c1fdfe602..0000000000 --- a/tool/lib/webrick/log.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: false -#-- -# log.rb -- Log Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: log.rb,v 1.26 2002/10/06 17:06:10 gotoyuzo Exp $ - -module WEBrick - - ## - # A generic logging class - - class BasicLog - - # Fatal log level which indicates a server crash - - FATAL = 1 - - # Error log level which indicates a recoverable error - - ERROR = 2 - - # Warning log level which indicates a possible problem - - WARN = 3 - - # Information log level which indicates possibly useful information - - INFO = 4 - - # Debugging error level for messages used in server development or - # debugging - - DEBUG = 5 - - # log-level, messages above this level will be logged - attr_accessor :level - - ## - # Initializes a new logger for +log_file+ that outputs messages at +level+ - # or higher. +log_file+ can be a filename, an IO-like object that - # responds to #<< or nil which outputs to $stderr. - # - # If no level is given INFO is chosen by default - - def initialize(log_file=nil, level=nil) - @level = level || INFO - case log_file - when String - @log = File.open(log_file, "a+") - @log.sync = true - @opened = true - when NilClass - @log = $stderr - else - @log = log_file # requires "<<". (see BasicLog#log) - end - end - - ## - # Closes the logger (also closes the log device associated to the logger) - def close - @log.close if @opened - @log = nil - end - - ## - # Logs +data+ at +level+ if the given level is above the current log - # level. - - def log(level, data) - if @log && level <= @level - data += "\n" if /\n\Z/ !~ data - @log << data - end - end - - ## - # Synonym for log(INFO, obj.to_s) - def <<(obj) - log(INFO, obj.to_s) - end - - # Shortcut for logging a FATAL message - def fatal(msg) log(FATAL, "FATAL " << format(msg)); end - # Shortcut for logging an ERROR message - def error(msg) log(ERROR, "ERROR " << format(msg)); end - # Shortcut for logging a WARN message - def warn(msg) log(WARN, "WARN " << format(msg)); end - # Shortcut for logging an INFO message - def info(msg) log(INFO, "INFO " << format(msg)); end - # Shortcut for logging a DEBUG message - def debug(msg) log(DEBUG, "DEBUG " << format(msg)); end - - # Will the logger output FATAL messages? - def fatal?; @level >= FATAL; end - # Will the logger output ERROR messages? - def error?; @level >= ERROR; end - # Will the logger output WARN messages? - def warn?; @level >= WARN; end - # Will the logger output INFO messages? - def info?; @level >= INFO; end - # Will the logger output DEBUG messages? - def debug?; @level >= DEBUG; end - - private - - ## - # Formats +arg+ for the logger - # - # * If +arg+ is an Exception, it will format the error message and - # the back trace. - # * If +arg+ responds to #to_str, it will return it. - # * Otherwise it will return +arg+.inspect. - def format(arg) - if arg.is_a?(Exception) - "#{arg.class}: #{AccessLog.escape(arg.message)}\n\t" << - arg.backtrace.join("\n\t") << "\n" - elsif arg.respond_to?(:to_str) - AccessLog.escape(arg.to_str) - else - arg.inspect - end - end - end - - ## - # A logging class that prepends a timestamp to each message. - - class Log < BasicLog - # Format of the timestamp which is applied to each logged line. The - # default is <tt>"[%Y-%m-%d %H:%M:%S]"</tt> - attr_accessor :time_format - - ## - # Same as BasicLog#initialize - # - # You can set the timestamp format through #time_format - def initialize(log_file=nil, level=nil) - super(log_file, level) - @time_format = "[%Y-%m-%d %H:%M:%S]" - end - - ## - # Same as BasicLog#log - def log(level, data) - tmp = Time.now.strftime(@time_format) - tmp << " " << data - super(level, tmp) - end - end -end diff --git a/tool/lib/webrick/server.rb b/tool/lib/webrick/server.rb deleted file mode 100644 index fd6b7a61b5..0000000000 --- a/tool/lib/webrick/server.rb +++ /dev/null @@ -1,381 +0,0 @@ -# frozen_string_literal: false -# -# server.rb -- GenericServer Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: server.rb,v 1.62 2003/07/22 19:20:43 gotoyuzo Exp $ - -require 'socket' -require_relative 'config' -require_relative 'log' - -module WEBrick - - ## - # Server error exception - - class ServerError < StandardError; end - - ## - # Base server class - - class SimpleServer - - ## - # A SimpleServer only yields when you start it - - def SimpleServer.start - yield - end - end - - ## - # A generic module for daemonizing a process - - class Daemon - - ## - # Performs the standard operations for daemonizing a process. Runs a - # block, if given. - - def Daemon.start - Process.daemon - File.umask(0) - yield if block_given? - end - end - - ## - # Base TCP server class. You must subclass GenericServer and provide a #run - # method. - - class GenericServer - - ## - # The server status. One of :Stop, :Running or :Shutdown - - attr_reader :status - - ## - # The server configuration - - attr_reader :config - - ## - # The server logger. This is independent from the HTTP access log. - - attr_reader :logger - - ## - # Tokens control the number of outstanding clients. The - # <code>:MaxClients</code> configuration sets this. - - attr_reader :tokens - - ## - # Sockets listening for connections. - - attr_reader :listeners - - ## - # Creates a new generic server from +config+. The default configuration - # comes from +default+. - - def initialize(config={}, default=Config::General) - @config = default.dup.update(config) - @status = :Stop - @config[:Logger] ||= Log::new - @logger = @config[:Logger] - - @tokens = Thread::SizedQueue.new(@config[:MaxClients]) - @config[:MaxClients].times{ @tokens.push(nil) } - - webrickv = WEBrick::VERSION - rubyv = "#{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]" - @logger.info("WEBrick #{webrickv}") - @logger.info("ruby #{rubyv}") - - @listeners = [] - @shutdown_pipe = nil - unless @config[:DoNotListen] - raise ArgumentError, "Port must an integer" unless @config[:Port].to_s == @config[:Port].to_i.to_s - - @config[:Port] = @config[:Port].to_i - if @config[:Listen] - warn(":Listen option is deprecated; use GenericServer#listen", uplevel: 1) - end - listen(@config[:BindAddress], @config[:Port]) - if @config[:Port] == 0 - @config[:Port] = @listeners[0].addr[1] - end - end - end - - ## - # Retrieves +key+ from the configuration - - def [](key) - @config[key] - end - - ## - # Adds listeners from +address+ and +port+ to the server. See - # WEBrick::Utils::create_listeners for details. - - def listen(address, port) - @listeners += Utils::create_listeners(address, port) - end - - ## - # Starts the server and runs the +block+ for each connection. This method - # does not return until the server is stopped from a signal handler or - # another thread using #stop or #shutdown. - # - # If the block raises a subclass of StandardError the exception is logged - # and ignored. If an IOError or Errno::EBADF exception is raised the - # exception is ignored. If an Exception subclass is raised the exception - # is logged and re-raised which stops the server. - # - # To completely shut down a server call #shutdown from ensure: - # - # server = WEBrick::GenericServer.new - # # or WEBrick::HTTPServer.new - # - # begin - # server.start - # ensure - # server.shutdown - # end - - def start(&block) - raise ServerError, "already started." if @status != :Stop - server_type = @config[:ServerType] || SimpleServer - - setup_shutdown_pipe - - server_type.start{ - @logger.info \ - "#{self.class}#start: pid=#{$$} port=#{@config[:Port]}" - @status = :Running - call_callback(:StartCallback) - - shutdown_pipe = @shutdown_pipe - - thgroup = ThreadGroup.new - begin - while @status == :Running - begin - sp = shutdown_pipe[0] - if svrs = IO.select([sp, *@listeners]) - if svrs[0].include? sp - # swallow shutdown pipe - buf = String.new - nil while String === - sp.read_nonblock([sp.nread, 8].max, buf, exception: false) - break - end - svrs[0].each{|svr| - @tokens.pop # blocks while no token is there. - if sock = accept_client(svr) - unless config[:DoNotReverseLookup].nil? - sock.do_not_reverse_lookup = !!config[:DoNotReverseLookup] - end - th = start_thread(sock, &block) - th[:WEBrickThread] = true - thgroup.add(th) - else - @tokens.push(nil) - end - } - end - rescue Errno::EBADF, Errno::ENOTSOCK, IOError => ex - # if the listening socket was closed in GenericServer#shutdown, - # IO::select raise it. - rescue StandardError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - rescue Exception => ex - @logger.fatal ex - raise - end - end - ensure - cleanup_shutdown_pipe(shutdown_pipe) - cleanup_listener - @status = :Shutdown - @logger.info "going to shutdown ..." - thgroup.list.each{|th| th.join if th[:WEBrickThread] } - call_callback(:StopCallback) - @logger.info "#{self.class}#start done." - @status = :Stop - end - } - end - - ## - # Stops the server from accepting new connections. - - def stop - if @status == :Running - @status = :Shutdown - end - - alarm_shutdown_pipe {|f| f.write_nonblock("\0")} - end - - ## - # Shuts down the server and all listening sockets. New listeners must be - # provided to restart the server. - - def shutdown - stop - - alarm_shutdown_pipe(&:close) - end - - ## - # You must subclass GenericServer and implement \#run which accepts a TCP - # client socket - - def run(sock) - @logger.fatal "run() must be provided by user." - end - - private - - # :stopdoc: - - ## - # Accepts a TCP client socket from the TCP server socket +svr+ and returns - # the client socket. - - def accept_client(svr) - case sock = svr.to_io.accept_nonblock(exception: false) - when :wait_readable - nil - else - if svr.respond_to?(:start_immediately) - sock = OpenSSL::SSL::SSLSocket.new(sock, ssl_context) - sock.sync_close = true - # we cannot do OpenSSL::SSL::SSLSocket#accept here because - # a slow client can prevent us from accepting connections - # from other clients - end - sock - end - rescue Errno::ECONNRESET, Errno::ECONNABORTED, - Errno::EPROTO, Errno::EINVAL - nil - rescue StandardError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - nil - end - - ## - # Starts a server thread for the client socket +sock+ that runs the given - # +block+. - # - # Sets the socket to the <code>:WEBrickSocket</code> thread local variable - # in the thread. - # - # If any errors occur in the block they are logged and handled. - - def start_thread(sock, &block) - Thread.start{ - begin - Thread.current[:WEBrickSocket] = sock - begin - addr = sock.peeraddr - @logger.debug "accept: #{addr[3]}:#{addr[1]}" - rescue SocketError - @logger.debug "accept: <address unknown>" - raise - end - if sock.respond_to?(:sync_close=) && @config[:SSLStartImmediately] - WEBrick::Utils.timeout(@config[:RequestTimeout]) do - begin - sock.accept # OpenSSL::SSL::SSLSocket#accept - rescue Errno::ECONNRESET, Errno::ECONNABORTED, - Errno::EPROTO, Errno::EINVAL - Thread.exit - end - end - end - call_callback(:AcceptCallback, sock) - block ? block.call(sock) : run(sock) - rescue Errno::ENOTCONN - @logger.debug "Errno::ENOTCONN raised" - rescue ServerError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - rescue Exception => ex - @logger.error ex - ensure - @tokens.push(nil) - Thread.current[:WEBrickSocket] = nil - if addr - @logger.debug "close: #{addr[3]}:#{addr[1]}" - else - @logger.debug "close: <address unknown>" - end - sock.close - end - } - end - - ## - # Calls the callback +callback_name+ from the configuration with +args+ - - def call_callback(callback_name, *args) - @config[callback_name]&.call(*args) - end - - def setup_shutdown_pipe - return @shutdown_pipe ||= IO.pipe - end - - def cleanup_shutdown_pipe(shutdown_pipe) - @shutdown_pipe = nil - shutdown_pipe&.each(&:close) - end - - def alarm_shutdown_pipe - _, pipe = @shutdown_pipe # another thread may modify @shutdown_pipe. - if pipe - if !pipe.closed? - begin - yield pipe - rescue IOError # closed by another thread. - end - end - end - end - - def cleanup_listener - @listeners.each{|s| - if @logger.debug? - addr = s.addr - @logger.debug("close TCPSocket(#{addr[2]}, #{addr[1]})") - end - begin - s.shutdown - rescue Errno::ENOTCONN - # when `Errno::ENOTCONN: Socket is not connected' on some platforms, - # call #close instead of #shutdown. - # (ignore @config[:ShutdownSocketWithoutClose]) - s.close - else - unless @config[:ShutdownSocketWithoutClose] - s.close - end - end - } - @listeners.clear - end - end # end of GenericServer -end diff --git a/tool/lib/webrick/ssl.rb b/tool/lib/webrick/ssl.rb deleted file mode 100644 index e448095a12..0000000000 --- a/tool/lib/webrick/ssl.rb +++ /dev/null @@ -1,215 +0,0 @@ -# frozen_string_literal: false -# -# ssl.rb -- SSL/TLS enhancement for GenericServer -# -# Copyright (c) 2003 GOTOU Yuuzou All rights reserved. -# -# $Id$ - -require 'webrick' -require 'openssl' - -module WEBrick - module Config - svrsoft = General[:ServerSoftware] - osslv = ::OpenSSL::OPENSSL_VERSION.split[1] - - ## - # Default SSL server configuration. - # - # WEBrick can automatically create a self-signed certificate if - # <code>:SSLCertName</code> is set. For more information on the various - # SSL options see OpenSSL::SSL::SSLContext. - # - # :ServerSoftware :: - # The server software name used in the Server: header. - # :SSLEnable :: false, - # Enable SSL for this server. Defaults to false. - # :SSLCertificate :: - # The SSL certificate for the server. - # :SSLPrivateKey :: - # The SSL private key for the server certificate. - # :SSLClientCA :: nil, - # Array of certificates that will be sent to the client. - # :SSLExtraChainCert :: nil, - # Array of certificates that will be added to the certificate chain - # :SSLCACertificateFile :: nil, - # Path to a CA certificate file - # :SSLCACertificatePath :: nil, - # Path to a directory containing CA certificates - # :SSLCertificateStore :: nil, - # OpenSSL::X509::Store used for certificate validation of the client - # :SSLTmpDhCallback :: nil, - # Callback invoked when DH parameters are required. - # :SSLVerifyClient :: - # Sets whether the client is verified. This defaults to VERIFY_NONE - # which is typical for an HTTPS server. - # :SSLVerifyDepth :: - # Number of CA certificates to walk when verifying a certificate chain - # :SSLVerifyCallback :: - # Custom certificate verification callback - # :SSLServerNameCallback:: - # Custom servername indication callback - # :SSLTimeout :: - # Maximum session lifetime - # :SSLOptions :: - # Various SSL options - # :SSLCiphers :: - # Ciphers to be used - # :SSLStartImmediately :: - # Immediately start SSL upon connection? Defaults to true - # :SSLCertName :: - # SSL certificate name. Must be set to enable automatic certificate - # creation. - # :SSLCertComment :: - # Comment used during automatic certificate creation. - - SSL = { - :ServerSoftware => "#{svrsoft} OpenSSL/#{osslv}", - :SSLEnable => false, - :SSLCertificate => nil, - :SSLPrivateKey => nil, - :SSLClientCA => nil, - :SSLExtraChainCert => nil, - :SSLCACertificateFile => nil, - :SSLCACertificatePath => nil, - :SSLCertificateStore => nil, - :SSLTmpDhCallback => nil, - :SSLVerifyClient => ::OpenSSL::SSL::VERIFY_NONE, - :SSLVerifyDepth => nil, - :SSLVerifyCallback => nil, # custom verification - :SSLTimeout => nil, - :SSLOptions => nil, - :SSLCiphers => nil, - :SSLStartImmediately => true, - # Must specify if you use auto generated certificate. - :SSLCertName => nil, - :SSLCertComment => "Generated by Ruby/OpenSSL" - } - General.update(SSL) - end - - module Utils - ## - # Creates a self-signed certificate with the given number of +bits+, - # the issuer +cn+ and a +comment+ to be stored in the certificate. - - def create_self_signed_cert(bits, cn, comment) - rsa = OpenSSL::PKey::RSA.new(bits){|p, n| - case p - when 0; $stderr.putc "." # BN_generate_prime - when 1; $stderr.putc "+" # BN_generate_prime - when 2; $stderr.putc "*" # searching good prime, - # n = #of try, - # but also data from BN_generate_prime - when 3; $stderr.putc "\n" # found good prime, n==0 - p, n==1 - q, - # but also data from BN_generate_prime - else; $stderr.putc "*" # BN_generate_prime - end - } - cert = OpenSSL::X509::Certificate.new - cert.version = 2 - cert.serial = 1 - name = (cn.kind_of? String) ? OpenSSL::X509::Name.parse(cn) - : OpenSSL::X509::Name.new(cn) - cert.subject = name - cert.issuer = name - cert.not_before = Time.now - cert.not_after = Time.now + (365*24*60*60) - cert.public_key = rsa.public_key - - ef = OpenSSL::X509::ExtensionFactory.new(nil,cert) - ef.issuer_certificate = cert - cert.extensions = [ - ef.create_extension("basicConstraints","CA:FALSE"), - ef.create_extension("keyUsage", "keyEncipherment, digitalSignature, keyAgreement, dataEncipherment"), - ef.create_extension("subjectKeyIdentifier", "hash"), - ef.create_extension("extendedKeyUsage", "serverAuth"), - ef.create_extension("nsComment", comment), - ] - aki = ef.create_extension("authorityKeyIdentifier", - "keyid:always,issuer:always") - cert.add_extension(aki) - cert.sign(rsa, "SHA256") - - return [ cert, rsa ] - end - module_function :create_self_signed_cert - end - - ## - #-- - # Updates WEBrick::GenericServer with SSL functionality - - class GenericServer - - ## - # SSL context for the server when run in SSL mode - - def ssl_context # :nodoc: - @ssl_context ||= begin - if @config[:SSLEnable] - ssl_context = setup_ssl_context(@config) - @logger.info("\n" + @config[:SSLCertificate].to_text) - ssl_context - end - end - end - - undef listen - - ## - # Updates +listen+ to enable SSL when the SSL configuration is active. - - def listen(address, port) # :nodoc: - listeners = Utils::create_listeners(address, port) - if @config[:SSLEnable] - listeners.collect!{|svr| - ssvr = ::OpenSSL::SSL::SSLServer.new(svr, ssl_context) - ssvr.start_immediately = @config[:SSLStartImmediately] - ssvr - } - end - @listeners += listeners - setup_shutdown_pipe - end - - ## - # Sets up an SSL context for +config+ - - def setup_ssl_context(config) # :nodoc: - unless config[:SSLCertificate] - cn = config[:SSLCertName] - comment = config[:SSLCertComment] - cert, key = Utils::create_self_signed_cert(2048, cn, comment) - config[:SSLCertificate] = cert - config[:SSLPrivateKey] = key - end - ctx = OpenSSL::SSL::SSLContext.new - ctx.key = config[:SSLPrivateKey] - ctx.cert = config[:SSLCertificate] - ctx.client_ca = config[:SSLClientCA] - ctx.extra_chain_cert = config[:SSLExtraChainCert] - ctx.ca_file = config[:SSLCACertificateFile] - ctx.ca_path = config[:SSLCACertificatePath] - ctx.cert_store = config[:SSLCertificateStore] - ctx.tmp_dh_callback = config[:SSLTmpDhCallback] - ctx.verify_mode = config[:SSLVerifyClient] - ctx.verify_depth = config[:SSLVerifyDepth] - ctx.verify_callback = config[:SSLVerifyCallback] - ctx.servername_cb = config[:SSLServerNameCallback] || proc { |args| ssl_servername_callback(*args) } - ctx.timeout = config[:SSLTimeout] - ctx.options = config[:SSLOptions] - ctx.ciphers = config[:SSLCiphers] - ctx - end - - ## - # ServerNameIndication callback - - def ssl_servername_callback(sslsocket, hostname = nil) - # default - end - - end -end diff --git a/tool/lib/webrick/utils.rb b/tool/lib/webrick/utils.rb deleted file mode 100644 index a96d6f03fd..0000000000 --- a/tool/lib/webrick/utils.rb +++ /dev/null @@ -1,265 +0,0 @@ -# frozen_string_literal: false -# -# utils.rb -- Miscellaneous utilities -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: utils.rb,v 1.10 2003/02/16 22:22:54 gotoyuzo Exp $ - -require 'socket' -require 'io/nonblock' -require 'etc' - -module WEBrick - module Utils - ## - # Sets IO operations on +io+ to be non-blocking - def set_non_blocking(io) - io.nonblock = true if io.respond_to?(:nonblock=) - end - module_function :set_non_blocking - - ## - # Sets the close on exec flag for +io+ - def set_close_on_exec(io) - io.close_on_exec = true if io.respond_to?(:close_on_exec=) - end - module_function :set_close_on_exec - - ## - # Changes the process's uid and gid to the ones of +user+ - def su(user) - if pw = Etc.getpwnam(user) - Process::initgroups(user, pw.gid) - Process::Sys::setgid(pw.gid) - Process::Sys::setuid(pw.uid) - else - warn("WEBrick::Utils::su doesn't work on this platform", uplevel: 1) - end - end - module_function :su - - ## - # The server hostname - def getservername - Socket::gethostname - end - module_function :getservername - - ## - # Creates TCP server sockets bound to +address+:+port+ and returns them. - # - # It will create IPV4 and IPV6 sockets on all interfaces. - def create_listeners(address, port) - unless port - raise ArgumentError, "must specify port" - end - sockets = Socket.tcp_server_sockets(address, port) - sockets = sockets.map {|s| - s.autoclose = false - ts = TCPServer.for_fd(s.fileno) - s.close - ts - } - return sockets - end - module_function :create_listeners - - ## - # Characters used to generate random strings - RAND_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + - "0123456789" + - "abcdefghijklmnopqrstuvwxyz" - - ## - # Generates a random string of length +len+ - def random_string(len) - rand_max = RAND_CHARS.bytesize - ret = "" - len.times{ ret << RAND_CHARS[rand(rand_max)] } - ret - end - module_function :random_string - - ########### - - require "timeout" - require "singleton" - - ## - # Class used to manage timeout handlers across multiple threads. - # - # Timeout handlers should be managed by using the class methods which are - # synchronized. - # - # id = TimeoutHandler.register(10, Timeout::Error) - # begin - # sleep 20 - # puts 'foo' - # ensure - # TimeoutHandler.cancel(id) - # end - # - # will raise Timeout::Error - # - # id = TimeoutHandler.register(10, Timeout::Error) - # begin - # sleep 5 - # puts 'foo' - # ensure - # TimeoutHandler.cancel(id) - # end - # - # will print 'foo' - # - class TimeoutHandler - include Singleton - - ## - # Mutex used to synchronize access across threads - TimeoutMutex = Thread::Mutex.new # :nodoc: - - ## - # Registers a new timeout handler - # - # +time+:: Timeout in seconds - # +exception+:: Exception to raise when timeout elapsed - def TimeoutHandler.register(seconds, exception) - at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds - instance.register(Thread.current, at, exception) - end - - ## - # Cancels the timeout handler +id+ - def TimeoutHandler.cancel(id) - instance.cancel(Thread.current, id) - end - - def self.terminate - instance.terminate - end - - ## - # Creates a new TimeoutHandler. You should use ::register and ::cancel - # instead of creating the timeout handler directly. - def initialize - TimeoutMutex.synchronize{ - @timeout_info = Hash.new - } - @queue = Thread::Queue.new - @watcher = nil - end - - # :nodoc: - private \ - def watch - to_interrupt = [] - while true - now = Process.clock_gettime(Process::CLOCK_MONOTONIC) - wakeup = nil - to_interrupt.clear - TimeoutMutex.synchronize{ - @timeout_info.each {|thread, ary| - next unless ary - ary.each{|info| - time, exception = *info - if time < now - to_interrupt.push [thread, info.object_id, exception] - elsif !wakeup || time < wakeup - wakeup = time - end - } - } - } - to_interrupt.each {|arg| interrupt(*arg)} - if !wakeup - @queue.pop - elsif (wakeup -= now) > 0 - begin - (th = Thread.start {@queue.pop}).join(wakeup) - ensure - th&.kill&.join - end - end - @queue.clear - end - end - - # :nodoc: - private \ - def watcher - (w = @watcher)&.alive? and return w # usual case - TimeoutMutex.synchronize{ - (w = @watcher)&.alive? and next w # pathological check - @watcher = Thread.start(&method(:watch)) - } - end - - ## - # Interrupts the timeout handler +id+ and raises +exception+ - def interrupt(thread, id, exception) - if cancel(thread, id) && thread.alive? - thread.raise(exception, "execution timeout") - end - end - - ## - # Registers a new timeout handler - # - # +time+:: Timeout in seconds - # +exception+:: Exception to raise when timeout elapsed - def register(thread, time, exception) - info = nil - TimeoutMutex.synchronize{ - (@timeout_info[thread] ||= []) << (info = [time, exception]) - } - @queue.push nil - watcher - return info.object_id - end - - ## - # Cancels the timeout handler +id+ - def cancel(thread, id) - TimeoutMutex.synchronize{ - if ary = @timeout_info[thread] - ary.delete_if{|info| info.object_id == id } - if ary.empty? - @timeout_info.delete(thread) - end - return true - end - return false - } - end - - ## - def terminate - TimeoutMutex.synchronize{ - @timeout_info.clear - @watcher&.kill&.join - } - end - end - - ## - # Executes the passed block and raises +exception+ if execution takes more - # than +seconds+. - # - # If +seconds+ is zero or nil, simply executes the block - def timeout(seconds, exception=Timeout::Error) - return yield if seconds.nil? or seconds.zero? - # raise ThreadError, "timeout within critical session" if Thread.critical - id = TimeoutHandler.register(seconds, exception) - begin - yield(seconds) - ensure - TimeoutHandler.cancel(id) - end - end - module_function :timeout - end -end diff --git a/tool/lib/webrick/version.rb b/tool/lib/webrick/version.rb deleted file mode 100644 index b62988bdbb..0000000000 --- a/tool/lib/webrick/version.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: false -#-- -# version.rb -- version and release date -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: version.rb,v 1.74 2003/07/22 19:20:43 gotoyuzo Exp $ - -module WEBrick - - ## - # The WEBrick version - - VERSION = "1.7.0" -end diff --git a/tool/ln_sr.rb b/tool/ln_sr.rb index 2aa8391e17..e1b5b6f76b 100755 --- a/tool/ln_sr.rb +++ b/tool/ln_sr.rb @@ -96,7 +96,7 @@ unless respond_to?(:ln_sr) while c = comp.shift if c == ".." and clean.last != ".." and !(fu_have_symlink? && File.symlink?(path)) clean.pop - path.chomp!(%r((?<=\A|/)[^/]+/\z), "") + path.sub!(%r((?<=\A|/)[^/]+/\z), "") else clean << c path << c << "/" diff --git a/tool/lrama/LEGAL.md b/tool/lrama/LEGAL.md new file mode 100644 index 0000000000..a3ef848514 --- /dev/null +++ b/tool/lrama/LEGAL.md @@ -0,0 +1,12 @@ +# LEGAL NOTICE INFORMATION + +All the files in this distribution are covered under the MIT License except some files +mentioned below. + +## GNU General Public License version 3 + +These files are licensed under the GNU General Public License version 3 or later. See these files for more information. + +* template/bison/_yacc.h +* template/bison/yacc.c +* template/bison/yacc.h diff --git a/tool/lrama/MIT b/tool/lrama/MIT new file mode 100644 index 0000000000..b23d5210d5 --- /dev/null +++ b/tool/lrama/MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 Yuichiro Kaneko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md new file mode 100644 index 0000000000..f71118a913 --- /dev/null +++ b/tool/lrama/NEWS.md @@ -0,0 +1,1032 @@ +# NEWS for Lrama + +## Lrama 0.7.1 (2025-12-24) + +### Optimize IELR + +Optimized performance to a level that allows for IELR testing in practical applications. + +https://github.com/ruby/lrama/pull/595 +https://github.com/ruby/lrama/pull/605 +https://github.com/ruby/lrama/pull/685 +https://github.com/ruby/lrama/pull/700 + +### Introduce counterexamples timeout + +Counterexample searches can sometimes take a long time, so we've added a timeout to abort the process after a set period. The current limits are: + +* 10 seconds per case +* 120 seconds total (cumulative) + +Please note that these are hard-coded and cannot be modified by the user in the current version. + +https://github.com/ruby/lrama/pull/623 + +### Optimize Counterexamples + +Optimized counterexample search performance. + +https://github.com/ruby/lrama/pull/607 +https://github.com/ruby/lrama/pull/610 +https://github.com/ruby/lrama/pull/614 +https://github.com/ruby/lrama/pull/622 +https://github.com/ruby/lrama/pull/627 +https://github.com/ruby/lrama/pull/629 +https://github.com/ruby/lrama/pull/659 + +### Support parameterized rule's arguments include inline + +Allow to use %inline directive with Parameterized rules arguments. When an inline rule is used as an argument to a Parameterized rule, it expands inline at the point of use. + +```yacc +%rule %inline op : '+' + | '-' + ; +%% +operation : op? + ; +``` + +This expands to: + +```yacc +operation : /* empty */ + | '+' + | '-' + ; +``` + +https://github.com/ruby/lrama/pull/637 + +### Render conflicts of each state on output file + +Added token information for conflicts in the output file. +These information are useful when a state has many actions. + +``` +State 1 + + 4 class: keyword_class • tSTRING "end" + 5 $@1: ε • [tSTRING] + 7 class: keyword_class • $@1 tSTRING '!' "end" $@2 + 8 $@3: ε • [tSTRING] + 10 class: keyword_class • $@3 tSTRING '?' "end" $@4 + + Conflict on tSTRING. shift/reduce($@1) + Conflict on tSTRING. shift/reduce($@3) + Conflict on tSTRING. reduce($@1)/reduce($@3) + + tSTRING shift, and go to state 6 + + tSTRING reduce using rule 5 ($@1) + tSTRING reduce using rule 8 ($@3) + + $@1 go to state 7 + $@3 go to state 8 +``` + +https://github.com/ruby/lrama/pull/541 + +### Render the origin of conflicted tokens on output file + +For example, for the grammar file like below: + +``` +%% + +program: expr + ; + +expr: expr '+' expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where `"plus"` (`'+'`) look ahead tokens come from: + +``` +State 6 + + 2 expr: expr • "plus" expr + 2 | expr "plus" expr • ["end of file", "plus"] + + Conflict on "plus". shift/reduce(expr) + "plus" comes from state 0 goto by expr + "plus" comes from state 5 goto by expr +``` + +state 0 and state 5 look like below: + +``` +State 0 + + 0 $accept: • program "end of file" + 1 program: • expr + 2 expr: • expr "plus" expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + program go to state 2 + expr go to state 3 + +State 5 + + 2 expr: • expr "plus" expr + 2 | expr "plus" • expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + expr go to state 6 +``` + +https://github.com/ruby/lrama/pull/726 + +### Render precedences usage information on output file + +For example, for the grammar file like below: + +``` +%left tPLUS +%right tUPLUS + +%% + +program: expr ; + +expr: tUPLUS expr + | expr tPLUS expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where these precedences are used to resolve conflicts: + +``` +Precedences + precedence on "unary+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + precedence on "+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + state 8. Conflict between reduce by "expr -> expr tPLUS expr" and shift "+" resolved as reduce (%left "+"). +``` + +https://github.com/ruby/lrama/pull/741 + +### Add support for reporting Rule Usage Frequency + +Support to report rule usage frequency statistics for analyzing grammar characteristics. +Run `exe/lrama --report=rules` to show how frequently each terminal and non-terminal symbol is used in the grammar rules. + +```console +$ exe/lrama --report=rules sample/calc.y +Rule Usage Frequency + 0 tSTRING (4 times) + 1 keyword_class (3 times) + 2 keyword_end (3 times) + 3 '+' (2 times) + 4 string (2 times) + 5 string_1 (2 times) + 6 '!' (1 times) + 7 '-' (1 times) + 8 '?' (1 times) + 9 EOI (1 times) + 10 class (1 times) + 11 program (1 times) + 12 string_2 (1 times) + 13 strings_1 (1 times) + 14 strings_2 (1 times) + 15 tNUMBER (1 times) +``` + +This feature provides insights into the language characteristics by showing: +- Which symbols are most frequently used in the grammar +- The distribution of terminal and non-terminal usage +- Potential areas for grammar optimization or refactoring + +The frequency statistics help developers understand the grammar structure and can be useful for: +- Grammar complexity analysis +- Performance optimization hints +- Language design decisions +- Documentation and educational purposes + +https://github.com/ruby/lrama/pull/677 + +### Render Split States information on output file + +For example, for the grammar file like below: + +``` +%token a +%token b +%token c +%define lr.type ielr + +%precedence tLOWEST +%precedence a +%precedence tHIGHEST + +%% + +S: a A B a + | b A B b + ; + +A: a C D E + ; + +B: c + | // empty + ; + +C: D + ; + +D: a + ; + +E: a + | %prec tHIGHEST // empty + ; + +%% +``` + +Lrama generates output file which describes where which new states are created when IELR is enabled: + +``` +Split States + + State 19 is split from state 4 + State 20 is split from state 9 + State 21 is split from state 14 +``` + +https://github.com/ruby/lrama/pull/624 + +### Add ioption support to the Standard library + +Support `ioption` (inline option) rule, which is expanded inline without creating intermediate rules. + +Unlike the regular `option` rule that generates a separate rule, `ioption` directly expands at the point of use: + +```yacc +program: ioption(number) expr + +// Expanded inline to: + +program: expr + | number expr +``` + +This differs from the regular `option` which would generate: + +```yacc +program: option(number) expr + +// Expanded to: + +program: option_number expr +option_number: %empty + | number +``` + +The `ioption` rule provides more compact grammar generation by avoiding intermediate rule creation, which can be beneficial for reducing the parser's rule count and potentially improving performance. + +This feature is inspired by Menhir's standard library and maintains compatibility with [Menhir's `ioption` behavior](https://github.com/let-def/menhir/blob/e8ba7bef219acd355798072c42abbd11335ecf09/src/standard.mly#L33-L41). + +https://github.com/ruby/lrama/pull/666 + +### Syntax Diagrams + +Lrama provides an API for generating HTML syntax diagrams. These visual diagrams are highly useful as grammar development tools and can also serve as a form of automatic self-documentation. + + + +If you use syntax diagrams, you add `--diagram` option. + +```console +$ exe/lrama --diagram sample.y +``` + +https://github.com/ruby/lrama/pull/523 + +### Support `--profile` option + +You can profile parser generation process without modification for Lrama source code. +Currently `--profile=call-stack` and `--profile=memory` are supported. + +```console +$ exe/lrama --profile=call-stack sample/calc.y +``` + +Then "tmp/stackprof-cpu-myapp.dump" is generated. + +https://github.com/ruby/lrama/pull/525 + +### Add support Start-Symbol: `%start` + +https://github.com/ruby/lrama/pull/576 + +## Lrama 0.7.0 (2025-01-21) + +### [EXPERIMENTAL] Support the generation of the IELR(1) parser described in this paper + +Support the generation of the IELR(1) parser described in this paper. +https://www.sciencedirect.com/science/article/pii/S0167642309001191 + +If you use IELR(1) parser, you can write the following directive in your grammar file. + +```yacc +%define lr.type ielr +``` + +But, currently IELR(1) parser is experimental feature. If you find any bugs, please report it to us. Thank you. + +### Support `-t` option as same as `--debug` option + +Support to `-t` option as same as `--debug` option. +These options align with Bison behavior. So same as `--debug` option. + +### Trace only explicit rules + +Support to trace only explicit rules. +If you use `--trace=rules` option, it shows include mid-rule actions. If you want to show only explicit rules, you can use `--trace=only-explicit-rules` option. + +Example: + +```yacc +%{ +%} +%union { + int i; +} +%token <i> number +%type <i> program +%% +program : number { printf("%d", $1); } number { $$ = $1 + $3; } + ; +%% +``` + +Result of `--trace=rules`: + +```console +$ exe/lrama --trace=rules sample.y +Grammar rules: +$accept -> program YYEOF +$@1 -> ε +program -> number $@1 number +``` + +Result of `--trace=only-explicit-rules`: + +```console +$ exe/lrama --trace=explicit-rules sample.y +Grammar rules: +$accept -> program YYEOF +program -> number number +``` + +## Lrama 0.6.11 (2024-12-23) + +### Add support for %type declarations using %nterm in Nonterminal Symbols + +Allow to use `%nterm` in Nonterminal Symbols for `%type` declarations. + +```yacc +%nterm <type> nonterminal… +``` + +This directive is also supported for compatibility with Bison, and only non-terminal symbols are allowed. In other words, definitions like the following will result in an error: + +```yacc +%{ +// Prologue +%} + +%token EOI 0 "EOI" +%nterm EOI + +%% + +program: /* empty */ + ; +``` + +It show an error message like the following: + +```command +❯ exe/lrama nterm.y +nterm.y:6:7: symbol EOI redeclared as a nonterminal +%nterm EOI + ^^^ +``` + +## Lrama 0.6.10 (2024-09-11) + +### Aliased Named References for actions of RHS in Parameterizing rules + +Allow to use aliased named references for actions of RHS in Parameterizing rules. + +```yacc +%rule sum(X, Y): X[summand] '+' Y[addend] { $$ = $summand + $addend } + ; +``` + +https://github.com/ruby/lrama/pull/410 + + +### Named References for actions of RHS in Parameterizing rules caller side + +Allow to use named references for actions of RHS in Parameterizing rules caller side. + +```yacc +opt_nl: '\n'?[nl] <str> { $$ = $nl; } + ; +``` + +https://github.com/ruby/lrama/pull/414 + +### Widen the definable position of Parameterizing rules + +Allow to define Parameterizing rules in the middle of the grammar. + +```yacc +%rule defined_option(X): /* empty */ + | X + ; + +%% + +program : defined_option(number) <i> + | defined_list(number) <i> + ; + +%rule defined_list(X): /* empty */ /* <--- here */ + | defined_list(X) number + ; +``` + +https://github.com/ruby/lrama/pull/420 + +### Report unused terminal symbols + +Support to report unused terminal symbols. +Run `exe/lrama --report=terms` to show unused terminal symbols. + +```console +$ exe/lrama --report=terms sample/calc.y + 11 Unused Terms + 0 YYerror + 1 YYUNDEF + 2 '\\\\' + 3 '\\13' + 4 keyword_class2 + 5 tNUMBER + 6 tPLUS + 7 tMINUS + 8 tEQ + 9 tEQEQ + 10 '>' +``` +https://github.com/ruby/lrama/pull/439 + +### Report unused rules + +Support to report unused rules. +Run `exe/lrama --report=rules` to show unused rules. + +```console +$ exe/lrama --report=rules sample/calc.y + 3 Unused Rules + 0 unused_option + 1 unused_list + 2 unused_nonempty_list +``` + +https://github.com/ruby/lrama/pull/441 + +### Ensure compatibility with Bison for `%locations` directive + +Support `%locations` directive to ensure compatibility with Bison. +Change to `%locations` directive not set by default. + +https://github.com/ruby/lrama/pull/446 + +### Diagnostics report for parameterized rules redefine + +Support to warning redefined parameterized rules. +Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterized rules. + +```console +$ exe/lrama -W sample/calc.y +parameterized rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) +``` + +https://github.com/ruby/lrama/pull/448 + +### Support `-v` and `--verbose` option + +Support to `-v` and `--verbose` option. +These options align with Bison behavior. So same as '--report=state' option. + +https://github.com/ruby/lrama/pull/457 + +## Lrama 0.6.9 (2024-05-02) + +### Callee side tag specification of Parameterizing rules + +Allow to specify tag on callee side of Parameterizing rules. + +```yacc +%union { + int i; +} + +%rule with_tag(X) <i>: X { $$ = $1; } + ; +``` + +### Named References for actions of RHS in Parameterizing rules + +Allow to use named references for actions of RHS in Parameterizing rules. + +```yacc +%rule option(number): /* empty */ + | number { $$ = $number; } + ; +``` + +## Lrama 0.6.8 (2024-04-29) + +### Nested Parameterizing rules with tag + +Allow to nested Parameterizing rules with tag. + +```yacc +%union { + int i; +} + +%rule nested_nested_option(X): /* empty */ + | X + ; + +%rule nested_option(X): /* empty */ + | nested_nested_option(X) <i> + ; + +%rule option(Y): /* empty */ + | nested_option(Y) <i> + ; +``` + +## Lrama 0.6.7 (2024-04-28) + +### RHS of user defined Parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. + +User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined Parameterizing rules. + +``` +%rule with_word_seps(X): /* empty */ + | X ' '+ + ; +``` + +## Lrama 0.6.6 (2024-04-27) + +### Trace actions + +Support trace actions for debugging. +Run `exe/lrama --trace=actions` to show grammar rules with actions. + +```console +$ exe/lrama --trace=actions sample/calc.y +Grammar rules with actions: +$accept -> list, YYEOF {} +list -> ε {} +list -> list, LF {} +list -> list, expr, LF { printf("=> %d\n", $2); } +expr -> NUM {} +expr -> expr, '+', expr { $$ = $1 + $3; } +expr -> expr, '-', expr { $$ = $1 - $3; } +expr -> expr, '*', expr { $$ = $1 * $3; } +expr -> expr, '/', expr { $$ = $1 / $3; } +expr -> '(', expr, ')' { $$ = $2; } +``` + +### Inlining + +Support inlining for rules. +The `%inline` directive causes all references to symbols to be replaced with its definition. + +```yacc +%rule %inline op: PLUS { + } + | TIMES { * } + ; + +%% + +expr : number { $$ = $1; } + | expr op expr { $$ = $1 $2 $3; } + ; +``` + +as same as + +```yacc +expr : number { $$ = $1; } + | expr '+' expr { $$ = $1 + $3; } + | expr '*' expr { $$ = $1 * $3; } + ; +``` + +## Lrama 0.6.5 (2024-03-25) + +### Typed Midrule Actions + +User can specify the type of mid-rule action by tag (`<bar>`) instead of specifying it with in an action. + +```yacc +primary: k_case expr_value terms? + { + $<val>$ = p->case_labels; + p->case_labels = Qnil; + } + case_body + k_end + { + ... + } +``` + +can be written as + +```yacc +primary: k_case expr_value terms? + { + $$ = p->case_labels; + p->case_labels = Qnil; + }<val> + case_body + k_end + { + ... + } +``` + +`%destructor` for midrule action is invoked only when tag is specified by Typed Midrule Actions. + +Difference from Bison's Typed Midrule Actions is that tag is postposed in Lrama however it's preposed in Bison. + +Bison supports this feature from 3.1. + +## Lrama 0.6.4 (2024-03-22) + +### Parameterizing rules (preceded, terminated, delimited) + +Support `preceded`, `terminated` and `delimited` rules. + +```text +program: preceded(opening, X) + +// Expanded to + +program: preceded_opening_X +preceded_opening_X: opening X +``` + +``` +program: terminated(X, closing) + +// Expanded to + +program: terminated_X_closing +terminated_X_closing: X closing +``` + +``` +program: delimited(opening, X, closing) + +// Expanded to + +program: delimited_opening_X_closing +delimited_opening_X_closing: opening X closing +``` + +https://github.com/ruby/lrama/pull/382 + +### Support `%destructor` declaration + +User can set codes for freeing semantic value resources by using `%destructor`. +In general, these resources are freed by actions or after parsing. +However, if syntax error happens in parsing, these codes may not be executed. +Codes associated to `%destructor` are executed when semantic value is popped from the stack by an error. + +```yacc +%token <val1> NUM +%type <val2> expr2 +%type <val3> expr + +%destructor { + printf("destructor for val1: %d\n", $$); +} <val1> // printer for TAG + +%destructor { + printf("destructor for val2: %d\n", $$); +} <val2> + +%destructor { + printf("destructor for expr: %d\n", $$); +} expr // printer for symbol +``` + +Bison supports this feature from 1.75b. + +https://github.com/ruby/lrama/pull/385 + +## Lrama 0.6.3 (2024-02-15) + +### Bring Your Own Stack + +Provide functionalities for Bring Your Own Stack. + +Ruby’s Ripper library requires their own semantic value stack to manage Ruby Objects returned by user defined callback method. Currently Ripper uses semantic value stack (`yyvsa`) which is used by parser to manage Node. This hack introduces some limitation on Ripper. For example, Ripper can not execute semantic analysis depending on Node structure. + +Lrama introduces two features to support another semantic value stack by parser generator users. + +1. Callback entry points + +User can emulate semantic value stack by these callbacks. +Lrama provides these five callbacks. Registered functions are called when each event happens. For example %after-shift function is called when shift happens on original semantic value stack. + +* `%after-shift` function_name +* `%before-reduce` function_name +* `%after-reduce` function_name +* `%after-shift-error-token` function_name +* `%after-pop-stack` function_name + +2. `$:n` variable to access index of each grammar symbols + +User also needs to access semantic value of their stack in grammar action. `$:n` provides the way to access to it. `$:n` is translated to the minus index from the top of the stack. +For example + +```yacc +primary: k_if expr_value then compstmt if_tail k_end + { + /*% ripper: if!($:2, $:4, $:5) %*/ + /* $:2 = -5, $:4 = -3, $:5 = -2. */ + } +``` + +https://github.com/ruby/lrama/pull/367 + +## Lrama 0.6.2 (2024-01-27) + +### %no-stdlib directive + +If `%no-stdlib` directive is set, Lrama doesn't load Lrama standard library for +parameterized rules, stdlib.y. + +https://github.com/ruby/lrama/pull/344 + +## Lrama 0.6.1 (2024-01-13) + +### Nested Parameterizing rules + +Allow to pass an instantiated rule to other Parameterizing rules. + +```yacc +%rule constant(X) : X + ; + +%rule option(Y) : /* empty */ + | Y + ; + +%% + +program : option(constant(number)) // Nested rule + ; +%% +``` + +Allow to use nested Parameterizing rules when define Parameterizing rules. + +```yacc +%rule option(x) : /* empty */ + | X + ; + +%rule double(Y) : Y Y + ; + +%rule double_opt(A) : option(double(A)) // Nested rule + ; + +%% + +program : double_opt(number) + ; + +%% +``` + +https://github.com/ruby/lrama/pull/337 + +## Lrama 0.6.0 (2023-12-25) + +### User defined Parameterizing rules + +Allow to define Parameterizing rule by `%rule` directive. + +```yacc +%rule pair(X, Y): X Y { $$ = $1 + $2; } + ; + +%% + +program: stmt + ; + +stmt: pair(ODD, EVEN) <num> + | pair(EVEN, ODD) <num> + ; +``` + +https://github.com/ruby/lrama/pull/285 + +## Lrama 0.5.11 (2023-12-02) + +### Type specification of Parameterizing rules + +Allow to specify type of rules by specifying tag, `<i>` in below example. +Tag is post-modification style. + +```yacc +%union { + int i; +} + +%% + +program : option(number) <i> + | number_alias? <i> + ; +``` + +https://github.com/ruby/lrama/pull/272 + + +## Lrama 0.5.10 (2023-11-18) + +### Parameterizing rules (option, nonempty_list, list) + +Support function call style Parameterizing rules for `option`, `nonempty_list` and `list`. + +https://github.com/ruby/lrama/pull/197 + +### Parameterizing rules (separated_list) + +Support `separated_list` and `separated_nonempty_list` Parameterizing rules. + +```text +program: separated_list(',', number) + +// Expanded to + +program: separated_list_number +separated_list_number: ε +separated_list_number: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +``` +program: separated_nonempty_list(',', number) + +// Expanded to + +program: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +https://github.com/ruby/lrama/pull/204 + +## Lrama 0.5.9 (2023-11-05) + +### Parameterizing rules (suffix) + +Parameterizing rules are template of rules. +It's very common pattern to write "list" grammar rule like: + +```yacc +opt_args: /* none */ + | args + ; + +args: arg + | args arg +``` + +Lrama supports these suffixes: + +* `?`: option +* `+`: nonempty list +* `*`: list + +Idea of Parameterizing rules comes from Menhir LR(1) parser generator (https://gallium.inria.fr/~fpottier/menhir/manual.html#sec32). + +https://github.com/ruby/lrama/pull/181 + +## Lrama 0.5.7 (2023-10-23) + +### Racc parser + +Replace Lrama's parser from handwritten parser to LR parser generated by Racc. +Lrama uses `--embedded` option to generate LR parser because Racc is changed from default gem to bundled gem by Ruby 3.3 (https://github.com/ruby/lrama/pull/132). + +https://github.com/ruby/lrama/pull/62 + +## Lrama 0.5.4 (2023-08-17) + +### Runtime configuration for error recovery + +Make error recovery function configurable on runtime by two new macros. + +* `YYMAXREPAIR`: Expected to return max length of repair operations. `%parse-param` is passed to this function. +* `YYERROR_RECOVERY_ENABLED`: Expected to return bool value to determine error recovery is enabled or not. `%parse-param` is passed to this function. + +https://github.com/ruby/lrama/pull/74 + +## Lrama 0.5.3 (2023-08-05) + +### Error Recovery + +Support token insert base Error Recovery. +`-e` option is needed to generate parser with error recovery functions. + +https://github.com/ruby/lrama/pull/44 + +## Lrama 0.5.2 (2023-06-14) + +### Named References + +Instead of positional references like `$1` or `$$`, +named references allow to access to symbol by name. + +```yacc +primary: k_class cpath superclass bodystmt k_end + { + $primary = new_class($cpath, $bodystmt, $superclass); + } +``` + +Alias name can be declared. + +```yacc +expr[result]: expr[ex-left] '+' expr[ex.right] + { + $result = $[ex-left] + $[ex.right]; + } +``` + +Bison supports this feature from 2.5. + +### Add parse params to some macros and functions + +`%parse-param` are added to these macros and functions to remove ytab.sed hack from Ruby. + +* `YY_LOCATION_PRINT` +* `YY_SYMBOL_PRINT` +* `yy_stack_print` +* `YY_STACK_PRINT` +* `YY_REDUCE_PRINT` +* `yysyntax_error` + +https://github.com/ruby/lrama/pull/40 + +See also: https://github.com/ruby/ruby/pull/7807 + +## Lrama 0.5.0 (2023-05-17) + +### stdin mode + +When `-` is given as grammar file name, reads the grammar source from STDIN, and takes the next argument as the input file name. This mode helps pre-process a grammar source. + +https://github.com/ruby/lrama/pull/8 + +## Lrama 0.4.0 (2023-05-13) + +This is the first version migrated to Ruby. +This version generates "parse.c" compatible with Bison 3.8.2. diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama new file mode 100755 index 0000000000..710ac0cb96 --- /dev/null +++ b/tool/lrama/exe/lrama @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +$LOAD_PATH << File.join(__dir__, "../lib") +require "lrama" + +Lrama::Command.new(ARGV.dup).run diff --git a/tool/lrama/lib/lrama.rb b/tool/lrama/lib/lrama.rb new file mode 100644 index 0000000000..56ba0044d4 --- /dev/null +++ b/tool/lrama/lib/lrama.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative "lrama/bitmap" +require_relative "lrama/command" +require_relative "lrama/context" +require_relative "lrama/counterexamples" +require_relative "lrama/diagram" +require_relative "lrama/digraph" +require_relative "lrama/erb" +require_relative "lrama/grammar" +require_relative "lrama/lexer" +require_relative "lrama/logger" +require_relative "lrama/option_parser" +require_relative "lrama/options" +require_relative "lrama/output" +require_relative "lrama/parser" +require_relative "lrama/reporter" +require_relative "lrama/state" +require_relative "lrama/states" +require_relative "lrama/tracer" +require_relative "lrama/version" +require_relative "lrama/warnings" diff --git a/tool/lrama/lib/lrama/bitmap.rb b/tool/lrama/lib/lrama/bitmap.rb new file mode 100644 index 0000000000..88b255b012 --- /dev/null +++ b/tool/lrama/lib/lrama/bitmap.rb @@ -0,0 +1,47 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + module Bitmap + # @rbs! + # type bitmap = Integer + + # @rbs (Array[Integer] ary) -> bitmap + def self.from_array(ary) + bit = 0 + + ary.each do |int| + bit |= (1 << int) + end + + bit + end + + # @rbs (Integer int) -> bitmap + def self.from_integer(int) + 1 << int + end + + # @rbs (bitmap int) -> Array[Integer] + def self.to_array(int) + a = [] #: Array[Integer] + i = 0 + + len = int.bit_length + while i < len do + if int[i] == 1 + a << i + end + + i += 1 + end + + a + end + + # @rbs (bitmap int, Integer size) -> Array[bool] + def self.to_bool_array(int, size) + Array.new(size) { |i| int[i] == 1 } + end + end +end diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb new file mode 100644 index 0000000000..17aad1a1c1 --- /dev/null +++ b/tool/lrama/lib/lrama/command.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Lrama + class Command + LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__))) + STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y') + + def initialize(argv) + @logger = Lrama::Logger.new + @options = OptionParser.parse(argv) + @tracer = Tracer.new(STDERR, **@options.trace_opts) + @reporter = Reporter.new(**@options.report_opts) + @warnings = Warnings.new(@logger, @options.warnings) + rescue => e + abort format_error_message(e.message) + end + + def run + Lrama::Reporter::Profile::CallStack.report(@options.profile_opts[:call_stack]) do + Lrama::Reporter::Profile::Memory.report(@options.profile_opts[:memory]) do + execute_command_workflow + end + end + end + + private + + def execute_command_workflow + @tracer.enable_duration + text = read_input + grammar = build_grammar(text) + states, context = compute_status(grammar) + render_reports(states) if @options.report_file + @tracer.trace(grammar) + render_diagram(grammar) + render_output(context, grammar) + states.validate!(@logger) + @warnings.warn(grammar, states) + end + + def read_input + text = @options.y.read + @options.y.close unless @options.y == STDIN + text + end + + def build_grammar(text) + grammar = + Lrama::Parser.new(text, @options.grammar_file, @options.debug, @options.locations, @options.define).parse + merge_stdlib(grammar) + prepare_grammar(grammar) + grammar + rescue => e + raise e if @options.debug + abort format_error_message(e.message) + end + + def format_error_message(message) + return message unless Exception.to_tty? + + message.gsub(/.+/, "\e[1m\\&\e[m") + end + + def merge_stdlib(grammar) + return if grammar.no_stdlib + + stdlib_text = File.read(STDLIB_FILE_PATH) + stdlib_grammar = Lrama::Parser.new( + stdlib_text, + STDLIB_FILE_PATH, + @options.debug, + @options.locations, + @options.define, + ).parse + + grammar.prepend_parameterized_rules(stdlib_grammar.parameterized_rules) + end + + def prepare_grammar(grammar) + grammar.prepare + grammar.validate! + end + + def compute_status(grammar) + states = Lrama::States.new(grammar, @tracer) + states.compute + states.compute_ielr if grammar.ielr_defined? + [states, Lrama::Context.new(states)] + end + + def render_reports(states) + File.open(@options.report_file, "w+") do |f| + @reporter.report(f, states) + end + end + + def render_diagram(grammar) + return unless @options.diagram + + File.open(@options.diagram_file, "w+") do |f| + Lrama::Diagram.render(out: f, grammar: grammar) + end + end + + def render_output(context, grammar) + File.open(@options.outfile, "w+") do |f| + Lrama::Output.new( + out: f, + output_file_path: @options.outfile, + template_name: @options.skeleton, + grammar_file_path: @options.grammar_file, + header_file_path: @options.header_file, + context: context, + grammar: grammar, + error_recovery: @options.error_recovery, + ).render + end + end + end +end diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb new file mode 100644 index 0000000000..eb068c1b9e --- /dev/null +++ b/tool/lrama/lib/lrama/context.rb @@ -0,0 +1,497 @@ +# frozen_string_literal: true + +require_relative "tracer/duration" + +module Lrama + # This is passed to a template + class Context + include Tracer::Duration + + ErrorActionNumber = -Float::INFINITY + BaseMin = -Float::INFINITY + + # TODO: It might be better to pass `states` to Output directly? + attr_reader :states, :yylast, :yypact_ninf, :yytable_ninf, :yydefact, :yydefgoto + + def initialize(states) + @states = states + @yydefact = nil + @yydefgoto = nil + # Array of array + @_actions = [] + + compute_tables + end + + # enum yytokentype + def yytokentype + @states.terms.reject do |term| + 0 < term.token_id && term.token_id < 128 + end.map do |term| + [term.id.s_value, term.token_id, term.display_name] + end.unshift(["YYEMPTY", -2, nil]) + end + + # enum yysymbol_kind_t + def yysymbol_kind_t + @states.symbols.map do |sym| + [sym.enum_name, sym.number, sym.comment] + end.unshift(["YYSYMBOL_YYEMPTY", -2, nil]) + end + + # State number of final (accepted) state + def yyfinal + @states.states.find do |state| + state.items.find do |item| + item.lhs.accept_symbol? && item.end_of_rule? + end + end.id + end + + # Number of terms + def yyntokens + @states.terms.count + end + + # Number of nterms + def yynnts + @states.nterms.count + end + + # Number of rules + def yynrules + @states.rules.count + end + + # Number of states + def yynstates + @states.states.count + end + + # Last token number + def yymaxutok + @states.terms.map(&:token_id).max + end + + # YYTRANSLATE + # + # yytranslate is a mapping from token id to symbol number + def yytranslate + # 2 is YYSYMBOL_YYUNDEF + a = Array.new(yymaxutok, 2) + + @states.terms.each do |term| + a[term.token_id] = term.number + end + + return a + end + + def yytranslate_inverted + a = Array.new(@states.symbols.count, @states.undef_symbol.token_id) + + @states.terms.each do |term| + a[term.number] = term.token_id + end + + return a + end + + # Mapping from rule number to line number of the rule is defined. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyrline + a = [0] + + @states.rules.each do |rule| + a << rule.lineno + end + + return a + end + + # Mapping from symbol number to its name + def yytname + @states.symbols.sort_by(&:number).map do |sym| + sym.display_name + end + end + + def yypact + @base[0...yynstates] + end + + def yypgoto + @base[yynstates..-1] + end + + def yytable + @table + end + + def yycheck + @check + end + + def yystos + @states.states.map do |state| + state.accessing_symbol.number + end + end + + # Mapping from rule number to symbol number of LHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr1 + a = [0] + + @states.rules.each do |rule| + a << rule.lhs.number + end + + return a + end + + # Mapping from rule number to length of RHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr2 + a = [0] + + @states.rules.each do |rule| + a << rule.rhs.count + end + + return a + end + + private + + # Compute these + # + # See also: "src/tables.c" of Bison. + # + # * yydefact + # * yydefgoto + # * yypact and yypgoto + # * yytable + # * yycheck + # * yypact_ninf + # * yytable_ninf + def compute_tables + report_duration(:compute_yydefact) { compute_yydefact } + report_duration(:compute_yydefgoto) { compute_yydefgoto } + report_duration(:sort_actions) { sort_actions } + # debug_sorted_actions + report_duration(:compute_packed_table) { compute_packed_table } + end + + def vectors_count + @states.states.count + @states.nterms.count + end + + # In compressed table, rule 0 is appended as an error case + # and reduce is represented as minus number. + def rule_id_to_action_number(rule_id) + (rule_id + 1) * -1 + end + + # Symbol number is assigned to term first then nterm. + # This method calculates sequence_number for nterm. + def nterm_number_to_sequence_number(nterm_number) + nterm_number - @states.terms.count + end + + # Vector is states + nterms + def nterm_number_to_vector_number(nterm_number) + @states.states.count + (nterm_number - @states.terms.count) + end + + def compute_yydefact + # Default action (shift/reduce/error) for each state. + # Index is state id, value is `rule id + 1` of a default reduction. + @yydefact = Array.new(@states.states.count, 0) + + @states.states.each do |state| + # Action number means + # + # * number = 0, default action + # * number = -Float::INFINITY, error by %nonassoc + # * number > 0, shift then move to state "number" + # * number < 0, reduce by "-number" rule. Rule "number" is already added by 1. + actions = Array.new(@states.terms.count, 0) + + if state.reduces.map(&:selected_look_ahead).any? {|la| !la.empty? } + # Iterate reduces with reverse order so that first rule is used. + state.reduces.reverse_each do |reduce| + reduce.look_ahead.each do |term| + actions[term.number] = rule_id_to_action_number(reduce.rule.id) + end + end + end + + # Shift is selected when S/R conflict exists. + state.selected_term_transitions.each do |shift| + actions[shift.next_sym.number] = shift.to_state.id + end + + state.resolved_conflicts.select do |conflict| + conflict.which == :error + end.each do |conflict| + actions[conflict.symbol.number] = ErrorActionNumber + end + + # If default_reduction_rule, replace default_reduction_rule in + # actions with zero. + if state.default_reduction_rule + actions.map! do |e| + if e == rule_id_to_action_number(state.default_reduction_rule.id) + 0 + else + e + end + end + end + + # If no default_reduction_rule, default behavior is an + # error then replace ErrorActionNumber with zero. + unless state.default_reduction_rule + actions.map! do |e| + if e == ErrorActionNumber + 0 + else + e + end + end + end + + s = actions.each_with_index.map do |n, i| + [i, n] + end.reject do |i, n| + # Remove default_reduction_rule entries + n == 0 + end + + if s.count != 0 + # Entry of @_actions is an array of + # + # * State id + # * Array of tuple, [from, to] where from is term number and to is action. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [state.id, s, s.count, s.last[0] - s.first[0] + 1] + end + + @yydefact[state.id] = state.default_reduction_rule ? state.default_reduction_rule.id + 1 : 0 + end + end + + def compute_yydefgoto + # Default GOTO (nterm transition) for each nterm. + # Index is sequence number of nterm, value is state id + # of a default nterm transition destination. + @yydefgoto = Array.new(@states.nterms.count, 0) + # Mapping from nterm to next_states + nterm_to_to_states = {} + + @states.states.each do |state| + state.nterm_transitions.each do |goto| + key = goto.next_sym + nterm_to_to_states[key] ||= [] + nterm_to_to_states[key] << [state, goto.to_state] # [from_state, to_state] + end + end + + @states.nterms.each do |nterm| + if (states = nterm_to_to_states[nterm]) + default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first + default_goto = default_state.id + not_default_gotos = [] + states.each do |from_state, to_state| + next if to_state.id == default_goto + not_default_gotos << [from_state.id, to_state.id] + end + else + default_goto = 0 + not_default_gotos = [] + end + + k = nterm_number_to_sequence_number(nterm.number) + @yydefgoto[k] = default_goto + + if not_default_gotos.count != 0 + v = nterm_number_to_vector_number(nterm.number) + + # Entry of @_actions is an array of + # + # * Nterm number as vector number + # * Array of tuple, [from, to] where from is state number and to is state number. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [v, not_default_gotos, not_default_gotos.count, not_default_gotos.last[0] - not_default_gotos.first[0] + 1] + end + end + end + + def sort_actions + # This is not same with #sort_actions + # + # @sorted_actions = @_actions.sort_by do |_, _, count, width| + # [-width, -count] + # end + + @sorted_actions = [] + + @_actions.each do |action| + if @sorted_actions.empty? + @sorted_actions << action + next + end + + j = @sorted_actions.count - 1 + _state_id, _froms_and_tos, count, width = action + + while (j >= 0) do + case + when @sorted_actions[j][3] < width + j -= 1 + when @sorted_actions[j][3] == width && @sorted_actions[j][2] < count + j -= 1 + else + break + end + end + + @sorted_actions.insert(j + 1, action) + end + end + + def debug_sorted_actions + ary = Array.new + @sorted_actions.each do |state_id, froms_and_tos, count, width| + ary[state_id] = [state_id, froms_and_tos, count, width] + end + + print sprintf("table_print:\n\n") + + print sprintf("order [\n") + vectors_count.times do |i| + print sprintf("%d, ", @sorted_actions[i] ? @sorted_actions[i][0] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("width [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][3] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("tally [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][2] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + end + + def compute_packed_table + # yypact and yypgoto + @base = Array.new(vectors_count, BaseMin) + # yytable + @table = [] + # yycheck + @check = [] + # Key is froms_and_tos, value is index position + pushed = {} + used_res = {} + lowzero = 0 + high = 0 + + @sorted_actions.each do |state_id, froms_and_tos, _, _| + if (res = pushed[froms_and_tos]) + @base[state_id] = res + next + end + + res = lowzero - froms_and_tos.first[0] + + # Find the smallest `res` such that `@table[res + from]` is empty for all `from` in `froms_and_tos` + while true do + advanced = false + + while used_res[res] + res += 1 + advanced = true + end + + froms_and_tos.each do |from, to| + while @table[res + from] + res += 1 + advanced = true + end + end + + unless advanced + # no advance means that the current `res` satisfies the condition + break + end + end + + loc = 0 + + froms_and_tos.each do |from, to| + loc = res + from + + @table[loc] = to + @check[loc] = from + end + + while (@table[lowzero]) do + lowzero += 1 + end + + high = loc if high < loc + + @base[state_id] = res + pushed[froms_and_tos] = res + used_res[res] = true + end + + @yylast = high + + # replace_ninf + @yypact_ninf = (@base.reject {|i| i == BaseMin } + [0]).min - 1 + @base.map! do |i| + case i + when BaseMin + @yypact_ninf + else + i + end + end + + @yytable_ninf = (@table.compact.reject {|i| i == ErrorActionNumber } + [0]).min - 1 + @table.map! do |i| + case i + when nil + 0 + when ErrorActionNumber + @yytable_ninf + else + i + end + end + + @check.map! do |i| + case i + when nil + -1 + else + i + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb new file mode 100644 index 0000000000..60d830d048 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -0,0 +1,426 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "set" +require "timeout" + +require_relative "counterexamples/derivation" +require_relative "counterexamples/example" +require_relative "counterexamples/node" +require_relative "counterexamples/path" +require_relative "counterexamples/state_item" +require_relative "counterexamples/triple" + +module Lrama + # See: https://www.cs.cornell.edu/andru/papers/cupex/cupex.pdf + # 4. Constructing Nonunifying Counterexamples + class Counterexamples + PathSearchTimeLimit = 10 # 10 sec + CumulativeTimeLimit = 120 # 120 sec + + # @rbs! + # @states: States + # @iterate_count: Integer + # @total_duration: Float + # @exceed_cumulative_time_limit: bool + # @state_items: Hash[[State, State::Item], StateItem] + # @triples: Hash[Integer, Triple] + # @transitions: Hash[[StateItem, Grammar::Symbol], StateItem] + # @reverse_transitions: Hash[[StateItem, Grammar::Symbol], Set[StateItem]] + # @productions: Hash[StateItem, Set[StateItem]] + # @reverse_productions: Hash[[State, Grammar::Symbol], Set[StateItem]] # Grammar::Symbol is nterm + # @state_item_shift: Integer + + attr_reader :transitions #: Hash[[StateItem, Grammar::Symbol], StateItem] + attr_reader :productions #: Hash[StateItem, Set[StateItem]] + + # @rbs (States states) -> void + def initialize(states) + @states = states + @iterate_count = 0 + @total_duration = 0 + @exceed_cumulative_time_limit = false + @triples = {} + setup_state_items + setup_transitions + setup_productions + end + + # @rbs () -> "#<Counterexamples>" + def to_s + "#<Counterexamples>" + end + alias :inspect :to_s + + # @rbs (State conflict_state) -> Array[Example] + def compute(conflict_state) + conflict_state.conflicts.flat_map do |conflict| + # Check cumulative time limit for not each path search method call but each conflict + # to avoid one of example's path to be nil. + next if @exceed_cumulative_time_limit + + case conflict.type + when :shift_reduce + # @type var conflict: State::ShiftReduceConflict + shift_reduce_example(conflict_state, conflict) + when :reduce_reduce + # @type var conflict: State::ReduceReduceConflict + reduce_reduce_examples(conflict_state, conflict) + end + rescue Timeout::Error => e + STDERR.puts "Counterexamples calculation for state #{conflict_state.id} #{e.message} with #{@iterate_count} iteration" + increment_total_duration(PathSearchTimeLimit) + nil + end.compact + end + + private + + # @rbs (State state, State::Item item) -> StateItem + def get_state_item(state, item) + @state_items[[state, item]] + end + + # For optimization, create all StateItem in advance + # and use them by fetching an instance from `@state_items`. + # Do not create new StateItem instance in the shortest path search process + # to avoid miss hash lookup. + # + # @rbs () -> void + def setup_state_items + @state_items = {} + count = 0 + + @states.states.each do |state| + state.items.each do |item| + @state_items[[state, item]] = StateItem.new(count, state, item) + count += 1 + end + end + + @state_item_shift = Math.log(count, 2).ceil + end + + # @rbs () -> void + def setup_transitions + @transitions = {} + @reverse_transitions = {} + + @states.states.each do |src_state| + trans = {} #: Hash[Grammar::Symbol, State] + + src_state.transitions.each do |transition| + trans[transition.next_sym] = transition.to_state + end + + src_state.items.each do |src_item| + next if src_item.end_of_rule? + sym = src_item.next_sym + dest_state = trans[sym] + + dest_state.kernels.each do |dest_item| + next unless (src_item.rule == dest_item.rule) && (src_item.position + 1 == dest_item.position) + src_state_item = get_state_item(src_state, src_item) + dest_state_item = get_state_item(dest_state, dest_item) + + @transitions[[src_state_item, sym]] = dest_state_item + + # @type var key: [StateItem, Grammar::Symbol] + key = [dest_state_item, sym] + @reverse_transitions[key] ||= Set.new + @reverse_transitions[key] << src_state_item + end + end + end + end + + # @rbs () -> void + def setup_productions + @productions = {} + @reverse_productions = {} + + @states.states.each do |state| + # Grammar::Symbol is LHS + h = {} #: Hash[Grammar::Symbol, Set[StateItem]] + + state.closure.each do |item| + sym = item.lhs + + h[sym] ||= Set.new + h[sym] << get_state_item(state, item) + end + + state.items.each do |item| + next if item.end_of_rule? + next if item.next_sym.term? + + sym = item.next_sym + state_item = get_state_item(state, item) + @productions[state_item] = h[sym] + + # @type var key: [State, Grammar::Symbol] + key = [state, sym] + @reverse_productions[key] ||= Set.new + @reverse_productions[key] << state_item + end + end + end + + # For optimization, use same Triple if it's already created. + # Do not create new Triple instance anywhere else + # to avoid miss hash lookup. + # + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> Triple + def get_triple(state_item, precise_lookahead_set) + key = (precise_lookahead_set << @state_item_shift) | state_item.id + @triples[key] ||= Triple.new(state_item, precise_lookahead_set) + end + + # @rbs (State conflict_state, State::ShiftReduceConflict conflict) -> Example + def shift_reduce_example(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + # @type var shift_conflict_item: ::Lrama::State::Item + shift_conflict_item = conflict_state.items.find { |item| item.next_sym == conflict_symbol } + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) + end + path1 = with_timeout("#find_shift_conflict_shortest_path:") do + find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + end + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + # @rbs (State conflict_state, State::ReduceReduceConflict conflict) -> Example + def reduce_reduce_examples(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + path1 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) + end + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + end + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + # @rbs (Array[StateItem]? reduce_state_items, State conflict_state, State::Item conflict_item) -> Array[StateItem] + def find_shift_conflict_shortest_path(reduce_state_items, conflict_state, conflict_item) + time1 = Time.now.to_f + @iterate_count = 0 + + target_state_item = get_state_item(conflict_state, conflict_item) + result = [target_state_item] + reversed_state_items = reduce_state_items.to_a.reverse + # Index for state_item + i = 0 + + while (state_item = reversed_state_items[i]) + # Index for prev_state_item + j = i + 1 + _j = j + + while (prev_state_item = reversed_state_items[j]) + if prev_state_item.type == :production + j += 1 + else + break + end + end + + if target_state_item == state_item || target_state_item.item.start_item? + result.concat( + reversed_state_items[_j..-1] #: Array[StateItem] + ) + break + end + + if target_state_item.type == :production + queue = [] #: Array[Node[StateItem]] + queue << Node.new(target_state_item, nil) + + # Find reverse production + while (sis = queue.shift) + @iterate_count += 1 + si = sis.elem + + # Reach to start state + if si.item.start_item? + a = Node.to_a(sis).reverse + a.shift + result.concat(a) + target_state_item = si + break + end + + if si.type == :production + # @type var key: [State, Grammar::Symbol] + key = [si.state, si.item.lhs] + @reverse_productions[key].each do |state_item| + queue << Node.new(state_item, sis) + end + else + # @type var key: [StateItem, Grammar::Symbol] + key = [si, si.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item&.state + a = Node.to_a(sis).reverse + a.shift + result.concat(a) + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + queue.clear + break + end + end + end + else + # Find reverse transition + # @type var key: [StateItem, Grammar::Symbol] + key = [target_state_item, target_state_item.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item&.state + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + break + end + end + end + + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "find_shift_conflict_shortest_path #{@iterate_count} iteration", duration) + end + + result.reverse + end + + # @rbs (StateItem target) -> Set[StateItem] + def reachable_state_items(target) + result = Set.new + queue = [target] + + while (state_item = queue.shift) + next if result.include?(state_item) + result << state_item + + @reverse_transitions[[state_item, state_item.item.previous_sym]]&.each do |prev_state_item| + queue << prev_state_item + end + + if state_item.item.beginning_of_rule? + @reverse_productions[[state_item.state, state_item.item.lhs]]&.each do |si| + queue << si + end + end + end + + result + end + + # @rbs (State conflict_state, State::Item conflict_reduce_item, Grammar::Symbol conflict_term) -> ::Array[StateItem]? + def shortest_path(conflict_state, conflict_reduce_item, conflict_term) + time1 = Time.now.to_f + @iterate_count = 0 + + queue = [] #: Array[[Triple, Path]] + visited = {} #: Hash[Triple, true] + start_state = @states.states.first #: Lrama::State + conflict_term_bit = Bitmap::from_integer(conflict_term.number) + raise "BUG: Start state should be just one kernel." if start_state.kernels.count != 1 + reachable = reachable_state_items(get_state_item(conflict_state, conflict_reduce_item)) + start = get_triple(get_state_item(start_state, start_state.kernels.first), Bitmap::from_integer(@states.eof_symbol.number)) + + queue << [start, Path.new(start.state_item, nil)] + + while (triple, path = queue.shift) + @iterate_count += 1 + + # Found + if (triple.state == conflict_state) && (triple.item == conflict_reduce_item) && (triple.l & conflict_term_bit != 0) + state_items = [path.state_item] + + while (path = path.parent) + state_items << path.state_item + end + + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "shortest_path #{@iterate_count} iteration", duration) + end + + return state_items.reverse + end + + # transition + next_state_item = @transitions[[triple.state_item, triple.item.next_sym]] + if next_state_item && reachable.include?(next_state_item) + # @type var t: Triple + t = get_triple(next_state_item, triple.l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end + end + + # production step + @productions[triple.state_item]&.each do |si| + next unless reachable.include?(si) + + l = follow_l(triple.item, triple.l) + # @type var t: Triple + t = get_triple(si, l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end + end + end + + return nil + end + + # @rbs (State::Item item, Bitmap::bitmap current_l) -> Bitmap::bitmap + def follow_l(item, current_l) + # 1. follow_L (A -> X1 ... Xn-1 • Xn) = L + # 2. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = {Xk+2} if Xk+2 is a terminal + # 3. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) if Xk+2 is a nonnullable nonterminal + # 4. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) + follow_L (A -> X1 ... Xk+1 • Xk+2 ... Xn) if Xk+2 is a nullable nonterminal + case + when item.number_of_rest_symbols == 1 + current_l + when item.next_next_sym.term? + item.next_next_sym.number_bitmap + when !item.next_next_sym.nullable + item.next_next_sym.first_set_bitmap + else + item.next_next_sym.first_set_bitmap | follow_l(item.new_by_next_position, current_l) + end + end + + # @rbs [T] (String message) { -> T } -> T + def with_timeout(message) + Timeout.timeout(PathSearchTimeLimit, Timeout::Error, message + " timeout of #{PathSearchTimeLimit} sec exceeded") do + yield + end + end + + # @rbs (Float|Integer duration) -> void + def increment_total_duration(duration) + @total_duration += duration + + if !@exceed_cumulative_time_limit && @total_duration > CumulativeTimeLimit + @exceed_cumulative_time_limit = true + STDERR.puts "CumulativeTimeLimit #{CumulativeTimeLimit} sec exceeded then skip following Counterexamples calculation" + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/derivation.rb b/tool/lrama/lib/lrama/counterexamples/derivation.rb new file mode 100644 index 0000000000..a2b74767a9 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/derivation.rb @@ -0,0 +1,76 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Derivation + # @rbs! + # @item: State::Item + # @left: Derivation? + + attr_reader :item #: State::Item + attr_reader :left #: Derivation? + attr_accessor :right #: Derivation? + + # @rbs (State::Item item, Derivation? left) -> void + def initialize(item, left) + @item = item + @left = left + end + + # @rbs () -> ::String + def to_s + "#<Derivation(#{item.display_name})>" + end + alias :inspect :to_s + + # @rbs () -> Array[String] + def render_strings_for_report + result = [] #: Array[String] + _render_for_report(self, 0, result, 0) + result.map(&:rstrip) + end + + # @rbs () -> String + def render_for_report + render_strings_for_report.join("\n") + end + + private + + # @rbs (Derivation derivation, Integer offset, Array[String] strings, Integer index) -> Integer + def _render_for_report(derivation, offset, strings, index) + item = derivation.item + if strings[index] + strings[index] << " " * (offset - strings[index].length) + else + strings[index] = " " * offset + end + str = strings[index] + str << "#{item.rule_id}: #{item.symbols_before_dot.map(&:display_name).join(" ")} " + + if derivation.left + len = str.length + str << "#{item.next_sym.display_name}" + length = _render_for_report(derivation.left, len, strings, index + 1) + # I want String#ljust! + str << " " * (length - str.length) if length > str.length + else + str << " • #{item.symbols_after_dot.map(&:display_name).join(" ")} " + return str.length + end + + if derivation.right&.left + left = derivation.right&.left #: Derivation + length = _render_for_report(left, str.length, strings, index + 1) + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore + str << " " * (length - str.length) if length > str.length + elsif item.next_next_sym + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore + end + + return str.length + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/example.rb b/tool/lrama/lib/lrama/counterexamples/example.rb new file mode 100644 index 0000000000..c007f45af4 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/example.rb @@ -0,0 +1,154 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Example + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @path1: ::Array[StateItem] + # @path2: ::Array[StateItem] + # @conflict: State::conflict + # @conflict_symbol: Grammar::Symbol + # @counterexamples: Counterexamples + # @derivations1: Derivation + # @derivations2: Derivation + + attr_reader :path1 #: ::Array[StateItem] + attr_reader :path2 #: ::Array[StateItem] + attr_reader :conflict #: State::conflict + attr_reader :conflict_symbol #: Grammar::Symbol + + # path1 is shift conflict when S/R conflict + # path2 is always reduce conflict + # + # @rbs (Array[StateItem]? path1, Array[StateItem]? path2, State::conflict conflict, Grammar::Symbol conflict_symbol, Counterexamples counterexamples) -> void + def initialize(path1, path2, conflict, conflict_symbol, counterexamples) + @path1 = path1 + @path2 = path2 + @conflict = conflict + @conflict_symbol = conflict_symbol + @counterexamples = counterexamples + end + + # @rbs () -> (:shift_reduce | :reduce_reduce) + def type + @conflict.type + end + + # @rbs () -> State::Item + def path1_item + @path1.last.item + end + + # @rbs () -> State::Item + def path2_item + @path2.last.item + end + + # @rbs () -> Derivation + def derivations1 + @derivations1 ||= _derivations(path1) + end + + # @rbs () -> Derivation + def derivations2 + @derivations2 ||= _derivations(path2) + end + + private + + # @rbs (Array[StateItem] state_items) -> Derivation + def _derivations(state_items) + derivation = nil #: Derivation + current = :production + last_state_item = state_items.last #: StateItem + lookahead_sym = last_state_item.item.end_of_rule? ? @conflict_symbol : nil + + state_items.reverse_each do |si| + item = si.item + + case current + when :production + case si.type + when :start + derivation = Derivation.new(item, derivation) + current = :start + when :transition + derivation = Derivation.new(item, derivation) + current = :transition + when :production + derivation = Derivation.new(item, derivation) + current = :production + else + raise "Unexpected. #{si}" + end + + if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym) + si2 = @counterexamples.transitions[[si, item.next_sym]] + derivation2 = find_derivation_for_symbol(si2, lookahead_sym) + derivation.right = derivation2 # steep:ignore + lookahead_sym = nil + end + + when :transition + case si.type + when :start + derivation = Derivation.new(item, derivation) + current = :start + when :transition + # ignore + current = :transition + when :production + # ignore + current = :production + end + else + raise "BUG: Unknown #{current}" + end + + break if current == :start + end + + derivation + end + + # @rbs (StateItem state_item, Grammar::Symbol sym) -> Derivation? + def find_derivation_for_symbol(state_item, sym) + queue = [] #: Array[Array[StateItem]] + queue << [state_item] + + while (sis = queue.shift) + si = sis.last + next_sym = si.item.next_sym + + if next_sym == sym + derivation = nil + + sis.reverse_each do |si| + derivation = Derivation.new(si.item, derivation) + end + + return derivation + end + + if next_sym.nterm? && next_sym.first_set.include?(sym) + @counterexamples.productions[si].each do |next_si| + next if next_si.item.empty_rule? + next if sis.include?(next_si) + queue << (sis + [next_si]) + end + + if next_sym.nullable + next_si = @counterexamples.transitions[[si, next_sym]] + queue << (sis + [next_si]) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/node.rb b/tool/lrama/lib/lrama/counterexamples/node.rb new file mode 100644 index 0000000000..9214a0e7f1 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/node.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + # @rbs generic E < Object -- Type of an element + class Node + attr_reader :elem #: E + attr_reader :next_node #: Node[E]? + + # @rbs [E < Object] (Node[E] node) -> Array[E] + def self.to_a(node) + a = [] # steep:ignore UnannotatedEmptyCollection + + while (node) + a << node.elem + node = node.next_node + end + + a + end + + # @rbs (E elem, Node[E]? next_node) -> void + def initialize(elem, next_node) + @elem = elem + @next_node = next_node + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/path.rb b/tool/lrama/lib/lrama/counterexamples/path.rb new file mode 100644 index 0000000000..6b1325f73b --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/path.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Path + # @rbs! + # @state_item: StateItem + # @parent: Path? + + attr_reader :state_item #: StateItem + attr_reader :parent #: Path? + + # @rbs (StateItem state_item, Path? parent) -> void + def initialize(state_item, parent) + @state_item = state_item + @parent = parent + end + + # @rbs () -> ::String + def to_s + "#<Path>" + end + alias :inspect :to_s + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/state_item.rb b/tool/lrama/lib/lrama/counterexamples/state_item.rb new file mode 100644 index 0000000000..8c2481d793 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/state_item.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class StateItem + attr_reader :id #: Integer + attr_reader :state #: State + attr_reader :item #: State::Item + + # @rbs (Integer id, State state, State::Item item) -> void + def initialize(id, state, item) + @id = id + @state = state + @item = item + end + + # @rbs () -> (:start | :transition | :production) + def type + case + when item.start_item? + :start + when item.beginning_of_rule? + :production + else + :transition + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/triple.rb b/tool/lrama/lib/lrama/counterexamples/triple.rb new file mode 100644 index 0000000000..98fe051f53 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/triple.rb @@ -0,0 +1,41 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Triple + attr_reader :precise_lookahead_set #: Bitmap::bitmap + + alias :l :precise_lookahead_set + + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> void + def initialize(state_item, precise_lookahead_set) + @state_item = state_item + @precise_lookahead_set = precise_lookahead_set + end + + # @rbs () -> State + def state + @state_item.state + end + alias :s :state + + # @rbs () -> State::Item + def item + @state_item.item + end + alias :itm :item + + # @rbs () -> StateItem + def state_item + @state_item + end + + # @rbs () -> ::String + def inspect + "#{state.inspect}. #{item.display_name}. #{l.to_s(2)}" + end + alias :to_s :inspect + end + end +end diff --git a/tool/lrama/lib/lrama/diagram.rb b/tool/lrama/lib/lrama/diagram.rb new file mode 100644 index 0000000000..985808933f --- /dev/null +++ b/tool/lrama/lib/lrama/diagram.rb @@ -0,0 +1,77 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Diagram + class << self + # @rbs (IO out, Grammar grammar, String template_name) -> void + def render(out:, grammar:, template_name: 'diagram/diagram.html') + return unless require_railroad_diagrams + new(out: out, grammar: grammar, template_name: template_name).render + end + + # @rbs () -> bool + def require_railroad_diagrams + require "railroad_diagrams" + true + rescue LoadError + warn "railroad_diagrams is not installed. Please run `bundle install`." + false + end + end + + # @rbs (IO out, Grammar grammar, String template_name) -> void + def initialize(out:, grammar:, template_name: 'diagram/diagram.html') + @grammar = grammar + @out = out + @template_name = template_name + end + + # @rbs () -> void + def render + RailroadDiagrams::TextDiagram.set_formatting(RailroadDiagrams::TextDiagram::PARTS_UNICODE) + @out << ERB.render(template_file, output: self) + end + + # @rbs () -> string + def default_style + RailroadDiagrams::Style::default_style + end + + # @rbs () -> string + def diagrams + result = +'' + @grammar.unique_rule_s_values.each do |s_value| + diagrams = + @grammar.select_rules_by_s_value(s_value).map { |r| r.to_diagrams } + add_diagram( + s_value, + RailroadDiagrams::Diagram.new( + RailroadDiagrams::Choice.new(0, *diagrams), + ), + result + ) + end + result + end + + private + + # @rbs () -> string + def template_dir + File.expand_path('../../template', __dir__) + end + + # @rbs () -> string + def template_file + File.join(template_dir, @template_name) + end + + # @rbs (String name, RailroadDiagrams::Diagram diagram, String result) -> void + def add_diagram(name, diagram, result) + result << "\n<h2 class=\"diagram-header\">#{RailroadDiagrams.escape_html(name)}</h2>" + diagram.write_svg(result.method(:<<)) + result << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/digraph.rb b/tool/lrama/lib/lrama/digraph.rb new file mode 100644 index 0000000000..52865f52dd --- /dev/null +++ b/tool/lrama/lib/lrama/digraph.rb @@ -0,0 +1,104 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Digraph Algorithm of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # + # Digraph is an algorithm for graph data structure. + # The algorithm efficiently traverses SCC (Strongly Connected Component) of graph + # and merges nodes attributes within the same SCC. + # + # `compute_read_sets` and `compute_follow_sets` have the same structure. + # Graph of gotos and attributes of gotos are given then compute propagated attributes for each node. + # + # In the case of `compute_read_sets`: + # + # * Set of gotos is nodes of graph + # * `reads_relation` is edges of graph + # * `direct_read_sets` is nodes attributes + # + # In the case of `compute_follow_sets`: + # + # * Set of gotos is nodes of graph + # * `includes_relation` is edges of graph + # * `read_sets` is nodes attributes + # + # + # @rbs generic X < Object -- Type of a node + # @rbs generic Y < _Or -- Type of attribute sets assigned to a node which should support merge operation (#| method) + class Digraph + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # interface _Or + # def |: (self) -> self + # end + # @sets: Array[X] + # @relation: Hash[X, Array[X]] + # @base_function: Hash[X, Y] + # @stack: Array[X] + # @h: Hash[X, (Integer|Float)?] + # @result: Hash[X, Y] + + # @rbs sets: Array[X] -- Nodes of graph + # @rbs relation: Hash[X, Array[X]] -- Edges of graph + # @rbs base_function: Hash[X, Y] -- Attributes of nodes + # @rbs return: void + def initialize(sets, relation, base_function) + + # X in the paper + @sets = sets + + # R in the paper + @relation = relation + + # F' in the paper + @base_function = base_function + + # S in the paper + @stack = [] + + # N in the paper + @h = Hash.new(0) + + # F in the paper + @result = {} + end + + # @rbs () -> Hash[X, Y] + def compute + @sets.each do |x| + next if @h[x] != 0 + traverse(x) + end + + return @result + end + + private + + # @rbs (X x) -> void + def traverse(x) + @stack.push(x) + d = @stack.count + @h[x] = d + @result[x] = @base_function[x] # F x = F' x + + @relation[x]&.each do |y| + traverse(y) if @h[y] == 0 + @h[x] = [@h[x], @h[y]].min + @result[x] |= @result[y] # F x = F x + F y + end + + if @h[x] == d + while (z = @stack.pop) do + @h[z] = Float::INFINITY + break if z == x + @result[z] = @result[x] # F (Top of S) = F x + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/erb.rb b/tool/lrama/lib/lrama/erb.rb new file mode 100644 index 0000000000..8f8be54811 --- /dev/null +++ b/tool/lrama/lib/lrama/erb.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "erb" + +module Lrama + class ERB + # @rbs (String file, **untyped kwargs) -> String + def self.render(file, **kwargs) + new(file).render(**kwargs) + end + + # @rbs (String file) -> void + def initialize(file) + input = File.read(file) + if ::ERB.instance_method(:initialize).parameters.last.first == :key + @erb = ::ERB.new(input, trim_mode: '-') + else + @erb = ::ERB.new(input, nil, '-') # steep:ignore UnexpectedPositionalArgument + end + @erb.filename = file + end + + # @rbs (**untyped kwargs) -> String + def render(**kwargs) + @erb.result_with_hash(kwargs) + end + end +end diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb new file mode 100644 index 0000000000..95a80bb01c --- /dev/null +++ b/tool/lrama/lib/lrama/grammar.rb @@ -0,0 +1,603 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "grammar/auxiliary" +require_relative "grammar/binding" +require_relative "grammar/code" +require_relative "grammar/counter" +require_relative "grammar/destructor" +require_relative "grammar/error_token" +require_relative "grammar/inline" +require_relative "grammar/parameterized" +require_relative "grammar/percent_code" +require_relative "grammar/precedence" +require_relative "grammar/printer" +require_relative "grammar/reference" +require_relative "grammar/rule" +require_relative "grammar/rule_builder" +require_relative "grammar/symbol" +require_relative "grammar/symbols" +require_relative "grammar/type" +require_relative "grammar/union" +require_relative "lexer" + +module Lrama + # Grammar is the result of parsing an input grammar file + class Grammar + # @rbs! + # + # interface _DelegatedMethods + # def rules: () -> Array[Rule] + # def accept_symbol: () -> Grammar::Symbol + # def eof_symbol: () -> Grammar::Symbol + # def undef_symbol: () -> Grammar::Symbol + # def precedences: () -> Array[Precedence] + # + # # delegate to @symbols_resolver + # def symbols: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def ielr_defined?: () -> bool + # end + # + # include Symbols::Resolver::_DelegatedMethods + # + # @rule_counter: Counter + # @percent_codes: Array[PercentCode] + # @printers: Array[Printer] + # @destructors: Array[Destructor] + # @error_tokens: Array[ErrorToken] + # @symbols_resolver: Symbols::Resolver + # @types: Array[Type] + # @rule_builders: Array[RuleBuilder] + # @rules: Array[Rule] + # @sym_to_rules: Hash[Integer, Array[Rule]] + # @parameterized_resolver: Parameterized::Resolver + # @empty_symbol: Grammar::Symbol + # @eof_symbol: Grammar::Symbol + # @error_symbol: Grammar::Symbol + # @undef_symbol: Grammar::Symbol + # @accept_symbol: Grammar::Symbol + # @aux: Auxiliary + # @no_stdlib: bool + # @locations: bool + # @define: Hash[String, String] + # @required: bool + # @union: Union + # @precedences: Array[Precedence] + # @start_nterm: Lrama::Lexer::Token::Base? + + extend Forwardable + + attr_reader :percent_codes #: Array[PercentCode] + attr_reader :eof_symbol #: Grammar::Symbol + attr_reader :error_symbol #: Grammar::Symbol + attr_reader :undef_symbol #: Grammar::Symbol + attr_reader :accept_symbol #: Grammar::Symbol + attr_reader :aux #: Auxiliary + attr_reader :parameterized_resolver #: Parameterized::Resolver + attr_reader :precedences #: Array[Precedence] + attr_accessor :union #: Union + attr_accessor :expect #: Integer + attr_accessor :printers #: Array[Printer] + attr_accessor :error_tokens #: Array[ErrorToken] + attr_accessor :lex_param #: String + attr_accessor :parse_param #: String + attr_accessor :initial_action #: Grammar::Code::InitialActionCode + attr_accessor :after_shift #: Lexer::Token::Base + attr_accessor :before_reduce #: Lexer::Token::Base + attr_accessor :after_reduce #: Lexer::Token::Base + attr_accessor :after_shift_error_token #: Lexer::Token::Base + attr_accessor :after_pop_stack #: Lexer::Token::Base + attr_accessor :symbols_resolver #: Symbols::Resolver + attr_accessor :types #: Array[Type] + attr_accessor :rules #: Array[Rule] + attr_accessor :rule_builders #: Array[RuleBuilder] + attr_accessor :sym_to_rules #: Hash[Integer, Array[Rule]] + attr_accessor :no_stdlib #: bool + attr_accessor :locations #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :required #: bool + + def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value, + :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, + :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type, + :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number! + + # @rbs (Counter rule_counter, bool locations, Hash[String, String] define) -> void + def initialize(rule_counter, locations, define = {}) + @rule_counter = rule_counter + + # Code defined by "%code" + @percent_codes = [] + @printers = [] + @destructors = [] + @error_tokens = [] + @symbols_resolver = Grammar::Symbols::Resolver.new + @types = [] + @rule_builders = [] + @rules = [] + @sym_to_rules = {} + @parameterized_resolver = Parameterized::Resolver.new + @empty_symbol = nil + @eof_symbol = nil + @error_symbol = nil + @undef_symbol = nil + @accept_symbol = nil + @aux = Auxiliary.new + @no_stdlib = false + @locations = locations + @define = define + @required = false + @precedences = [] + @start_nterm = nil + + append_special_symbols + end + + # @rbs (Counter rule_counter, Counter midrule_action_counter) -> RuleBuilder + def create_rule_builder(rule_counter, midrule_action_counter) + RuleBuilder.new(rule_counter, midrule_action_counter, @parameterized_resolver) + end + + # @rbs (id: Lexer::Token::Base, code: Lexer::Token::UserCode) -> Array[PercentCode] + def add_percent_code(id:, code:) + @percent_codes << PercentCode.new(id.s_value, code.s_value) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Destructor] + def add_destructor(ident_or_tags:, token_code:, lineno:) + @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Printer] + def add_printer(ident_or_tags:, token_code:, lineno:) + @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[ErrorToken] + def add_error_token(ident_or_tags:, token_code:, lineno:) + @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> Array[Type] + def add_type(id:, tag:) + @types << Type.new(id: id, tag: tag) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_nonassoc(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :nonassoc, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_left(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :left, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_right(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :right, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_precedence(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :precedence, precedence: precedence, lineno: lineno)) + end + + # @rbs (Lrama::Lexer::Token::Base id) -> Lrama::Lexer::Token::Base + def set_start_nterm(id) + # When multiple `%start` directives are defined, Bison does not generate an error, + # whereas Lrama does generate an error. + # Related Bison's specification are + # refs: https://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html + if @start_nterm.nil? + @start_nterm = id + else + start = @start_nterm #: Lrama::Lexer::Token::Base + raise "Start non-terminal is already set to #{start.s_value} (line: #{start.first_line}). Cannot set to #{id.s_value} (line: #{id.first_line})." + end + end + + # @rbs (Grammar::Symbol sym, Precedence precedence) -> (Precedence | bot) + def set_precedence(sym, precedence) + @precedences << precedence + sym.precedence = precedence + end + + # @rbs (Grammar::Code::NoReferenceCode code, Integer lineno) -> Union + def set_union(code, lineno) + @union = Union.new(code: code, lineno: lineno) + end + + # @rbs (RuleBuilder builder) -> Array[RuleBuilder] + def add_rule_builder(builder) + @rule_builders << builder + end + + # @rbs (Parameterized::Rule rule) -> Array[Parameterized::Rule] + def add_parameterized_rule(rule) + @parameterized_resolver.add_rule(rule) + end + + # @rbs () -> Array[Parameterized::Rule] + def parameterized_rules + @parameterized_resolver.rules + end + + # @rbs (Array[Parameterized::Rule] rules) -> Array[Parameterized::Rule] + def prepend_parameterized_rules(rules) + @parameterized_resolver.rules = rules + @parameterized_resolver.rules + end + + # @rbs (Integer prologue_first_lineno) -> Integer + def prologue_first_lineno=(prologue_first_lineno) + @aux.prologue_first_lineno = prologue_first_lineno + end + + # @rbs (String prologue) -> String + def prologue=(prologue) + @aux.prologue = prologue + end + + # @rbs (Integer epilogue_first_lineno) -> Integer + def epilogue_first_lineno=(epilogue_first_lineno) + @aux.epilogue_first_lineno = epilogue_first_lineno + end + + # @rbs (String epilogue) -> String + def epilogue=(epilogue) + @aux.epilogue = epilogue + end + + # @rbs () -> void + def prepare + resolve_inline_rules + normalize_rules + collect_symbols + set_lhs_and_rhs + fill_default_precedence + fill_symbols + fill_sym_to_rules + sort_precedence + compute_nullable + compute_first_set + set_locations + end + + # TODO: More validation methods + # + # * Validation for no_declared_type_reference + # + # @rbs () -> void + def validate! + @symbols_resolver.validate! + validate_no_precedence_for_nterm! + validate_rule_lhs_is_nterm! + validate_duplicated_precedence! + end + + # @rbs (Grammar::Symbol sym) -> Array[Rule] + def find_rules_by_symbol!(sym) + find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found") + end + + # @rbs (Grammar::Symbol sym) -> Array[Rule]? + def find_rules_by_symbol(sym) + @sym_to_rules[sym.number] + end + + # @rbs (String s_value) -> Array[Rule] + def select_rules_by_s_value(s_value) + @rules.select {|rule| rule.lhs.id.s_value == s_value } + end + + # @rbs () -> Array[String] + def unique_rule_s_values + @rules.map {|rule| rule.lhs.id.s_value }.uniq + end + + # @rbs () -> bool + def ielr_defined? + @define.key?('lr.type') && @define['lr.type'] == 'ielr' + end + + private + + # @rbs () -> void + def sort_precedence + @precedences.sort_by! do |prec| + prec.symbol.number + end + @precedences.freeze + end + + # @rbs () -> Array[Grammar::Symbol] + def compute_nullable + @rules.each do |rule| + case + when rule.empty_rule? + rule.nullable = true + when rule.rhs.any?(&:term) + rule.nullable = false + else + # noop + end + end + + while true do + rs = @rules.select {|e| e.nullable.nil? } + nts = nterms.select {|e| e.nullable.nil? } + rule_count_1 = rs.count + nterm_count_1 = nts.count + + rs.each do |rule| + if rule.rhs.all?(&:nullable) + rule.nullable = true + end + end + + nts.each do |nterm| + find_rules_by_symbol!(nterm).each do |rule| + if rule.nullable + nterm.nullable = true + end + end + end + + rule_count_2 = @rules.count {|e| e.nullable.nil? } + nterm_count_2 = nterms.count {|e| e.nullable.nil? } + + if (rule_count_1 == rule_count_2) && (nterm_count_1 == nterm_count_2) + break + end + end + + rules.select {|r| r.nullable.nil? }.each do |rule| + rule.nullable = false + end + + nterms.select {|e| e.nullable.nil? }.each do |nterm| + nterm.nullable = false + end + end + + # @rbs () -> Array[Grammar::Symbol] + def compute_first_set + terms.each do |term| + term.first_set = Set.new([term]).freeze + term.first_set_bitmap = Lrama::Bitmap.from_array([term.number]) + end + + nterms.each do |nterm| + nterm.first_set = Set.new([]).freeze + nterm.first_set_bitmap = Lrama::Bitmap.from_array([]) + end + + while true do + changed = false + + @rules.each do |rule| + rule.rhs.each do |r| + if rule.lhs.first_set_bitmap | r.first_set_bitmap != rule.lhs.first_set_bitmap + changed = true + rule.lhs.first_set_bitmap = rule.lhs.first_set_bitmap | r.first_set_bitmap + end + + break unless r.nullable + end + end + + break unless changed + end + + nterms.each do |nterm| + nterm.first_set = Lrama::Bitmap.to_array(nterm.first_set_bitmap).map do |number| + find_symbol_by_number!(number) + end.to_set + end + end + + # @rbs () -> Array[RuleBuilder] + def setup_rules + @rule_builders.each do |builder| + builder.setup_rules + end + end + + # @rbs () -> Grammar::Symbol + def append_special_symbols + # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated + # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) + # term.number = -2 + # @empty_symbol = term + + # YYEOF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYEOF"), alias_name: "\"end of file\"", token_id: 0) + term.number = 0 + term.eof_symbol = true + @eof_symbol = term + + # YYerror + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYerror"), alias_name: "error") + term.number = 1 + term.error_symbol = true + @error_symbol = term + + # YYUNDEF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYUNDEF"), alias_name: "\"invalid token\"") + term.number = 2 + term.undef_symbol = true + @undef_symbol = term + + # $accept + term = add_nterm(id: Lrama::Lexer::Token::Ident.new(s_value: "$accept")) + term.accept_symbol = true + @accept_symbol = term + end + + # @rbs () -> void + def resolve_inline_rules + while @rule_builders.any?(&:has_inline_rules?) do + @rule_builders = @rule_builders.flat_map do |builder| + if builder.has_inline_rules? + Inline::Resolver.new(builder).resolve + else + builder + end + end + end + end + + # @rbs () -> void + def normalize_rules + add_accept_rule + setup_rules + @rule_builders.each do |builder| + builder.rules.each do |rule| + add_nterm(id: rule._lhs, tag: rule.lhs_tag) + @rules << rule + end + end + + nterms.freeze + @rules.sort_by!(&:id).freeze + end + + # Add $accept rule to the top of rules + def add_accept_rule + if @start_nterm + start = @start_nterm #: Lrama::Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [start, @eof_symbol.id], token_code: nil, lineno: start.line) + else + rule_builder = @rule_builders.first #: RuleBuilder + lineno = rule_builder ? rule_builder.line : 0 + lhs = rule_builder.lhs #: Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + end + end + + # Collect symbols from rules + # + # @rbs () -> void + def collect_symbols + @rules.flat_map(&:_rhs).each do |s| + case s + when Lrama::Lexer::Token::Char + add_term(id: s) + when Lrama::Lexer::Token::Base + # skip + else + raise "Unknown class: #{s}" + end + end + + terms.freeze + end + + # @rbs () -> void + def set_lhs_and_rhs + @rules.each do |rule| + rule.lhs = token_to_symbol(rule._lhs) if rule._lhs + + rule.rhs = rule._rhs.map do |t| + token_to_symbol(t) + end + end + end + + # Rule inherits precedence from the last term in RHS. + # + # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html + # + # @rbs () -> void + def fill_default_precedence + @rules.each do |rule| + # Explicitly specified precedence has the highest priority + next if rule.precedence_sym + + precedence_sym = nil + rule.rhs.each do |sym| + precedence_sym = sym if sym.term? + end + + rule.precedence_sym = precedence_sym + end + end + + # @rbs () -> Array[Grammar::Symbol] + def fill_symbols + fill_symbol_number + fill_nterm_type(@types) + fill_printer(@printers) + fill_destructor(@destructors) + fill_error_token(@error_tokens) + sort_by_number! + end + + # @rbs () -> Array[Rule] + def fill_sym_to_rules + @rules.each do |rule| + key = rule.lhs.number + @sym_to_rules[key] ||= [] + @sym_to_rules[key] << rule + end + end + + # @rbs () -> void + def validate_no_precedence_for_nterm! + errors = [] #: Array[String] + + nterms.each do |nterm| + next if nterm.precedence.nil? + + errors << "[BUG] Precedence #{nterm.name} (line: #{nterm.precedence.lineno}) is defined for nonterminal symbol (line: #{nterm.id.first_line}). Precedence can be defined for only terminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void + def validate_rule_lhs_is_nterm! + errors = [] #: Array[String] + + rules.each do |rule| + next if rule.lhs.nterm? + + errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is terminal symbol. It should be nonterminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # # @rbs () -> void + def validate_duplicated_precedence! + errors = [] #: Array[String] + seen = {} #: Hash[String, Precedence] + + precedences.each do |prec| + s_value = prec.s_value + if first = seen[s_value] + errors << "%#{prec.type} redeclaration for #{s_value} (line: #{prec.lineno}) previous declaration was %#{first.type} (line: #{first.lineno})" + else + seen[s_value] = prec + end + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void + def set_locations + @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? } + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/auxiliary.rb b/tool/lrama/lib/lrama/grammar/auxiliary.rb new file mode 100644 index 0000000000..76cfb74d4d --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/auxiliary.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # Grammar file information not used by States but by Output + class Auxiliary + attr_accessor :prologue_first_lineno #: Integer? + attr_accessor :prologue #: String? + attr_accessor :epilogue_first_lineno #: Integer? + attr_accessor :epilogue #: String? + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/binding.rb b/tool/lrama/lib/lrama/grammar/binding.rb new file mode 100644 index 0000000000..5940d153a9 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/binding.rb @@ -0,0 +1,79 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Binding + # @rbs @actual_args: Array[Lexer::Token::Base] + # @rbs @param_to_arg: Hash[String, Lexer::Token::Base] + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> void + def initialize(params, actual_args) + @actual_args = actual_args + @param_to_arg = build_param_to_arg(params, @actual_args) + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def resolve_symbol(sym) + return create_instantiate_rule(sym) if sym.is_a?(Lexer::Token::InstantiateRule) + find_arg_for_param(sym) + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def concatenated_args_str(token) + "#{token.rule_name}_#{format_args(token)}" + end + + private + + # @rbs (Lexer::Token::InstantiateRule sym) -> Lexer::Token::InstantiateRule + def create_instantiate_rule(sym) + Lrama::Lexer::Token::InstantiateRule.new( + s_value: sym.s_value, + location: sym.location, + args: resolve_args(sym.args), + lhs_tag: sym.lhs_tag + ) + end + + # @rbs (Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def resolve_args(args) + args.map { |arg| resolve_symbol(arg) } + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def find_arg_for_param(sym) + if (arg = @param_to_arg[sym.s_value]&.dup) + arg.alias_name = sym.alias_name + arg + else + sym + end + end + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> Hash[String, Lexer::Token::Base?] + def build_param_to_arg(params, actual_args) + params.zip(actual_args).map do |param, arg| + [param.s_value, arg] + end.to_h + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def format_args(token) + token_to_args_s_values(token).join('_') + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Array[String] + def token_to_args_s_values(token) + token.args.flat_map do |arg| + resolved = resolve_symbol(arg) + if resolved.is_a?(Lexer::Token::InstantiateRule) + [resolved.s_value] + resolved.args.map(&:s_value) + else + [resolved.s_value] + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb new file mode 100644 index 0000000000..f1b860eeba --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -0,0 +1,68 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "code/destructor_code" +require_relative "code/initial_action_code" +require_relative "code/no_reference_code" +require_relative "code/printer_code" +require_relative "code/rule_action" + +module Lrama + class Grammar + class Code + # @rbs! + # + # # delegated + # def s_value: -> String + # def line: -> Integer + # def column: -> Integer + # def references: -> Array[Lrama::Grammar::Reference] + + extend Forwardable + + def_delegators "token_code", :s_value, :line, :column, :references + + attr_reader :type #: ::Symbol + attr_reader :token_code #: Lexer::Token::UserCode + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode) -> void + def initialize(type:, token_code:) + @type = type + @token_code = token_code + end + + # @rbs (Code other) -> bool + def ==(other) + self.class == other.class && + self.type == other.type && + self.token_code == other.token_code + end + + # $$, $n, @$, @n are translated to C code + # + # @rbs () -> String + def translated_code + t_code = s_value.dup + + references.reverse_each do |ref| + first_column = ref.first_column + last_column = ref.last_column + + str = reference_to_c(ref) + + t_code[first_column...last_column] = str + end + + return t_code + end + + private + + # @rbs (Lrama::Grammar::Reference ref) -> bot + def reference_to_c(ref) + raise NotImplementedError.new("#reference_to_c is not implemented") + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb new file mode 100644 index 0000000000..d71b62e513 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class DestructorCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb new file mode 100644 index 0000000000..cb36041524 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class InitialActionCode < Code + private + + # * ($$) yylval + # * (@$) yylloc + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + "yylval" + when ref.type == :at && ref.name == "$" # @$ + "yylloc" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in initial_action." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in initial_action." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in initial_action." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in initial_action." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb new file mode 100644 index 0000000000..1d39919979 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class NoReferenceCode < Code + private + + # * ($$) error + # * (@$) error + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> bot + def reference_to_c(ref) + case + when ref.type == :dollar # $$, $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @$, @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:$, $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/printer_code.rb b/tool/lrama/lib/lrama/grammar/code/printer_code.rb new file mode 100644 index 0000000000..c6e25d5235 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/printer_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class PrinterCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/rule_action.rb b/tool/lrama/lib/lrama/grammar/code/rule_action.rb new file mode 100644 index 0000000000..e71e93e5a5 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/rule_action.rb @@ -0,0 +1,109 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class RuleAction < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @rule: Rule + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, rule: Rule) -> void + def initialize(type:, token_code:, rule:) + super(type: type, token_code: token_code) + @rule = rule + end + + private + + # * ($$) yyval + # * (@$) yyloc + # * ($:$) error + # * ($1) yyvsp[i] + # * (@1) yylsp[i] + # * ($:1) i - 1 + # + # + # Consider a rule like + # + # class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # + # For the semantic action of original rule: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 $2 $3 $4 $5 + # "Index for yyvsp" -4 -3 -2 -1 0 + # "$:n" $:1 $:2 $:3 $:4 $:5 + # "index of $:n" -5 -4 -3 -2 -1 + # + # + # For the first midrule action: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 + # "Index for yyvsp" 0 + # "$:n" $:1 + # + # @rbs (Reference ref) -> String + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + tag = ref.ex_tag || lhs.tag + raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag + "(yyval.#{tag.member})" + when ref.type == :at && ref.name == "$" # @$ + "(yyloc)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:$ is not supported" + when ref.type == :dollar # $n + i = -position_in_rhs + ref.index + tag = ref.ex_tag || rhs[ref.index - 1].tag + raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag + "(yyvsp[#{i}].#{tag.member})" + when ref.type == :at # @n + i = -position_in_rhs + ref.index + "(yylsp[#{i}])" + when ref.type == :index # $:n + i = -position_in_rhs + ref.index + "(#{i} - 1)" + else + raise "Unexpected. #{self}, #{ref}" + end + end + + # @rbs () -> Integer + def position_in_rhs + # If rule is not derived rule, User Code is only action at + # the end of rule RHS. In such case, the action is located on + # `@rule.rhs.count`. + @rule.position_in_original_rule_rhs || @rule.rhs.count + end + + # If this is midrule action, RHS is an RHS of the original rule. + # + # @rbs () -> Array[Grammar::Symbol] + def rhs + (@rule.original_rule || @rule).rhs + end + + # Unlike `rhs`, LHS is always an LHS of the rule. + # + # @rbs () -> Grammar::Symbol + def lhs + @rule.lhs + end + + # @rbs (Reference ref) -> bot + def raise_tag_not_found_error(ref) + raise "Tag is not specified for '$#{ref.value}' in '#{@rule.display_name}'" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/counter.rb b/tool/lrama/lib/lrama/grammar/counter.rb new file mode 100644 index 0000000000..ced934309d --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/counter.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Counter + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @number: Integer + + # @rbs (Integer number) -> void + def initialize(number) + @number = number + end + + # @rbs () -> Integer + def increment + n = @number + @number += 1 + n + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/destructor.rb b/tool/lrama/lib/lrama/grammar/destructor.rb new file mode 100644 index 0000000000..0ce8611e77 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/destructor.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Destructor + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident|Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::DestructorCode.new(type: :destructor, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/error_token.rb b/tool/lrama/lib/lrama/grammar/error_token.rb new file mode 100644 index 0000000000..9d9ed54ae2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/error_token.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class ErrorToken + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident | Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::PrinterCode.new(type: :error_token, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/inline.rb b/tool/lrama/lib/lrama/grammar/inline.rb new file mode 100644 index 0000000000..c02ab6002b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative 'inline/resolver' diff --git a/tool/lrama/lib/lrama/grammar/inline/resolver.rb b/tool/lrama/lib/lrama/grammar/inline/resolver.rb new file mode 100644 index 0000000000..aca689ccfb --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline/resolver.rb @@ -0,0 +1,80 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Inline + class Resolver + # @rbs (Lrama::Grammar::RuleBuilder rule_builder) -> void + def initialize(rule_builder) + @rule_builder = rule_builder + end + + # @rbs () -> Array[Lrama::Grammar::RuleBuilder] + def resolve + resolved_builders = [] #: Array[Lrama::Grammar::RuleBuilder] + @rule_builder.rhs.each_with_index do |token, i| + if (rule = @rule_builder.parameterized_resolver.find_inline(token)) + rule.rhs.each do |rhs| + builder = build_rule(rhs, token, i, rule) + resolved_builders << builder + end + break + end + end + resolved_builders + end + + private + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Lrama::Lexer::Token token, Integer index, Lrama::Grammar::Parameterized::Rule rule) -> Lrama::Grammar::RuleBuilder + def build_rule(rhs, token, index, rule) + builder = RuleBuilder.new( + @rule_builder.rule_counter, + @rule_builder.midrule_action_counter, + @rule_builder.parameterized_resolver, + lhs_tag: @rule_builder.lhs_tag + ) + resolve_rhs(builder, rhs, index, token, rule) + builder.lhs = @rule_builder.lhs + builder.line = @rule_builder.line + builder.precedence_sym = @rule_builder.precedence_sym + builder.user_code = replace_user_code(rhs, index) + builder + end + + # @rbs (Lrama::Grammar::RuleBuilder builder, Lrama::Grammar::Parameterized::Rhs rhs, Integer index, Lrama::Lexer::Token token, Lrama::Grammar::Parameterized::Rule rule) -> void + def resolve_rhs(builder, rhs, index, token, rule) + @rule_builder.rhs.each_with_index do |tok, i| + if i == index + rhs.symbols.each do |sym| + if token.is_a?(Lexer::Token::InstantiateRule) + bindings = Binding.new(rule.parameters, token.args) + builder.add_rhs(bindings.resolve_symbol(sym)) + else + builder.add_rhs(sym) + end + end + else + builder.add_rhs(tok) + end + end + end + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Integer index) -> Lrama::Lexer::Token::UserCode + def replace_user_code(rhs, index) + user_code = @rule_builder.user_code + return user_code if rhs.user_code.nil? || user_code.nil? + + code = user_code.s_value.gsub(/\$#{index + 1}/, rhs.user_code.s_value) + user_code.references.each do |ref| + next if ref.index.nil? || ref.index <= index # nil は $$ の場合 + code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (rhs.symbols.count - 1)}") + code = code.gsub(/@#{ref.index}/, "@#{ref.index + (rhs.symbols.count - 1)}") + end + Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized.rb b/tool/lrama/lib/lrama/grammar/parameterized.rb new file mode 100644 index 0000000000..48db3433f3 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'parameterized/resolver' +require_relative 'parameterized/rhs' +require_relative 'parameterized/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb new file mode 100644 index 0000000000..558f308190 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Resolver + attr_accessor :rules #: Array[Rule] + attr_accessor :created_lhs_list #: Array[Lexer::Token::Base] + + # @rbs () -> void + def initialize + @rules = [] + @created_lhs_list = [] + end + + # @rbs (Rule rule) -> Array[Rule] + def add_rule(rule) + @rules << rule + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Rule? + def find_rule(token) + select_rules(@rules, token).last + end + + # @rbs (Lexer::Token::Base token) -> Rule? + def find_inline(token) + @rules.reverse.find { |rule| rule.name == token.s_value && rule.inline? } + end + + # @rbs (String lhs_s_value) -> Lexer::Token::Base? + def created_lhs(lhs_s_value) + @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } + end + + # @rbs () -> Array[Rule] + def redefined_rules + @rules.select { |rule| @rules.count { |r| r.name == rule.name && r.required_parameters_count == rule.required_parameters_count } > 1 } + end + + private + + # @rbs (Array[Rule] rules, Lexer::Token::InstantiateRule token) -> Array[Rule] + def select_rules(rules, token) + rules = reject_inline_rules(rules) + rules = select_rules_by_name(rules, token.rule_name) + rules = rules.select { |rule| rule.required_parameters_count == token.args_count } + if rules.empty? + raise "Invalid number of arguments. `#{token.rule_name}`" + else + rules + end + end + + # @rbs (Array[Rule] rules) -> Array[Rule] + def reject_inline_rules(rules) + rules.reject(&:inline?) + end + + # @rbs (Array[Rule] rules, String rule_name) -> Array[Rule] + def select_rules_by_name(rules, rule_name) + rules = rules.select { |rule| rule.name == rule_name } + if rules.empty? + raise "Parameterized rule does not exist. `#{rule_name}`" + else + rules + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb new file mode 100644 index 0000000000..663de49100 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rhs + attr_accessor :symbols #: Array[Lexer::Token::Base] + attr_accessor :user_code #: Lexer::Token::UserCode? + attr_accessor :precedence_sym #: Grammar::Symbol? + + # @rbs () -> void + def initialize + @symbols = [] + @user_code = nil + @precedence_sym = nil + end + + # @rbs (Grammar::Binding bindings) -> Lexer::Token::UserCode? + def resolve_user_code(bindings) + return unless user_code + + resolved = Lexer::Token::UserCode.new(s_value: user_code.s_value, location: user_code.location) + var_to_arg = {} #: Hash[String, String] + symbols.each do |sym| + resolved_sym = bindings.resolve_symbol(sym) + if resolved_sym != sym + var_to_arg[sym.s_value] = resolved_sym.s_value + end + end + + var_to_arg.each do |var, arg| + resolved.references.each do |ref| + if ref.name == var + ref.name = arg + end + end + end + + return resolved + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rule.rb b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb new file mode 100644 index 0000000000..7048be3cff --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb @@ -0,0 +1,36 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rule + attr_reader :name #: String + attr_reader :parameters #: Array[Lexer::Token::Base] + attr_reader :rhs #: Array[Rhs] + attr_reader :required_parameters_count #: Integer + attr_reader :tag #: Lexer::Token::Tag? + + # @rbs (String name, Array[Lexer::Token::Base] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void + def initialize(name, parameters, rhs, tag: nil, is_inline: false) + @name = name + @parameters = parameters + @rhs = rhs + @tag = tag + @is_inline = is_inline + @required_parameters_count = parameters.count + end + + # @rbs () -> String + def to_s + "#{@name}(#{@parameters.map(&:s_value).join(', ')})" + end + + # @rbs () -> bool + def inline? + @is_inline + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/percent_code.rb b/tool/lrama/lib/lrama/grammar/percent_code.rb new file mode 100644 index 0000000000..9afb903056 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/percent_code.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class PercentCode + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @name: String + # @code: String + + attr_reader :name #: String + attr_reader :code #: String + + # @rbs (String name, String code) -> void + def initialize(name, code) + @name = name + @code = code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/precedence.rb b/tool/lrama/lib/lrama/grammar/precedence.rb new file mode 100644 index 0000000000..b4c6403372 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/precedence.rb @@ -0,0 +1,55 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Precedence < Struct.new(:type, :symbol, :precedence, :s_value, :lineno, keyword_init: true) + include Comparable + # @rbs! + # type type_enum = :left | :right | :nonassoc | :precedence + # + # attr_accessor type: type_enum + # attr_accessor symbol: Grammar::Symbol + # attr_accessor precedence: Integer + # attr_accessor s_value: String + # attr_accessor lineno: Integer + # + # def initialize: (?type: type_enum, ?symbol: Grammar::Symbol, ?precedence: Integer, ?s_value: ::String, ?lineno: Integer) -> void + + attr_reader :used_by_lalr #: Array[State::ResolvedConflict] + attr_reader :used_by_ielr #: Array[State::ResolvedConflict] + + # @rbs (Precedence other) -> Integer + def <=>(other) + self.precedence <=> other.precedence + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_lalr(resolved_conflict) + @used_by_lalr ||= [] #: Array[State::ResolvedConflict] + @used_by_lalr << resolved_conflict + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_ielr(resolved_conflict) + @used_by_ielr ||= [] #: Array[State::ResolvedConflict] + @used_by_ielr << resolved_conflict + end + + # @rbs () -> bool + def used_by? + used_by_lalr? || used_by_ielr? + end + + # @rbs () -> bool + def used_by_lalr? + !@used_by_lalr.nil? && !@used_by_lalr.empty? + end + + # @rbs () -> bool + def used_by_ielr? + !@used_by_ielr.nil? && !@used_by_ielr.empty? + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/printer.rb b/tool/lrama/lib/lrama/grammar/printer.rb new file mode 100644 index 0000000000..490fe701db --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/printer.rb @@ -0,0 +1,20 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Printer < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + # @rbs! + # attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag] + # attr_accessor token_code: Lexer::Token::UserCode + # attr_accessor lineno: Integer + # + # def initialize: (?ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], ?token_code: Lexer::Token::UserCode, ?lineno: Integer) -> void + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::PrinterCode.new(type: :printer, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/reference.rb b/tool/lrama/lib/lrama/grammar/reference.rb new file mode 100644 index 0000000000..7e3badfecc --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/reference.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # type: :dollar or :at + # name: String (e.g. $$, $foo, $expr.right) + # number: Integer (e.g. $1) + # index: Integer + # ex_tag: "$<tag>1" (Optional) + class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + # @rbs! + # attr_accessor type: ::Symbol + # attr_accessor name: String + # attr_accessor number: Integer + # attr_accessor index: Integer + # attr_accessor ex_tag: Lexer::Token::Base? + # attr_accessor first_column: Integer + # attr_accessor last_column: Integer + # + # def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token::Base?, first_column: Integer, last_column: Integer) -> void + + # @rbs () -> (String|Integer) + def value + name || number + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule.rb b/tool/lrama/lib/lrama/grammar/rule.rb new file mode 100644 index 0000000000..d00d6a8883 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule.rb @@ -0,0 +1,135 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # _rhs holds original RHS element. Use rhs to refer to Symbol. + class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true) + # @rbs! + # + # interface _DelegatedMethods + # def lhs: -> Grammar::Symbol + # def rhs: -> Array[Grammar::Symbol] + # end + # + # attr_accessor id: Integer + # attr_accessor _lhs: Lexer::Token::Base + # attr_accessor lhs: Grammar::Symbol + # attr_accessor lhs_tag: Lexer::Token::Tag? + # attr_accessor _rhs: Array[Lexer::Token::Base] + # attr_accessor rhs: Array[Grammar::Symbol] + # attr_accessor token_code: Lexer::Token::UserCode? + # attr_accessor position_in_original_rule_rhs: Integer + # attr_accessor nullable: bool + # attr_accessor precedence_sym: Grammar::Symbol? + # attr_accessor lineno: Integer? + # + # def initialize: ( + # ?id: Integer, ?_lhs: Lexer::Token::Base?, ?lhs: Lexer::Token::Base, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token::Base], ?rhs: Array[Grammar::Symbol], + # ?token_code: Lexer::Token::UserCode?, ?position_in_original_rule_rhs: Integer?, ?nullable: bool, + # ?precedence_sym: Grammar::Symbol?, ?lineno: Integer? + # ) -> void + + attr_accessor :original_rule #: Rule + + # @rbs (Rule other) -> bool + def ==(other) + self.class == other.class && + self.lhs == other.lhs && + self.lhs_tag == other.lhs_tag && + self.rhs == other.rhs && + self.token_code == other.token_code && + self.position_in_original_rule_rhs == other.position_in_original_rule_rhs && + self.nullable == other.nullable && + self.precedence_sym == other.precedence_sym && + self.lineno == other.lineno + end + + # @rbs () -> String + def display_name + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map {|r| r.id.s_value }.join(" ") + "#{l} -> #{r}" + end + + # @rbs () -> String + def display_name_without_action + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map do |r| + r.id.s_value if r.first_set.any? + end.compact.join(" ") + + "#{l} -> #{r}" + end + + # @rbs () -> (RailroadDiagrams::Skip | RailroadDiagrams::Sequence) + def to_diagrams + if rhs.empty? + RailroadDiagrams::Skip.new + else + RailroadDiagrams::Sequence.new(*rhs_to_diagram) + end + end + + # Used by #user_actions + # + # @rbs () -> String + def as_comment + l = lhs.id.s_value + r = empty_rule? ? "%empty" : rhs.map(&:display_name).join(" ") + + "#{l}: #{r}" + end + + # @rbs () -> String + def with_actions + "#{display_name} {#{token_code&.s_value}}" + end + + # opt_nl: ε <-- empty_rule + # | '\n' <-- not empty_rule + # + # @rbs () -> bool + def empty_rule? + rhs.empty? + end + + # @rbs () -> Precedence? + def precedence + precedence_sym&.precedence + end + + # @rbs () -> bool + def initial_rule? + id == 0 + end + + # @rbs () -> String? + def translated_code + return nil unless token_code + + Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self).translated_code + end + + # @rbs () -> bool + def contains_at_reference? + return false unless token_code + + token_code.references.any? {|r| r.type == :at } + end + + private + + # @rbs () -> Array[(RailroadDiagrams::Terminal | RailroadDiagrams::NonTerminal)] + def rhs_to_diagram + rhs.map do |r| + if r.term + RailroadDiagrams::Terminal.new(r.id.s_value) + else + RailroadDiagrams::NonTerminal.new(r.id.s_value) + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb new file mode 100644 index 0000000000..34fdca6c86 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -0,0 +1,270 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class RuleBuilder + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @position_in_original_rule_rhs: Integer? + # @skip_preprocess_references: bool + # @rules: Array[Rule] + # @rule_builders_for_parameterized: Array[RuleBuilder] + # @rule_builders_for_derived_rules: Array[RuleBuilder] + # @parameterized_rules: Array[Rule] + # @midrule_action_rules: Array[Rule] + # @replaced_rhs: Array[Lexer::Token::Base]? + + attr_accessor :lhs #: Lexer::Token::Base? + attr_accessor :line #: Integer? + attr_reader :rule_counter #: Counter + attr_reader :midrule_action_counter #: Counter + attr_reader :parameterized_resolver #: Grammar::Parameterized::Resolver + attr_reader :lhs_tag #: Lexer::Token::Tag? + attr_reader :rhs #: Array[Lexer::Token::Base] + attr_reader :user_code #: Lexer::Token::UserCode? + attr_reader :precedence_sym #: Grammar::Symbol? + + # @rbs (Counter rule_counter, Counter midrule_action_counter, Grammar::Parameterized::Resolver parameterized_resolver, ?Integer position_in_original_rule_rhs, ?lhs_tag: Lexer::Token::Tag?, ?skip_preprocess_references: bool) -> void + def initialize(rule_counter, midrule_action_counter, parameterized_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) + @rule_counter = rule_counter + @midrule_action_counter = midrule_action_counter + @parameterized_resolver = parameterized_resolver + @position_in_original_rule_rhs = position_in_original_rule_rhs + @skip_preprocess_references = skip_preprocess_references + + @lhs = nil + @lhs_tag = lhs_tag + @rhs = [] + @user_code = nil + @precedence_sym = nil + @line = nil + @rules = [] + @rule_builders_for_parameterized = [] + @rule_builders_for_derived_rules = [] + @parameterized_rules = [] + @midrule_action_rules = [] + end + + # @rbs (Lexer::Token::Base rhs) -> void + def add_rhs(rhs) + @line ||= rhs.line + + flush_user_code + + @rhs << rhs + end + + # @rbs (Lexer::Token::UserCode? user_code) -> void + def user_code=(user_code) + @line ||= user_code&.line + + flush_user_code + + @user_code = user_code + end + + # @rbs (Grammar::Symbol? precedence_sym) -> void + def precedence_sym=(precedence_sym) + flush_user_code + + @precedence_sym = precedence_sym + end + + # @rbs () -> void + def complete_input + freeze_rhs + end + + # @rbs () -> void + def setup_rules + preprocess_references unless @skip_preprocess_references + process_rhs + resolve_inline_rules + build_rules + end + + # @rbs () -> Array[Grammar::Rule] + def rules + @parameterized_rules + @midrule_action_rules + @rules + end + + # @rbs () -> bool + def has_inline_rules? + rhs.any? { |token| @parameterized_resolver.find_inline(token) } + end + + private + + # @rbs () -> void + def freeze_rhs + @rhs.freeze + end + + # @rbs () -> void + def preprocess_references + numberize_references + end + + # @rbs () -> void + def build_rules + tokens = @replaced_rhs #: Array[Lexer::Token::Base] + return if tokens.any? { |t| @parameterized_resolver.find_inline(t) } + + rule = Rule.new( + id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code, + position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line + ) + @rules = [rule] + @parameterized_rules = @rule_builders_for_parameterized.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules.each do |r| + r.original_rule = rule + end + end + + # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on. + # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`. + # + # @rbs () -> void + def process_rhs + return if @replaced_rhs + + replaced_rhs = [] #: Array[Lexer::Token::Base] + + rhs.each_with_index do |token, i| + case token + when Lrama::Lexer::Token::Char + replaced_rhs << token + when Lrama::Lexer::Token::Ident + replaced_rhs << token + when Lrama::Lexer::Token::InstantiateRule + parameterized_rule = @parameterized_resolver.find_rule(token) + raise "Unexpected token. #{token}" unless parameterized_rule + + bindings = Binding.new(parameterized_rule.parameters, token.args) + lhs_s_value = bindings.concatenated_args_str(token) + if (created_lhs = @parameterized_resolver.created_lhs(lhs_s_value)) + replaced_rhs << created_lhs + else + lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) + replaced_rhs << lhs_token + @parameterized_resolver.created_lhs_list << lhs_token + parameterized_rule.rhs.each do |r| + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, lhs_tag: token.lhs_tag || parameterized_rule.tag) + rule_builder.lhs = lhs_token + r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } + rule_builder.line = line + rule_builder.precedence_sym = r.precedence_sym + rule_builder.user_code = r.resolve_user_code(bindings) + rule_builder.complete_input + rule_builder.setup_rules + @rule_builders_for_parameterized << rule_builder + end + end + when Lrama::Lexer::Token::UserCode + prefix = token.referred ? "@" : "$@" + tag = token.tag || lhs_tag + new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) + replaced_rhs << new_token + + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, i, lhs_tag: tag, skip_preprocess_references: true) + rule_builder.lhs = new_token + rule_builder.user_code = token + rule_builder.complete_input + rule_builder.setup_rules + + @rule_builders_for_derived_rules << rule_builder + when Lrama::Lexer::Token::Empty + # Noop + else + raise "Unexpected token. #{token}" + end + end + + @replaced_rhs = replaced_rhs + end + + # @rbs () -> void + def resolve_inline_rules + while @rule_builders_for_parameterized.any?(&:has_inline_rules?) do + @rule_builders_for_parameterized = @rule_builders_for_parameterized.flat_map do |rule_builder| + if rule_builder.has_inline_rules? + inlined_builders = Inline::Resolver.new(rule_builder).resolve + inlined_builders.each { |builder| builder.setup_rules } + inlined_builders + else + rule_builder + end + end + end + end + + # @rbs () -> void + def numberize_references + # Bison n'th component is 1-origin + (rhs + [user_code]).compact.each.with_index(1) do |token, i| + next unless token.is_a?(Lrama::Lexer::Token::UserCode) + + token.references.each do |ref| + ref_name = ref.name + + if ref_name + if ref_name == '$' + ref.name = '$' + else + candidates = ([lhs] + rhs).each_with_index.select do |token, _i| + # @type var token: Lexer::Token::Base + token.referred_by?(ref_name) + end + + if candidates.size >= 2 + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is duplicated.") + end + + unless (referring_symbol = candidates.first) + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is not found.") + end + + if referring_symbol[1] == 0 # Refers to LHS + ref.name = '$' + else + ref.number = referring_symbol[1] + end + end + end + + if ref.number + ref.index = ref.number + end + + # TODO: Need to check index of @ too? + next if ref.type == :at + + if ref.index + # TODO: Prohibit $0 even so Bison allows it? + # See: https://www.gnu.org/software/bison/manual/html_node/Actions.html + token.invalid_ref(ref, "Can not refer following component. #{ref.index} >= #{i}.") if ref.index >= i + rhs[ref.index - 1].referred = true + end + end + end + end + + # @rbs () -> void + def flush_user_code + if (c = @user_code) + @rhs << c + @user_code = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/stdlib.y b/tool/lrama/lib/lrama/grammar/stdlib.y new file mode 100644 index 0000000000..dd397c9e08 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/stdlib.y @@ -0,0 +1,142 @@ +/********************************************************************** + + stdlib.y + + This is lrama's standard library. It provides a number of + parameterized rule definitions, such as options and lists, + that should be useful in a number of situations. + +**********************************************************************/ + +%% + +// ------------------------------------------------------------------- +// Options + +/* + * program: option(X) + * + * => + * + * program: option_X + * option_X: %empty + * option_X: X + */ +%rule option(X) + : /* empty */ + | X + ; + + +/* + * program: ioption(X) + * + * => + * + * program: %empty + * program: X + */ +%rule %inline ioption(X) + : /* empty */ + | X + ; + +// ------------------------------------------------------------------- +// Sequences + +/* + * program: preceded(opening, X) + * + * => + * + * program: preceded_opening_X + * preceded_opening_X: opening X + */ +%rule preceded(opening, X) + : opening X { $$ = $2; } + ; + +/* + * program: terminated(X, closing) + * + * => + * + * program: terminated_X_closing + * terminated_X_closing: X closing + */ +%rule terminated(X, closing) + : X closing { $$ = $1; } + ; + +/* + * program: delimited(opening, X, closing) + * + * => + * + * program: delimited_opening_X_closing + * delimited_opening_X_closing: opening X closing + */ +%rule delimited(opening, X, closing) + : opening X closing { $$ = $2; } + ; + +// ------------------------------------------------------------------- +// Lists + +/* + * program: list(X) + * + * => + * + * program: list_X + * list_X: %empty + * list_X: list_X X + */ +%rule list(X) + : /* empty */ + | list(X) X + ; + +/* + * program: nonempty_list(X) + * + * => + * + * program: nonempty_list_X + * nonempty_list_X: X + * nonempty_list_X: nonempty_list_X X + */ +%rule nonempty_list(X) + : X + | nonempty_list(X) X + ; + +/* + * program: separated_nonempty_list(separator, X) + * + * => + * + * program: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separated_nonempty_list_separator_X separator X + */ +%rule separated_nonempty_list(separator, X) + : X + | separated_nonempty_list(separator, X) separator X + ; + +/* + * program: separated_list(separator, X) + * + * => + * + * program: separated_list_separator_X + * separated_list_separator_X: option_separated_nonempty_list_separator_X + * option_separated_nonempty_list_separator_X: %empty + * option_separated_nonempty_list_separator_X: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separator separated_nonempty_list_separator_X X + */ +%rule separated_list(separator, X) + : option(separated_nonempty_list(separator, X)) + ; diff --git a/tool/lrama/lib/lrama/grammar/symbol.rb b/tool/lrama/lib/lrama/grammar/symbol.rb new file mode 100644 index 0000000000..07aee0c0a2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbol.rb @@ -0,0 +1,149 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# Symbol is both of nterm and term +# `number` is both for nterm and term +# `token_id` is tokentype for term, internal sequence number for nterm +# + +module Lrama + class Grammar + class Symbol + attr_accessor :id #: Lexer::Token::Base + attr_accessor :alias_name #: String? + attr_reader :number #: Integer + attr_accessor :number_bitmap #: Bitmap::bitmap + attr_accessor :tag #: Lexer::Token::Tag? + attr_accessor :token_id #: Integer + attr_accessor :nullable #: bool + attr_accessor :precedence #: Precedence? + attr_accessor :printer #: Printer? + attr_accessor :destructor #: Destructor? + attr_accessor :error_token #: ErrorToken + attr_accessor :first_set #: Set[Grammar::Symbol] + attr_accessor :first_set_bitmap #: Bitmap::bitmap + attr_reader :term #: bool + attr_writer :eof_symbol #: bool + attr_writer :error_symbol #: bool + attr_writer :undef_symbol #: bool + attr_writer :accept_symbol #: bool + + # @rbs (id: Lexer::Token::Base, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token::Tag?, + # ?token_id: Integer?, ?nullable: bool?, ?precedence: Precedence?, ?printer: Printer?) -> void + def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil) + @id = id + @alias_name = alias_name + @number = number + @tag = tag + @term = term + @token_id = token_id + @nullable = nullable + @precedence = precedence + @printer = printer + @destructor = destructor + end + + # @rbs (Integer) -> void + def number=(number) + @number = number + @number_bitmap = Bitmap::from_integer(number) + end + + # @rbs () -> bool + def term? + term + end + + # @rbs () -> bool + def nterm? + !term + end + + # @rbs () -> bool + def eof_symbol? + !!@eof_symbol + end + + # @rbs () -> bool + def error_symbol? + !!@error_symbol + end + + # @rbs () -> bool + def undef_symbol? + !!@undef_symbol + end + + # @rbs () -> bool + def accept_symbol? + !!@accept_symbol + end + + # @rbs () -> bool + def midrule? + return false if term? + + name.include?("$") || name.include?("@") + end + + # @rbs () -> String + def name + id.s_value + end + + # @rbs () -> String + def display_name + alias_name || name + end + + # name for yysymbol_kind_t + # + # See: b4_symbol_kind_base + # @type var name: String + # @rbs () -> String + def enum_name + case + when accept_symbol? + res = "YYACCEPT" + when eof_symbol? + res = "YYEOF" + when term? && id.is_a?(Lrama::Lexer::Token::Char) + res = number.to_s + display_name + when term? && id.is_a?(Lrama::Lexer::Token::Ident) + res = name + when midrule? + res = number.to_s + name + when nterm? + res = name + else + raise "Unexpected #{self}" + end + + "YYSYMBOL_" + res.gsub(/\W+/, "_") + end + + # comment for yysymbol_kind_t + # + # @rbs () -> String? + def comment + case + when accept_symbol? + # YYSYMBOL_YYACCEPT + name + when eof_symbol? + # YYEOF + alias_name + when (term? && 0 < token_id && token_id < 128) + # YYSYMBOL_3_backslash_, YYSYMBOL_14_ + display_name + when midrule? + # YYSYMBOL_21_1 + name + else + # YYSYMBOL_keyword_class, YYSYMBOL_strings_1 + display_name + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/symbols.rb b/tool/lrama/lib/lrama/grammar/symbols.rb new file mode 100644 index 0000000000..337241d1b2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative "symbols/resolver" diff --git a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb new file mode 100644 index 0000000000..085a835d28 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb @@ -0,0 +1,362 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Symbols + class Resolver + # @rbs! + # + # interface _DelegatedMethods + # def symbols: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def add_nterm: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + # def add_term: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + # def find_symbol_by_number!: (Integer number) -> Grammar::Symbol + # def find_symbol_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol + # def token_to_symbol: (Lexer::Token::Base token) -> Grammar::Symbol + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def fill_nterm_type: (Array[Grammar::Type] types) -> void + # def fill_symbol_number: () -> void + # def fill_printer: (Array[Grammar::Printer] printers) -> void + # def fill_destructor: (Array[Destructor] destructors) -> (Destructor | bot) + # def fill_error_token: (Array[Grammar::ErrorToken] error_tokens) -> void + # def sort_by_number!: () -> Array[Grammar::Symbol] + # end + # + # @symbols: Array[Grammar::Symbol]? + # @number: Integer + # @used_numbers: Hash[Integer, bool] + + attr_reader :terms #: Array[Grammar::Symbol] + attr_reader :nterms #: Array[Grammar::Symbol] + + # @rbs () -> void + def initialize + @terms = [] + @nterms = [] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols + @symbols ||= (@terms + @nterms) + end + + # @rbs () -> Array[Grammar::Symbol] + def sort_by_number! + symbols.sort_by!(&:number) + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) + if token_id && (sym = find_symbol_by_token_id(token_id)) + if replace + sym.id = id + sym.alias_name = alias_name + sym.tag = tag + end + + return sym + end + + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + term = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: true, token_id: token_id, nullable: false + ) + @terms << term + term + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + def add_nterm(id:, alias_name: nil, tag: nil) + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + nterm = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: false, token_id: nil, nullable: nil, + ) + @nterms << nterm + nterm + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_term_by_s_value(s_value) + terms.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_symbol_by_s_value(s_value) + symbols.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol + def find_symbol_by_s_value!(s_value) + find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`") + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol? + def find_symbol_by_id(id) + symbols.find do |s| + s.id == id || s.alias_name == id.s_value + end + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_symbol_by_id!(id) + find_symbol_by_id(id) || (raise "Symbol not found. #{id}") + end + + # @rbs (Integer token_id) -> Grammar::Symbol? + def find_symbol_by_token_id(token_id) + symbols.find {|s| s.token_id == token_id } + end + + # @rbs (Integer number) -> Grammar::Symbol + def find_symbol_by_number!(number) + sym = symbols[number] + + raise "Symbol not found. number: `#{number}`" unless sym + raise "[BUG] Symbol number mismatch. #{number}, #{sym}" if sym.number != number + + sym + end + + # @rbs () -> void + def fill_symbol_number + # YYEMPTY = -2 + # YYEOF = 0 + # YYerror = 1 + # YYUNDEF = 2 + @number = 3 + fill_terms_number + fill_nterms_number + end + + # @rbs (Array[Grammar::Type] types) -> void + def fill_nterm_type(types) + types.each do |type| + nterm = find_nterm_by_id!(type.id) + nterm.tag = type.tag + end + end + + # @rbs (Array[Grammar::Printer] printers) -> void + def fill_printer(printers) + symbols.each do |sym| + printers.each do |printer| + printer.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.printer = printer if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.printer = printer if sym.tag == ident_or_tag + else + raise "Unknown token type. #{printer}" + end + end + end + end + end + + # @rbs (Array[Destructor] destructors) -> (Array[Grammar::Symbol] | bot) + def fill_destructor(destructors) + symbols.each do |sym| + destructors.each do |destructor| + destructor.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.destructor = destructor if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.destructor = destructor if sym.tag == ident_or_tag + else + raise "Unknown token type. #{destructor}" + end + end + end + end + end + + # @rbs (Array[Grammar::ErrorToken] error_tokens) -> void + def fill_error_token(error_tokens) + symbols.each do |sym| + error_tokens.each do |token| + token.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.error_token = token if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.error_token = token if sym.tag == ident_or_tag + else + raise "Unknown token type. #{token}" + end + end + end + end + end + + # @rbs (Lexer::Token::Base token) -> Grammar::Symbol + def token_to_symbol(token) + case token + when Lrama::Lexer::Token::Base + find_symbol_by_id!(token) + else + raise "Unknown class: #{token}" + end + end + + # @rbs () -> void + def validate! + validate_number_uniqueness! + validate_alias_name_uniqueness! + validate_symbols! + end + + private + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_nterm_by_id!(id) + @nterms.find do |s| + s.id == id + end || (raise "Symbol not found. #{id}") + end + + # @rbs () -> void + def fill_terms_number + # Character literal in grammar file has + # token id corresponding to ASCII code by default, + # so start token_id from 256. + token_id = 256 + + @terms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + # If id is Token::Char, it uses ASCII code + if sym.token_id.nil? + if sym.id.is_a?(Lrama::Lexer::Token::Char) + # Ignore ' on the both sides + case sym.id.s_value[1..-2] + when "\\b" + sym.token_id = 8 + when "\\f" + sym.token_id = 12 + when "\\n" + sym.token_id = 10 + when "\\r" + sym.token_id = 13 + when "\\t" + sym.token_id = 9 + when "\\v" + sym.token_id = 11 + when "\"" + sym.token_id = 34 + when "'" + sym.token_id = 39 + when "\\\\" + sym.token_id = 92 + when /\A\\(\d+)\z/ + unless (id = Integer($1, 8)).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + when /\A(.)\z/ + unless (id = $1&.bytes&.first).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + else + raise "Unknown Char s_value #{sym}" + end + else + sym.token_id = token_id + token_id += 1 + end + end + end + end + + # @rbs () -> void + def fill_nterms_number + token_id = 0 + + @nterms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + if sym.token_id.nil? + sym.token_id = token_id + token_id += 1 + end + end + end + + # @rbs () -> Hash[Integer, bool] + def used_numbers + return @used_numbers if defined?(@used_numbers) + + @used_numbers = {} + symbols.map(&:number).each do |n| + @used_numbers[n] = true + end + @used_numbers + end + + # @rbs () -> void + def validate_number_uniqueness! + invalid = symbols.group_by(&:number).select do |number, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol number is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_alias_name_uniqueness! + invalid = symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol alias name is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_symbols! + symbols.each { |sym| sym.id.validate } + errors = symbols.map { |sym| sym.id.errors }.flatten.compact + return if errors.empty? + + raise errors.join("\n") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/type.rb b/tool/lrama/lib/lrama/grammar/type.rb new file mode 100644 index 0000000000..c631769447 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/type.rb @@ -0,0 +1,32 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Type + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @id: Lexer::Token::Base + # @tag: Lexer::Token::Tag + + attr_reader :id #: Lexer::Token::Base + attr_reader :tag #: Lexer::Token::Tag + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> void + def initialize(id:, tag:) + @id = id + @tag = tag + end + + # @rbs (Grammar::Type other) -> bool + def ==(other) + self.class == other.class && + self.id == other.id && + self.tag == other.tag + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/union.rb b/tool/lrama/lib/lrama/grammar/union.rb new file mode 100644 index 0000000000..774cc66fc6 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/union.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Union + attr_reader :code #: Grammar::Code::NoReferenceCode + attr_reader :lineno #: Integer + + # @rbs (code: Grammar::Code::NoReferenceCode, lineno: Integer) -> void + def initialize(code:, lineno:) + @code = code + @lineno = lineno + end + + # @rbs () -> String + def braces_less_code + # Braces is already removed by lexer + code.s_value + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb new file mode 100644 index 0000000000..ce98b505a7 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer.rb @@ -0,0 +1,219 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "strscan" + +require_relative "lexer/grammar_file" +require_relative "lexer/location" +require_relative "lexer/token" + +module Lrama + class Lexer + # @rbs! + # + # type token = lexer_token | c_token + # + # type lexer_token = [String, Token::Token] | + # [::Symbol, Token::Tag] | + # [::Symbol, Token::Char] | + # [::Symbol, Token::Str] | + # [::Symbol, Token::Int] | + # [::Symbol, Token::Ident] + # + # type c_token = [:C_DECLARATION, Token::UserCode] + + attr_reader :head_line #: Integer + attr_reader :head_column #: Integer + attr_reader :line #: Integer + attr_accessor :status #: :initial | :c_declaration + attr_accessor :end_symbol #: String? + + SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze #: Array[String] + PERCENT_TOKENS = %w( + %union + %token + %type + %nterm + %left + %right + %nonassoc + %expect + %define + %require + %printer + %destructor + %lex-param + %parse-param + %initial-action + %precedence + %prec + %error-token + %before-reduce + %after-reduce + %after-shift-error-token + %after-shift + %after-pop-stack + %empty + %code + %rule + %no-stdlib + %inline + %locations + %categories + %start + ).freeze #: Array[String] + + # @rbs (GrammarFile grammar_file) -> void + def initialize(grammar_file) + @grammar_file = grammar_file + @scanner = StringScanner.new(grammar_file.text) + @head_column = @head = @scanner.pos + @head_line = @line = 1 + @status = :initial + @end_symbol = nil + end + + # @rbs () -> token? + def next_token + case @status + when :initial + lex_token + when :c_declaration + lex_c_code + end + end + + # @rbs () -> Integer + def column + @scanner.pos - @head + end + + # @rbs () -> Location + def location + Location.new( + grammar_file: @grammar_file, + first_line: @head_line, first_column: @head_column, + last_line: line, last_column: column + ) + end + + # @rbs () -> lexer_token? + def lex_token + until @scanner.eos? do + case + when @scanner.scan(/\n/) + newline + when @scanner.scan(/\s+/) + @scanner.matched.count("\n").times { newline } + when @scanner.scan(/\/\*/) + lex_comment + when @scanner.scan(/\/\/.*(?<newline>\n)?/) + newline if @scanner[:newline] + else + break + end + end + + reset_first_position + + case + when @scanner.eos? + return + when @scanner.scan(/#{SYMBOLS.join('|')}/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/[\?\+\*]/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/<\w+>/) + return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'.'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/".*?"/) + return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] + when @scanner.scan(/\d+/) + return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)] + when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) + token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location) + type = + if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/) + :IDENT_COLON + else + :IDENTIFIER + end + return [type, token] + else + raise ParseError, location.generate_error_message("Unexpected token") # steep:ignore UnknownConstant + end + end + + # @rbs () -> c_token + def lex_c_code + nested = 0 + code = +'' + reset_first_position + + until @scanner.eos? do + case + when @scanner.scan(/{/) + code << @scanner.matched + nested += 1 + when @scanner.scan(/}/) + if nested == 0 && @end_symbol == '}' + @scanner.unscan + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + else + code << @scanner.matched + nested -= 1 + end + when @scanner.check(/#{@end_symbol}/) + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + when @scanner.scan(/\n/) + code << @scanner.matched + newline + when @scanner.scan(/".*?"/) + code << %Q(#{@scanner.matched}) + @line += @scanner.matched.count("\n") + when @scanner.scan(/'.*?'/) + code << %Q(#{@scanner.matched}) + when @scanner.scan(/[^\"'\{\}\n]+/) + code << @scanner.matched + when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) # steep:ignore + code << @scanner.matched + else + code << @scanner.getch + end + end + raise ParseError, location.generate_error_message("Unexpected code: #{code}") # steep:ignore UnknownConstant + end + + private + + # @rbs () -> void + def lex_comment + until @scanner.eos? do + case + when @scanner.scan_until(/[\s\S]*?\*\//) + @scanner.matched.count("\n").times { newline } + return + when @scanner.scan_until(/\n/) + newline + end + end + end + + # @rbs () -> void + def reset_first_position + @head_line = line + @head_column = column + end + + # @rbs () -> void + def newline + @line += 1 + @head = @scanner.pos + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/grammar_file.rb b/tool/lrama/lib/lrama/lexer/grammar_file.rb new file mode 100644 index 0000000000..37e82ff18d --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/grammar_file.rb @@ -0,0 +1,40 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + class GrammarFile + class Text < String + # @rbs () -> String + def inspect + length <= 50 ? super : "#{self[0..47]}...".inspect + end + end + + attr_reader :path #: String + attr_reader :text #: String + + # @rbs (String path, String text) -> void + def initialize(path, text) + @path = path + @text = Text.new(text).freeze + end + + # @rbs () -> String + def inspect + "<#{self.class}: @path=#{path}, @text=#{text.inspect}>" + end + + # @rbs (GrammarFile other) -> bool + def ==(other) + self.class == other.class && + self.path == other.path + end + + # @rbs () -> Array[String] + def lines + @lines ||= text.split("\n") + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/location.rb b/tool/lrama/lib/lrama/lexer/location.rb new file mode 100644 index 0000000000..4465576d53 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/location.rb @@ -0,0 +1,132 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + class Location + attr_reader :grammar_file #: GrammarFile + attr_reader :first_line #: Integer + attr_reader :first_column #: Integer + attr_reader :last_line #: Integer + attr_reader :last_column #: Integer + + # @rbs (grammar_file: GrammarFile, first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void + def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:) + @grammar_file = grammar_file + @first_line = first_line + @first_column = first_column + @last_line = last_line + @last_column = last_column + end + + # @rbs (Location other) -> bool + def ==(other) + self.class == other.class && + self.grammar_file == other.grammar_file && + self.first_line == other.first_line && + self.first_column == other.first_column && + self.last_line == other.last_line && + self.last_column == other.last_column + end + + # @rbs (Integer left, Integer right) -> Location + def partial_location(left, right) + offset = -first_column + new_first_line = -1 + new_first_column = -1 + new_last_line = -1 + new_last_column = -1 + + _text.each.with_index do |line, index| + new_offset = offset + line.length + 1 + + if offset <= left && left <= new_offset + new_first_line = first_line + index + new_first_column = left - offset + end + + if offset <= right && right <= new_offset + new_last_line = first_line + index + new_last_column = right - offset + end + + offset = new_offset + end + + Location.new( + grammar_file: grammar_file, + first_line: new_first_line, first_column: new_first_column, + last_line: new_last_line, last_column: new_last_column + ) + end + + # @rbs () -> String + def to_s + "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})" + end + + # @rbs (String error_message) -> String + def generate_error_message(error_message) + <<~ERROR.chomp + #{path}:#{first_line}:#{first_column}: #{error_message} + #{error_with_carets} + ERROR + end + + # @rbs () -> String + def error_with_carets + <<~TEXT + #{formatted_first_lineno} | #{text} + #{line_number_padding} | #{carets_line} + TEXT + end + + private + + # @rbs () -> String + def path + grammar_file.path + end + + # @rbs () -> String + def carets_line + leading_whitespace + highlight_marker + end + + # @rbs () -> String + def leading_whitespace + (text[0...first_column] or raise "Invalid first_column: #{first_column}") + .gsub(/[^\t]/, ' ') + end + + # @rbs () -> String + def highlight_marker + length = last_column - first_column + '^' + '~' * [0, length - 1].max + end + + # @rbs () -> String + def formatted_first_lineno + first_line.to_s.rjust(4) + end + + # @rbs () -> String + def line_number_padding + ' ' * formatted_first_lineno.length + end + + # @rbs () -> String + def text + @text ||= _text.join("\n") + end + + # @rbs () -> Array[String] + def _text + @_text ||=begin + range = (first_line - 1)...last_line + grammar_file.lines[range] or raise "#{range} is invalid" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb new file mode 100644 index 0000000000..37f77aa069 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -0,0 +1,20 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'token/base' +require_relative 'token/char' +require_relative 'token/empty' +require_relative 'token/ident' +require_relative 'token/instantiate_rule' +require_relative 'token/int' +require_relative 'token/str' +require_relative 'token/tag' +require_relative 'token/token' +require_relative 'token/user_code' + +module Lrama + class Lexer + module Token + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/base.rb b/tool/lrama/lib/lrama/lexer/token/base.rb new file mode 100644 index 0000000000..3df93bbc73 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/base.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Base + attr_reader :s_value #: String + attr_reader :location #: Location + attr_accessor :alias_name #: String + attr_accessor :referred #: bool + attr_reader :errors #: Array[String] + + # @rbs (s_value: String, ?alias_name: String, ?location: Location) -> void + def initialize(s_value:, alias_name: nil, location: nil) + s_value.freeze + @s_value = s_value + @alias_name = alias_name + @location = location + @errors = [] + end + + # @rbs () -> String + def to_s + "value: `#{s_value}`, location: #{location}" + end + + # @rbs (String string) -> bool + def referred_by?(string) + [self.s_value, self.alias_name].compact.include?(string) + end + + # @rbs (Lexer::Token::Base other) -> bool + def ==(other) + self.class == other.class && self.s_value == other.s_value + end + + # @rbs () -> Integer + def first_line + location.first_line + end + alias :line :first_line + + # @rbs () -> Integer + def first_column + location.first_column + end + alias :column :first_column + + # @rbs () -> Integer + def last_line + location.last_line + end + + # @rbs () -> Integer + def last_column + location.last_column + end + + # @rbs (Lrama::Grammar::Reference ref, String message) -> bot + def invalid_ref(ref, message) + location = self.location.partial_location(ref.first_column, ref.last_column) + raise location.generate_error_message(message) + end + + # @rbs () -> bool + def validate + true + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/char.rb b/tool/lrama/lib/lrama/lexer/token/char.rb new file mode 100644 index 0000000000..f4ef7c9fbc --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/char.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Char < Base + # @rbs () -> void + def validate + validate_ascii_code_range + end + + private + + # @rbs () -> void + def validate_ascii_code_range + unless s_value.ascii_only? + errors << "Invalid character: `#{s_value}`. Only ASCII characters are allowed." + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/empty.rb b/tool/lrama/lib/lrama/lexer/token/empty.rb new file mode 100644 index 0000000000..375e256493 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/empty.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Empty < Base + def initialize(location: nil) + super(s_value: '%empty', location: location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/ident.rb b/tool/lrama/lib/lrama/lexer/token/ident.rb new file mode 100644 index 0000000000..4880be9073 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/ident.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Ident < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb new file mode 100644 index 0000000000..7051ba75a4 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class InstantiateRule < Base + attr_reader :args #: Array[Lexer::Token::Base] + attr_reader :lhs_tag #: Lexer::Token::Tag? + + # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token::Base], ?lhs_tag: Lexer::Token::Tag?) -> void + def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil) + super s_value: s_value, alias_name: alias_name, location: location + @args = args + @lhs_tag = lhs_tag + end + + # @rbs () -> String + def rule_name + s_value + end + + # @rbs () -> Integer + def args_count + args.count + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/int.rb b/tool/lrama/lib/lrama/lexer/token/int.rb new file mode 100644 index 0000000000..7daf48d4d3 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/int.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Int < Base + # @rbs! + # def initialize: (s_value: Integer, ?alias_name: String, ?location: Location) -> void + # def s_value: () -> Integer + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/str.rb b/tool/lrama/lib/lrama/lexer/token/str.rb new file mode 100644 index 0000000000..cf9de6cf0f --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/str.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Str < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/tag.rb b/tool/lrama/lib/lrama/lexer/token/tag.rb new file mode 100644 index 0000000000..68c6268219 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/tag.rb @@ -0,0 +1,16 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Tag < Base + # @rbs () -> String + def member + # Omit "<>" + s_value[1..-2] or raise "Unexpected Tag format (#{s_value})" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/token.rb b/tool/lrama/lib/lrama/lexer/token/token.rb new file mode 100644 index 0000000000..935797efc6 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/token.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Token < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb new file mode 100644 index 0000000000..166f04954a --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -0,0 +1,109 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "strscan" + +module Lrama + class Lexer + module Token + class UserCode < Base + attr_accessor :tag #: Lexer::Token::Tag + + # @rbs () -> Array[Lrama::Grammar::Reference] + def references + @references ||= _references + end + + private + + # @rbs () -> Array[Lrama::Grammar::Reference] + def _references + scanner = StringScanner.new(s_value) + references = [] #: Array[Grammar::Reference] + + until scanner.eos? do + case + when reference = scan_reference(scanner) + references << reference + when scanner.scan(/\/\*/) + scanner.scan_until(/\*\//) + else + scanner.getch + end + end + + references + end + + # @rbs (StringScanner scanner) -> Lrama::Grammar::Reference? + def scan_reference(scanner) + start = scanner.pos + if scanner.scan(/ + # $ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + \$(<[a-zA-Z0-9_]+>)?(?: + (\$) # $$, $<long>$ + | (\d+) # $1, $2, $<long>1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $foo, $expr, $<long>program (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + ) + | + # @ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + @(?: + (\$) # @$ + | (\d+) # @1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # @foo, @expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # @[expr.right], @[expr-right] (named reference with brackets) + ) + | + # $: references + \$: + (?: + (\$) # $:$ + | (\d+) # $:1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $:foo, $:expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $:[expr.right], $:[expr-right] (named reference with brackets) + ) + /x) + case + # $ references + when scanner[2] # $$, $<long>$ + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[3] # $1, $2, $<long>1 + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[3]), index: Integer(scanner[3]), ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[4] # $foo, $expr, $<long>program (named reference without brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[4], ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[5] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[5], ex_tag: tag, first_column: start, last_column: scanner.pos) + + # @ references + when scanner[6] # @$ + return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) + when scanner[7] # @1 + return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[7]), index: Integer(scanner[7]), first_column: start, last_column: scanner.pos) + when scanner[8] # @foo, @expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[8], first_column: start, last_column: scanner.pos) + when scanner[9] # @[expr.right], @[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[9], first_column: start, last_column: scanner.pos) + + # $: references + when scanner[10] # $:$ + return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) + when scanner[11] # $:1 + return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[11]), index: Integer(scanner[11]), first_column: start, last_column: scanner.pos) + when scanner[12] # $:foo, $:expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[12], first_column: start, last_column: scanner.pos) + when scanner[13] # $:[expr.right], $:[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[13], first_column: start, last_column: scanner.pos) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/logger.rb b/tool/lrama/lib/lrama/logger.rb new file mode 100644 index 0000000000..291eea5296 --- /dev/null +++ b/tool/lrama/lib/lrama/logger.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Logger + # @rbs (IO out) -> void + def initialize(out = STDERR) + @out = out + end + + # @rbs () -> void + def line_break + @out << "\n" + end + + # @rbs (String message) -> void + def trace(message) + @out << message << "\n" + end + + # @rbs (String message) -> void + def warn(message) + @out << 'warning: ' << message << "\n" + end + + # @rbs (String message) -> void + def error(message) + @out << 'error: ' << message << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/option_parser.rb b/tool/lrama/lib/lrama/option_parser.rb new file mode 100644 index 0000000000..5a15d59c7b --- /dev/null +++ b/tool/lrama/lib/lrama/option_parser.rb @@ -0,0 +1,223 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require 'optparse' + +module Lrama + # Handle option parsing for the command line interface. + class OptionParser + # @rbs! + # @options: Lrama::Options + # @trace: Array[String] + # @report: Array[String] + # @profile: Array[String] + + # @rbs (Array[String]) -> Lrama::Options + def self.parse(argv) + new.parse(argv) + end + + # @rbs () -> void + def initialize + @options = Options.new + @trace = [] + @report = [] + @profile = [] + end + + # @rbs (Array[String]) -> Lrama::Options + def parse(argv) + parse_by_option_parser(argv) + + @options.trace_opts = validate_trace(@trace) + @options.report_opts = validate_report(@report) + @options.profile_opts = validate_profile(@profile) + @options.grammar_file = argv.shift + + unless @options.grammar_file + abort "File should be specified\n" + end + + if @options.grammar_file == '-' + @options.grammar_file = argv.shift or abort "File name for STDIN should be specified\n" + else + @options.y = File.open(@options.grammar_file, 'r') + end + + if !@report.empty? && @options.report_file.nil? && @options.grammar_file + @options.report_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".output" + end + + if !@options.header_file && @options.header + case + when @options.outfile + @options.header_file = File.dirname(@options.outfile) + "/" + File.basename(@options.outfile, ".*") + ".h" + when @options.grammar_file + @options.header_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".h" + end + end + + @options + end + + private + + # @rbs (Array[String]) -> void + def parse_by_option_parser(argv) + ::OptionParser.new do |o| + o.banner = <<~BANNER + Lrama is LALR (1) parser generator written by Ruby. + + Usage: lrama [options] FILE + BANNER + o.separator '' + o.separator 'STDIN mode:' + o.separator 'lrama [options] - FILE read grammar from STDIN' + o.separator '' + o.separator 'Tuning the Parser:' + o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v } + o.on('-t', '--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.separator " same as '-Dparse.trace'" + o.on('--locations', 'enable location support') {|v| @options.locations = true } + o.on('-D', '--define=NAME[=VALUE]', Array, "similar to '%define NAME VALUE'") do |v| + @options.define = v.each_with_object({}) do |item, hash| # steep:ignore UnannotatedEmptyCollection + key, value = item.split('=', 2) + hash[key] = value + end + end + o.separator '' + o.separator 'Output:' + o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v } + o.on('-d', 'also produce a header file') { @options.header = true } + o.on('-r', '--report=REPORTS', Array, 'also produce details on the automaton') {|v| @report = v } + o.on_tail '' + o.on_tail 'REPORTS is a list of comma-separated words that can include:' + o.on_tail ' states describe the states' + o.on_tail ' itemsets complete the core item sets with their closure' + o.on_tail ' lookaheads explicitly associate lookahead tokens to items' + o.on_tail ' solved describe shift/reduce conflicts solving' + o.on_tail ' counterexamples, cex generate conflict counterexamples' + o.on_tail ' rules list unused rules' + o.on_tail ' terms list unused terminals' + o.on_tail ' verbose report detailed internal state and analysis results' + o.on_tail ' all include all the above reports' + o.on_tail ' none disable all reports' + o.on('--report-file=FILE', 'also produce details on the automaton output to a file named FILE') {|v| @options.report_file = v } + o.on('-o', '--output=FILE', 'leave output to FILE') {|v| @options.outfile = v } + o.on('--trace=TRACES', Array, 'also output trace logs at runtime') {|v| @trace = v } + o.on_tail '' + o.on_tail 'TRACES is a list of comma-separated words that can include:' + o.on_tail ' automaton display states' + o.on_tail ' closure display states' + o.on_tail ' rules display grammar rules' + o.on_tail ' only-explicit-rules display only explicit grammar rules' + o.on_tail ' actions display grammar rules with actions' + o.on_tail ' time display generation time' + o.on_tail ' all include all the above traces' + o.on_tail ' none disable all traces' + o.on('--diagram=[FILE]', 'generate a diagram of the rules') do |v| + @options.diagram = true + @options.diagram_file = v if v + end + o.on('--profile=PROFILES', Array, 'profiles parser generation parts') {|v| @profile = v } + o.on_tail '' + o.on_tail 'PROFILES is a list of comma-separated words that can include:' + o.on_tail ' call-stack use sampling call-stack profiler (stackprof gem)' + o.on_tail ' memory use memory profiler (memory_profiler gem)' + o.on('-v', '--verbose', "same as '--report=state'") {|_v| @report << 'states' } + o.separator '' + o.separator 'Diagnostics:' + o.on('-W', '--warnings', 'report the warnings') {|v| @options.warnings = true } + o.separator '' + o.separator 'Error Recovery:' + o.on('-e', 'enable error recovery') {|v| @options.error_recovery = true } + o.separator '' + o.separator 'Other options:' + o.on('-V', '--version', "output version information and exit") {|v| puts "lrama #{Lrama::VERSION}"; exit 0 } + o.on('-h', '--help', "display this help and exit") {|v| puts o; exit 0 } + o.on_tail + o.parse!(argv) + end + end + + ALIASED_REPORTS = { cex: :counterexamples }.freeze #: Hash[Symbol, Symbol] + VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze #: Array[Symbol] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_report(report) + h = { grammar: true } + return h if report.empty? + return {} if report == ['none'] + if report == ['all'] + VALID_REPORTS.each { |r| h[r] = true } + return h + end + + report.each do |r| + aliased = aliased_report_option(r) + if VALID_REPORTS.include?(aliased) + h[aliased] = true + else + raise "Invalid report option \"#{r}\"." + end + end + + return h + end + + # @rbs (String) -> Symbol + def aliased_report_option(opt) + (ALIASED_REPORTS[opt.to_sym] || opt).to_sym + end + + VALID_TRACES = %w[ + locations scan parse automaton bitsets closure + grammar rules only-explicit-rules actions resource + sets muscles tools m4-early m4 skeleton time ielr cex + ].freeze #: Array[String] + NOT_SUPPORTED_TRACES = %w[ + locations scan parse bitsets grammar resource + sets muscles tools m4-early m4 skeleton ielr cex + ].freeze #: Array[String] + SUPPORTED_TRACES = VALID_TRACES - NOT_SUPPORTED_TRACES #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_trace(trace) + h = {} #: Hash[Symbol, bool] + return h if trace.empty? || trace == ['none'] + all_traces = SUPPORTED_TRACES - %w[only-explicit-rules] + if trace == ['all'] + all_traces.each { |t| h[t.gsub(/-/, '_').to_sym] = true } + return h + end + + trace.each do |t| + if SUPPORTED_TRACES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid trace option \"#{t}\".\nValid options are [#{SUPPORTED_TRACES.join(", ")}]." + end + end + + return h + end + + VALID_PROFILES = %w[call-stack memory].freeze #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_profile(profile) + h = {} #: Hash[Symbol, bool] + return h if profile.empty? + + profile.each do |t| + if VALID_PROFILES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid profile option \"#{t}\".\nValid options are [#{VALID_PROFILES.join(", ")}]." + end + end + + return h + end + end +end diff --git a/tool/lrama/lib/lrama/options.rb b/tool/lrama/lib/lrama/options.rb new file mode 100644 index 0000000000..87aec62448 --- /dev/null +++ b/tool/lrama/lib/lrama/options.rb @@ -0,0 +1,46 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Command line options. + class Options + attr_accessor :skeleton #: String + attr_accessor :locations #: bool + attr_accessor :header #: bool + attr_accessor :header_file #: String? + attr_accessor :report_file #: String? + attr_accessor :outfile #: String + attr_accessor :error_recovery #: bool + attr_accessor :grammar_file #: String + attr_accessor :trace_opts #: Hash[Symbol, bool]? + attr_accessor :report_opts #: Hash[Symbol, bool]? + attr_accessor :warnings #: bool + attr_accessor :y #: IO + attr_accessor :debug #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :diagram #: bool + attr_accessor :diagram_file #: String + attr_accessor :profile_opts #: Hash[Symbol, bool]? + + # @rbs () -> void + def initialize + @skeleton = "bison/yacc.c" + @locations = false + @define = {} + @header = false + @header_file = nil + @report_file = nil + @outfile = "y.tab.c" + @error_recovery = false + @grammar_file = '' + @trace_opts = nil + @report_opts = nil + @warnings = false + @y = STDIN + @debug = false + @diagram = false + @diagram_file = "diagram.html" + @profile_opts = nil + end + end +end diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb new file mode 100644 index 0000000000..d527be8bd4 --- /dev/null +++ b/tool/lrama/lib/lrama/output.rb @@ -0,0 +1,452 @@ +# frozen_string_literal: true + +require "forwardable" +require_relative "tracer/duration" + +module Lrama + class Output + extend Forwardable + include Tracer::Duration + + attr_reader :grammar_file_path, :context, :grammar, :error_recovery, :include_header + + def_delegators "@context", :yyfinal, :yylast, :yyntokens, :yynnts, :yynrules, :yynstates, + :yymaxutok, :yypact_ninf, :yytable_ninf + + def_delegators "@grammar", :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + + def initialize( + out:, output_file_path:, template_name:, grammar_file_path:, + context:, grammar:, header_out: nil, header_file_path: nil, error_recovery: false + ) + @out = out + @output_file_path = output_file_path + @template_name = template_name + @grammar_file_path = grammar_file_path + @header_out = header_out + @header_file_path = header_file_path + @context = context + @grammar = grammar + @error_recovery = error_recovery + @include_header = header_file_path ? header_file_path.sub("./", "") : nil + end + + if ERB.instance_method(:initialize).parameters.last.first == :key + def self.erb(input) + ERB.new(input, trim_mode: '-') + end + else + def self.erb(input) + ERB.new(input, nil, '-') + end + end + + def render_partial(file) + ERB.render(partial_file(file), context: @context, output: self) + end + + def render + report_duration(:render) do + tmp = eval_template(template_file, @output_file_path) + @out << tmp + + if @header_file_path + tmp = eval_template(header_template_file, @header_file_path) + + if @header_out + @header_out << tmp + else + File.write(@header_file_path, tmp) + end + end + end + end + + # A part of b4_token_enums + def token_enums + @context.yytokentype.map do |s_value, token_id, display_name| + s = sprintf("%s = %d%s", s_value, token_id, token_id == yymaxutok ? "" : ",") + + if display_name + sprintf(" %-30s /* %s */\n", s, display_name) + else + sprintf(" %s\n", s) + end + end.join + end + + # b4_symbol_enum + def symbol_enum + last_sym_number = @context.yysymbol_kind_t.last[1] + @context.yysymbol_kind_t.map do |s_value, sym_number, display_name| + s = sprintf("%s = %d%s", s_value, sym_number, (sym_number == last_sym_number) ? "" : ",") + + if display_name + sprintf(" %-40s /* %s */\n", s, display_name) + else + sprintf(" %s\n", s) + end + end.join + end + + def yytranslate + int_array_to_string(@context.yytranslate) + end + + def yytranslate_inverted + int_array_to_string(@context.yytranslate_inverted) + end + + def yyrline + int_array_to_string(@context.yyrline) + end + + def yytname + string_array_to_string(@context.yytname) + " YY_NULLPTR" + end + + # b4_int_type_for + def int_type_for(ary) + min = ary.min + max = ary.max + + case + when (-127 <= min && min <= 127) && (-127 <= max && max <= 127) + "yytype_int8" + when (0 <= min && min <= 255) && (0 <= max && max <= 255) + "yytype_uint8" + when (-32767 <= min && min <= 32767) && (-32767 <= max && max <= 32767) + "yytype_int16" + when (0 <= min && min <= 65535) && (0 <= max && max <= 65535) + "yytype_uint16" + else + "int" + end + end + + def symbol_actions_for_printer + @grammar.symbols.map do |sym| + next unless sym.printer + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.printer.lineno} "#{@grammar_file_path}" + {#{sym.printer.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + def symbol_actions_for_destructor + @grammar.symbols.map do |sym| + next unless sym.destructor + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.destructor.lineno} "#{@grammar_file_path}" + {#{sym.destructor.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + # b4_user_initial_action + def user_initial_action(comment = "") + return "" unless @grammar.initial_action + + <<-STR + #{comment} +#line #{@grammar.initial_action.line} "#{@grammar_file_path}" + {#{@grammar.initial_action.translated_code}} + STR + end + + def after_shift_function(comment = "") + return "" unless @grammar.after_shift + + <<-STR + #{comment} +#line #{@grammar.after_shift.line} "#{@grammar_file_path}" + {#{@grammar.after_shift.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def before_reduce_function(comment = "") + return "" unless @grammar.before_reduce + + <<-STR + #{comment} +#line #{@grammar.before_reduce.line} "#{@grammar_file_path}" + {#{@grammar.before_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_reduce_function(comment = "") + return "" unless @grammar.after_reduce + + <<-STR + #{comment} +#line #{@grammar.after_reduce.line} "#{@grammar_file_path}" + {#{@grammar.after_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_shift_error_token_function(comment = "") + return "" unless @grammar.after_shift_error_token + + <<-STR + #{comment} +#line #{@grammar.after_shift_error_token.line} "#{@grammar_file_path}" + {#{@grammar.after_shift_error_token.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def after_pop_stack_function(len, comment = "") + return "" unless @grammar.after_pop_stack + + <<-STR + #{comment} +#line #{@grammar.after_pop_stack.line} "#{@grammar_file_path}" + {#{@grammar.after_pop_stack.s_value}(#{len}#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def symbol_actions_for_error_token + @grammar.symbols.map do |sym| + next unless sym.error_token + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.error_token.lineno} "#{@grammar_file_path}" + {#{sym.error_token.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + # b4_user_actions + def user_actions + action = @context.states.rules.map do |rule| + next unless rule.token_code + + code = rule.token_code + spaces = " " * (code.column - 1) + + <<-STR + case #{rule.id + 1}: /* #{rule.as_comment} */ +#line #{code.line} "#{@grammar_file_path}" +#{spaces}{#{rule.translated_code}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + + action + <<-STR + +#line [@oline@] [@ofile@] + STR + end + + def omit_blanks(param) + param.strip + end + + # b4_parse_param + def parse_param + if @grammar.parse_param + omit_blanks(@grammar.parse_param) + else + "" + end + end + + def lex_param + if @grammar.lex_param + omit_blanks(@grammar.lex_param) + else + "" + end + end + + # b4_user_formals + def user_formals + if @grammar.parse_param + ", #{parse_param}" + else + "" + end + end + + # b4_user_args + def user_args + if @grammar.parse_param + ", #{parse_param_name}" + else + "" + end + end + + def extract_param_name(param) + param[/\b([a-zA-Z0-9_]+)(?=\s*\z)/] + end + + def parse_param_name + if @grammar.parse_param + extract_param_name(parse_param) + else + "" + end + end + + def lex_param_name + if @grammar.lex_param + extract_param_name(lex_param) + else + "" + end + end + + # b4_parse_param_use + def parse_param_use(val, loc) + str = <<-STR.dup + YY_USE (#{val}); + YY_USE (#{loc}); + STR + + if @grammar.parse_param + str << " YY_USE (#{parse_param_name});" + end + + str + end + + # b4_yylex_formals + def yylex_formals + ary = ["&yylval"] + ary << "&yylloc" if @grammar.locations + + if @grammar.lex_param + ary << lex_param_name + end + + "(#{ary.join(', ')})" + end + + # b4_table_value_equals + def table_value_equals(table, value, literal, symbol) + if literal < table.min || table.max < literal + "0" + else + "((#{value}) == #{symbol})" + end + end + + # b4_yyerror_args + def yyerror_args + ary = ["&yylloc"] + + if @grammar.parse_param + ary << parse_param_name + end + + "#{ary.join(', ')}" + end + + def template_basename + File.basename(template_file) + end + + def aux + @grammar.aux + end + + def int_array_to_string(ary) + last = ary.count - 1 + + ary.each_with_index.each_slice(10).map do |slice| + " " + slice.map { |e, i| sprintf("%6d%s", e, (i == last) ? "" : ",") }.join + end.join("\n") + end + + def spec_mapped_header_file + @header_file_path + end + + def b4_cpp_guard__b4_spec_mapped_header_file + if @header_file_path + "YY_YY_" + @header_file_path.gsub(/[^a-zA-Z_0-9]+/, "_").upcase + "_INCLUDED" + else + "" + end + end + + # b4_percent_code_get + def percent_code(name) + @grammar.percent_codes.select do |percent_code| + percent_code.name == name + end.map do |percent_code| + percent_code.code + end.join + end + + private + + def eval_template(file, path) + tmp = ERB.render(file, context: @context, output: self) + replace_special_variables(tmp, path) + end + + def template_file + File.join(template_dir, @template_name) + end + + def header_template_file + File.join(template_dir, "bison/yacc.h") + end + + def partial_file(file) + File.join(template_dir, file) + end + + def template_dir + File.expand_path('../../template', __dir__) + end + + def string_array_to_string(ary) + result = "" + tmp = " " + + ary.each do |s| + replaced = s.gsub('\\', '\\\\\\\\').gsub('"', '\\"') + if (tmp + replaced + " \"\",").length > 75 + result = "#{result}#{tmp}\n" + tmp = " \"#{replaced}\"," + else + tmp = "#{tmp} \"#{replaced}\"," + end + end + + result + tmp + end + + def replace_special_variables(str, ofile) + str.each_line.with_index(1).map do |line, i| + line.gsub!("[@oline@]", (i + 1).to_s) + line.gsub!("[@ofile@]", "\"#{ofile}\"") + line + end.join + end + end +end diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb new file mode 100644 index 0000000000..20c3ad347f --- /dev/null +++ b/tool/lrama/lib/lrama/parser.rb @@ -0,0 +1,2275 @@ +# frozen_string_literal: true +# +# DO NOT MODIFY!!!! +# This file is automatically generated by Racc 1.8.1 +# from Racc grammar file "parser.y". +# + +###### racc/parser.rb begin +unless $".find {|p| p.end_with?('/racc/parser.rb')} +$".push "#{__dir__}/racc/parser.rb" +self.class.module_eval(<<'...end racc/parser.rb/module_eval...', 'racc/parser.rb', 1) +#-- +# Copyright (c) 1999-2006 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the same terms of ruby. +# +# As a special exception, when this code is copied by Racc +# into a Racc output file, you may use that output file +# without restriction. +#++ + +unless $".find {|p| p.end_with?('/racc/info.rb')} +$".push "#{__dir__}/racc/info.rb" + +module Racc + VERSION = '1.8.1' + Version = VERSION + Copyright = 'Copyright (c) 1999-2006 Minero Aoki' +end + +end + + +module Racc + class ParseError < StandardError; end +end +unless defined?(::ParseError) + ParseError = Racc::ParseError # :nodoc: +end + +# Racc is an LALR(1) parser generator. +# It is written in Ruby itself, and generates Ruby programs. +# +# == Command-line Reference +# +# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] +# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] +# [-v] [--verbose] +# [-O<var>filename</var>] [--log-file=<var>filename</var>] +# [-g] [--debug] +# [-E] [--embedded] +# [-l] [--no-line-convert] +# [-c] [--line-convert-all] +# [-a] [--no-omit-actions] +# [-C] [--check-only] +# [-S] [--output-status] +# [--version] [--copyright] [--help] <var>grammarfile</var> +# +# [+grammarfile+] +# Racc grammar file. Any extension is permitted. +# [-o+outfile+, --output-file=+outfile+] +# A filename for output. default is <+filename+>.tab.rb +# [-O+filename+, --log-file=+filename+] +# Place logging output in file +filename+. +# Default log file name is <+filename+>.output. +# [-e+rubypath+, --executable=+rubypath+] +# output executable file(mode 755). where +path+ is the Ruby interpreter. +# [-v, --verbose] +# verbose mode. create +filename+.output file, like yacc's y.output file. +# [-g, --debug] +# add debug code to parser class. To display debugging information, +# use this '-g' option and set @yydebug true in parser class. +# [-E, --embedded] +# Output parser which doesn't need runtime files (racc/parser.rb). +# [-F, --frozen] +# Output parser which declares frozen_string_literals: true +# [-C, --check-only] +# Check syntax of racc grammar file and quit. +# [-S, --output-status] +# Print messages time to time while compiling. +# [-l, --no-line-convert] +# turns off line number converting. +# [-c, --line-convert-all] +# Convert line number of actions, inner, header and footer. +# [-a, --no-omit-actions] +# Call all actions, even if an action is empty. +# [--version] +# print Racc version and quit. +# [--copyright] +# Print copyright and quit. +# [--help] +# Print usage and quit. +# +# == Generating Parser Using Racc +# +# To compile Racc grammar file, simply type: +# +# $ racc parse.y +# +# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. +# +# == Writing A Racc Grammar File +# +# If you want your own parser, you have to write a grammar file. +# A grammar file contains the name of your parser class, grammar for the parser, +# user code, and anything else. +# When writing a grammar file, yacc's knowledge is helpful. +# If you have not used yacc before, Racc is not too difficult. +# +# Here's an example Racc grammar file. +# +# class Calcparser +# rule +# target: exp { print val[0] } +# +# exp: exp '+' exp +# | exp '*' exp +# | '(' exp ')' +# | NUMBER +# end +# +# Racc grammar files resemble yacc files. +# But (of course), this is Ruby code. +# yacc's $$ is the 'result', $0, $1... is +# an array called 'val', and $-1, $-2... is an array called '_values'. +# +# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for +# more information on grammar files. +# +# == Parser +# +# Then you must prepare the parse entry method. There are two types of +# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse +# +# Racc::Parser#do_parse is simple. +# +# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). +# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. +# EOF is [false, false]. +# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. +# If you want to change this, see the grammar reference. +# +# Racc::Parser#yyparse is little complicated, but useful. +# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. +# +# For example, <code>yyparse(obj, :scan)</code> causes +# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. +# +# == Debugging +# +# When debugging, "-v" or/and the "-g" option is helpful. +# +# "-v" creates verbose log file (.output). +# "-g" creates a "Verbose Parser". +# Verbose Parser prints the internal status when parsing. +# But it's _not_ automatic. +# You must use -g option and set +@yydebug+ to +true+ in order to get output. +# -g option only creates the verbose parser. +# +# === Racc reported syntax error. +# +# Isn't there too many "end"? +# grammar of racc file is changed in v0.10. +# +# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. +# +# === Racc reported "XXXX conflicts". +# +# Try "racc -v xxxx.y". +# It causes producing racc's internal log file, xxxx.output. +# +# === Generated parsers does not work correctly +# +# Try "racc -g xxxx.y". +# This command let racc generate "debugging parser". +# Then set @yydebug=true in your parser. +# It produces a working log of your parser. +# +# == Re-distributing Racc runtime +# +# A parser, which is created by Racc, requires the Racc runtime module; +# racc/parser.rb. +# +# Ruby 1.8.x comes with Racc runtime module, +# you need NOT distribute Racc runtime files. +# +# If you want to include the Racc runtime module with your parser. +# This can be done by using '-E' option: +# +# $ racc -E -omyparser.rb myparser.y +# +# This command creates myparser.rb which `includes' Racc runtime. +# Only you must do is to distribute your parser file (myparser.rb). +# +# Note: parser.rb is ruby license, but your parser is not. +# Your own parser is completely yours. +module Racc + + unless defined?(Racc_No_Extensions) + Racc_No_Extensions = false # :nodoc: + end + + class Parser + + Racc_Runtime_Version = ::Racc::VERSION + Racc_Runtime_Core_Version_R = ::Racc::VERSION + + begin + if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' + require 'jruby' + require 'racc/cparse-jruby.jar' + com.headius.racc.Cparse.new.load(JRuby.runtime, false) + else + require 'racc/cparse' + end + + unless new.respond_to?(:_racc_do_parse_c, true) + raise LoadError, 'old cparse.so' + end + if Racc_No_Extensions + raise LoadError, 'selecting ruby version of racc runtime core' + end + + Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: + Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: + Racc_Runtime_Type = 'c' # :nodoc: + rescue LoadError + Racc_Main_Parsing_Routine = :_racc_do_parse_rb + Racc_YY_Parse_Method = :_racc_yyparse_rb + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R + Racc_Runtime_Type = 'ruby' + end + + def Parser.racc_runtime_type # :nodoc: + Racc_Runtime_Type + end + + def _racc_setup + @yydebug = false unless self.class::Racc_debug_parser + @yydebug = false unless defined?(@yydebug) + if @yydebug + @racc_debug_out = $stderr unless defined?(@racc_debug_out) + @racc_debug_out ||= $stderr + end + arg = self.class::Racc_arg + arg[13] = true if arg.size < 14 + arg + end + + def _racc_init_sysvars + @racc_state = [0] + @racc_tstack = [] + @racc_vstack = [] + + @racc_t = nil + @racc_val = nil + + @racc_read_next = true + + @racc_user_yyerror = false + @racc_error_status = 0 + end + + # The entry point of the parser. This method is used with #next_token. + # If Racc wants to get token (and its value), calls next_token. + # + # Example: + # def parse + # @q = [[1,1], + # [2,2], + # [3,3], + # [false, '$']] + # do_parse + # end + # + # def next_token + # @q.shift + # end + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def do_parse + #{Racc_Main_Parsing_Routine}(_racc_setup(), false) + end + RUBY + + # The method to fetch next token. + # If you use #do_parse method, you must implement #next_token. + # + # The format of return value is [TOKEN_SYMBOL, VALUE]. + # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT + # for 'IDENT'. ";" (String) for ';'. + # + # The final symbol (End of file) must be false. + def next_token + raise NotImplementedError, "#{self.class}\#next_token is not defined" + end + + def _racc_do_parse_rb(arg, in_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + tok = act = i = nil + + catch(:racc_end_parse) { + while true + if i = action_pointer[@racc_state[-1]] + if @racc_read_next + if @racc_t != 0 # not EOF + tok, @racc_val = next_token() + unless tok # EOF + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + racc_read_token(@racc_t, tok, @racc_val) if @yydebug + @racc_read_next = false + end + end + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + else + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + } + end + + # Another entry point for the parser. + # If you use this method, you must implement RECEIVER#METHOD_ID method. + # + # RECEIVER#METHOD_ID is a method to get next token. + # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def yyparse(recv, mid) + #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) + end + RUBY + + def _racc_yyparse_rb(recv, mid, arg, c_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + + catch(:racc_end_parse) { + until i = action_pointer[@racc_state[-1]] + while act = _racc_evalact(action_default[@racc_state[-1]], arg) + ; + end + end + recv.__send__(mid) do |tok, val| + unless tok + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + @racc_val = val + @racc_read_next = false + + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + + while !(i = action_pointer[@racc_state[-1]]) || + ! @racc_read_next || + @racc_t == 0 # $ + unless i and i += @racc_t and + i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + end + } + end + + ### + ### common + ### + + def _racc_evalact(act, arg) + action_table, action_check, _, action_pointer, + _, _, _, _, + _, _, _, shift_n, + reduce_n, * = arg + nerr = 0 # tmp + + if act > 0 and act < shift_n + # + # shift + # + if @racc_error_status > 0 + @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF + end + @racc_vstack.push @racc_val + @racc_state.push act + @racc_read_next = true + if @yydebug + @racc_tstack.push @racc_t + racc_shift @racc_t, @racc_tstack, @racc_vstack + end + + elsif act < 0 and act > -reduce_n + # + # reduce + # + code = catch(:racc_jump) { + @racc_state.push _racc_do_reduce(arg, act) + false + } + if code + case code + when 1 # yyerror + @racc_user_yyerror = true # user_yyerror + return -reduce_n + when 2 # yyaccept + return shift_n + else + raise '[Racc Bug] unknown jump code' + end + end + + elsif act == shift_n + # + # accept + # + racc_accept if @yydebug + throw :racc_end_parse, @racc_vstack[0] + + elsif act == -reduce_n + # + # error + # + case @racc_error_status + when 0 + unless arg[21] # user_yyerror + nerr += 1 + on_error @racc_t, @racc_val, @racc_vstack + end + when 3 + if @racc_t == 0 # is $ + # We're at EOF, and another error occurred immediately after + # attempting auto-recovery + throw :racc_end_parse, nil + end + @racc_read_next = true + end + @racc_user_yyerror = false + @racc_error_status = 3 + while true + if i = action_pointer[@racc_state[-1]] + i += 1 # error token + if i >= 0 and + (act = action_table[i]) and + action_check[i] == @racc_state[-1] + break + end + end + throw :racc_end_parse, nil if @racc_state.size <= 1 + @racc_state.pop + @racc_vstack.pop + if @yydebug + @racc_tstack.pop + racc_e_pop @racc_state, @racc_tstack, @racc_vstack + end + end + return act + + else + raise "[Racc Bug] unknown action #{act.inspect}" + end + + racc_next_state(@racc_state[-1], @racc_state) if @yydebug + + nil + end + + def _racc_do_reduce(arg, act) + _, _, _, _, + goto_table, goto_check, goto_default, goto_pointer, + nt_base, reduce_table, _, _, + _, use_result, * = arg + + state = @racc_state + vstack = @racc_vstack + tstack = @racc_tstack + + i = act * -3 + len = reduce_table[i] + reduce_to = reduce_table[i+1] + method_id = reduce_table[i+2] + void_array = [] + + tmp_t = tstack[-len, len] if @yydebug + tmp_v = vstack[-len, len] + tstack[-len, len] = void_array if @yydebug + vstack[-len, len] = void_array + state[-len, len] = void_array + + # tstack must be updated AFTER method call + if use_result + vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) + else + vstack.push __send__(method_id, tmp_v, vstack) + end + tstack.push reduce_to + + racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug + + k1 = reduce_to - nt_base + if i = goto_pointer[k1] + i += state[-1] + if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 + return curstate + end + end + goto_default[k1] + end + + # This method is called when a parse error is found. + # + # ERROR_TOKEN_ID is an internal ID of token which caused error. + # You can get string representation of this ID by calling + # #token_to_str. + # + # ERROR_VALUE is a value of error token. + # + # value_stack is a stack of symbol values. + # DO NOT MODIFY this object. + # + # This method raises ParseError by default. + # + # If this method returns, parsers enter "error recovering mode". + def on_error(t, val, vstack) + raise ParseError, sprintf("parse error on value %s (%s)", + val.inspect, token_to_str(t) || '?') + end + + # Enter error recovering mode. + # This method does not call #on_error. + def yyerror + throw :racc_jump, 1 + end + + # Exit parser. + # Return value is +Symbol_Value_Stack[0]+. + def yyaccept + throw :racc_jump, 2 + end + + # Leave error recovering mode. + def yyerrok + @racc_error_status = 0 + end + + # For debugging output + def racc_read_token(t, tok, val) + @racc_debug_out.print 'read ' + @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' + @racc_debug_out.puts val.inspect + @racc_debug_out.puts + end + + def racc_shift(tok, tstack, vstack) + @racc_debug_out.puts "shift #{racc_token2str tok}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_reduce(toks, sim, tstack, vstack) + out = @racc_debug_out + out.print 'reduce ' + if toks.empty? + out.print ' <none>' + else + toks.each {|t| out.print ' ', racc_token2str(t) } + end + out.puts " --> #{racc_token2str(sim)}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_accept + @racc_debug_out.puts 'accept' + @racc_debug_out.puts + end + + def racc_e_pop(state, tstack, vstack) + @racc_debug_out.puts 'error recovering mode: pop token' + racc_print_states state + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_next_state(curstate, state) + @racc_debug_out.puts "goto #{curstate}" + racc_print_states state + @racc_debug_out.puts + end + + def racc_print_stacks(t, v) + out = @racc_debug_out + out.print ' [' + t.each_index do |i| + out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' + end + out.puts ' ]' + end + + def racc_print_states(s) + out = @racc_debug_out + out.print ' [' + s.each {|st| out.print ' ', st } + out.puts ' ]' + end + + def racc_token2str(tok) + self.class::Racc_token_to_s_table[tok] or + raise "[Racc Bug] can't convert token #{tok} to string" + end + + # Convert internal ID of token symbol to the string. + def token_to_str(t) + self.class::Racc_token_to_s_table[t] + end + + end + +end + +...end racc/parser.rb/module_eval... +end +###### racc/parser.rb end +module Lrama + class Parser < Racc::Parser + +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 504) + +include Lrama::Tracer::Duration + +def initialize(text, path, debug = false, locations = false, define = {}) + @path = path + @grammar_file = Lrama::Lexer::GrammarFile.new(path, text) + @yydebug = debug || define.key?('parse.trace') + @rule_counter = Lrama::Grammar::Counter.new(0) + @midrule_action_counter = Lrama::Grammar::Counter.new(1) + @locations = locations + @define = define +end + +def parse + message = "parse '#{File.basename(@path)}'" + report_duration(message) do + @lexer = Lrama::Lexer.new(@grammar_file) + @grammar = Lrama::Grammar.new(@rule_counter, @locations, @define) + @precedence_number = 0 + reset_precs + do_parse + @grammar + end +end + +def next_token + @lexer.next_token +end + +def on_error(error_token_id, error_value, value_stack) + case error_value + when Lrama::Lexer::Token::Int + location = error_value.location + value = "#{error_value.s_value}" + when Lrama::Lexer::Token::Token + location = error_value.location + value = "\"#{error_value.s_value}\"" + when Lrama::Lexer::Token::Base + location = error_value.location + value = "'#{error_value.s_value}'" + else + location = @lexer.location + value = error_value.inspect + end + + error_message = "parse error on value #{value} (#{token_to_str(error_token_id) || '?'})" + + raise_parse_error(error_message, location) +end + +def on_action_error(error_message, error_value) + if error_value.is_a?(Lrama::Lexer::Token::Base) + location = error_value.location + else + location = @lexer.location + end + + raise_parse_error(error_message, location) +end + +private + +def reset_precs + @opening_prec_seen = false + @trailing_prec_seen = false + @code_after_prec = false +end + +def prec_seen? + @opening_prec_seen || @trailing_prec_seen +end + +def begin_c_declaration(end_symbol) + @lexer.status = :c_declaration + @lexer.end_symbol = end_symbol +end + +def end_c_declaration + @lexer.status = :initial + @lexer.end_symbol = nil +end + +def raise_parse_error(error_message, location) + raise ParseError, location.generate_error_message(error_message) +end +...end parser.y/module_eval... +##### State transition tables begin ### + +racc_action_table = [ + 98, 98, 99, 99, 87, 53, 53, 52, 178, 110, + 110, 97, 53, 53, 184, 178, 110, 110, 53, 181, + 184, 162, 110, 6, 163, 181, 181, 53, 53, 52, + 52, 181, 79, 79, 53, 53, 52, 52, 43, 79, + 79, 53, 4, 52, 5, 110, 88, 94, 182, 125, + 126, 163, 100, 100, 180, 193, 194, 195, 137, 185, + 188, 180, 4, 44, 5, 185, 188, 94, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 46, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 47, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 47, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 50, + 57, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 57, + 60, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 53, 53, 52, + 52, 110, 105, 53, 53, 52, 52, 110, 105, 53, + 53, 52, 52, 110, 105, 53, 53, 52, 52, 110, + 105, 53, 53, 52, 52, 110, 110, 53, 53, 52, + 209, 110, 110, 53, 53, 209, 52, 110, 110, 53, + 53, 209, 52, 110, 193, 194, 195, 137, 216, 222, + 229, 217, 217, 217, 53, 53, 52, 52, 193, 194, + 195, 57, 57, 57, 57, 66, 67, 68, 69, 70, + 72, 72, 72, 86, 89, 47, 57, 57, 113, 117, + 117, 79, 123, 124, 131, 47, 133, 137, 139, 143, + 149, 150, 151, 152, 133, 155, 156, 157, 110, 166, + 149, 169, 172, 173, 72, 175, 176, 183, 189, 166, + 196, 137, 200, 202, 137, 166, 211, 166, 137, 72, + 176, 218, 176, 72, 72, 227, 137, 72 ] + +racc_action_check = [ + 51, 97, 51, 97, 41, 75, 165, 75, 165, 75, + 165, 51, 171, 190, 171, 190, 171, 190, 201, 165, + 201, 148, 201, 1, 148, 171, 190, 36, 37, 36, + 37, 201, 36, 37, 38, 39, 38, 39, 5, 38, + 39, 117, 0, 117, 0, 117, 41, 46, 168, 88, + 88, 168, 51, 97, 165, 177, 177, 177, 177, 171, + 171, 190, 2, 6, 2, 201, 201, 90, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 9, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 10, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 11, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 3, 3, 12, + 14, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 15, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 8, 8, 16, + 17, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 18, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 63, 13, 63, + 13, 63, 63, 64, 73, 64, 73, 64, 64, 65, + 78, 65, 78, 65, 65, 106, 79, 106, 79, 106, + 106, 118, 180, 118, 180, 118, 180, 188, 196, 188, + 196, 188, 196, 202, 217, 202, 217, 202, 217, 218, + 113, 218, 113, 218, 186, 186, 186, 186, 208, 213, + 226, 208, 213, 226, 114, 123, 114, 123, 210, 210, + 210, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 40, 42, 47, 55, 60, 71, 74, + 76, 80, 81, 87, 91, 92, 93, 94, 102, 116, + 124, 125, 126, 127, 133, 136, 137, 138, 144, 150, + 151, 153, 156, 158, 162, 163, 164, 170, 174, 176, + 178, 179, 182, 184, 187, 189, 199, 200, 204, 205, + 207, 209, 212, 214, 216, 221, 222, 228 ] + +racc_action_pointer = [ + 32, 23, 52, 93, nil, 31, 63, nil, 123, 68, + 74, 84, 103, 165, 94, 111, 123, 135, 141, nil, + nil, nil, nil, nil, 215, 216, 217, 218, 230, 231, + 232, 233, 234, 232, 233, 234, 24, 25, 31, 32, + 238, -1, 242, nil, nil, nil, 43, 232, nil, nil, + nil, -5, nil, nil, nil, 230, nil, nil, nil, nil, + 231, nil, nil, 164, 170, 176, nil, nil, nil, nil, + nil, 240, nil, 171, 241, 2, 242, nil, 177, 183, + 243, 244, nil, nil, nil, nil, nil, 209, 45, nil, + 63, 245, 242, 243, 202, nil, nil, -4, nil, nil, + nil, nil, 256, nil, nil, nil, 182, nil, nil, nil, + nil, nil, nil, 207, 221, nil, 253, 38, 188, nil, + nil, nil, nil, 222, 255, 215, 218, 252, nil, nil, + nil, nil, nil, 251, nil, nil, 219, 261, 250, nil, + nil, nil, nil, nil, 261, nil, nil, nil, -24, nil, + 219, 265, nil, 269, nil, nil, 216, nil, 256, nil, + nil, nil, 266, 270, 227, 3, nil, nil, 3, nil, + 228, 9, nil, nil, 232, nil, 229, 3, 236, 226, + 189, nil, 236, nil, 239, nil, 162, 229, 194, 235, + 10, nil, nil, nil, nil, nil, 195, nil, nil, 284, + 237, 15, 200, nil, 233, 281, nil, 241, 173, 247, + 176, nil, 243, 174, 285, nil, 286, 201, 206, nil, + nil, 278, 241, nil, nil, nil, 175, nil, 289, nil, + nil ] + +racc_action_default = [ + -1, -136, -1, -3, -10, -136, -136, -2, -3, -136, + -14, -14, -136, -136, -136, -136, -136, -136, -136, -28, + -29, -34, -35, -36, -136, -136, -136, -136, -136, -136, + -136, -136, -136, -54, -54, -54, -136, -136, -136, -136, + -136, -136, -136, -13, 231, -4, -136, -14, -16, -17, + -20, -131, -100, -101, -130, -18, -23, -89, -24, -25, + -136, -27, -37, -136, -136, -136, -41, -42, -43, -44, + -45, -46, -55, -136, -47, -136, -48, -49, -92, -136, + -95, -97, -98, -50, -51, -52, -53, -136, -136, -11, + -5, -7, -14, -136, -72, -15, -21, -131, -132, -133, + -134, -19, -136, -26, -30, -31, -32, -38, -87, -88, + -135, -39, -40, -136, -56, -58, -60, -136, -83, -85, + -93, -94, -96, -136, -136, -136, -136, -136, -6, -8, + -9, -128, -104, -102, -105, -73, -136, -136, -136, -90, + -33, -59, -57, -61, -80, -86, -84, -99, -136, -66, + -70, -136, -12, -136, -103, -109, -136, -22, -136, -62, + -81, -82, -54, -136, -64, -68, -71, -74, -136, -129, + -106, -107, -127, -91, -136, -67, -70, -72, -100, -72, + -136, -124, -136, -109, -100, -110, -72, -72, -136, -70, + -69, -75, -76, -116, -117, -118, -136, -78, -79, -136, + -70, -108, -136, -111, -72, -54, -115, -63, -136, -100, + -119, -125, -65, -136, -54, -114, -54, -136, -136, -120, + -121, -136, -72, -112, -77, -122, -136, -126, -54, -123, + -113 ] + +racc_goto_table = [ + 73, 118, 136, 54, 48, 49, 164, 96, 91, 120, + 121, 93, 187, 148, 107, 111, 112, 119, 134, 171, + 56, 58, 59, 3, 61, 7, 78, 78, 78, 78, + 62, 63, 64, 65, 115, 74, 76, 192, 1, 129, + 168, 95, 187, 118, 118, 207, 204, 201, 77, 83, + 84, 85, 128, 138, 147, 93, 212, 140, 154, 145, + 146, 101, 130, 116, 42, 127, 103, 208, 78, 78, + 219, 9, 51, 213, 141, 142, 45, 71, 159, 144, + 190, 160, 161, 102, 158, 191, 132, 197, 122, 226, + 170, 177, 220, 199, 203, 205, 221, 186, 153, nil, + nil, nil, nil, 116, 116, nil, 198, nil, nil, nil, + nil, nil, 214, 78, 206, nil, 177, nil, nil, nil, + nil, nil, 210, nil, nil, nil, nil, 186, 210, 174, + 228, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 225, 210, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 215, nil, nil, nil, nil, nil, nil, nil, + nil, 223, nil, 224, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 230 ] + +racc_goto_check = [ + 29, 22, 42, 31, 14, 14, 35, 16, 8, 48, + 48, 13, 40, 34, 24, 24, 24, 45, 52, 54, + 18, 18, 18, 6, 17, 6, 31, 31, 31, 31, + 17, 17, 17, 17, 30, 26, 26, 38, 1, 5, + 34, 14, 40, 22, 22, 35, 38, 54, 27, 27, + 27, 27, 8, 16, 48, 13, 35, 24, 52, 45, + 45, 18, 9, 31, 10, 11, 17, 39, 31, 31, + 38, 7, 15, 39, 30, 30, 7, 25, 32, 33, + 36, 43, 44, 46, 47, 42, 14, 42, 50, 39, + 53, 22, 55, 56, 42, 42, 57, 22, 58, nil, + nil, nil, nil, 31, 31, nil, 22, nil, nil, nil, + nil, nil, 42, 31, 22, nil, 22, nil, nil, nil, + nil, nil, 22, nil, nil, nil, nil, 22, 22, 29, + 42, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 22, 22, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 29, nil, nil, nil, nil, nil, nil, nil, + nil, 29, nil, 29, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 29 ] + +racc_goto_pointer = [ + nil, 38, nil, nil, nil, -52, 23, 68, -38, -29, + 60, -24, nil, -35, -6, 59, -44, 6, 6, nil, + nil, nil, -74, nil, -49, 44, 1, 12, nil, -33, + -39, -10, -66, -37, -111, -144, -96, nil, -140, -129, + -159, nil, -92, -63, -62, -58, 26, -55, -69, nil, + 8, nil, -75, -65, -136, -118, -88, -115, -33 ] + +racc_goto_default = [ + nil, nil, 2, 8, 90, nil, nil, nil, nil, nil, + nil, nil, 10, 11, nil, nil, nil, 55, nil, 21, + 22, 23, 104, 106, nil, nil, nil, nil, 114, 75, + nil, 108, nil, nil, nil, nil, 165, 135, nil, nil, + 179, 167, nil, 109, nil, nil, nil, nil, 81, 80, + 82, 92, nil, nil, nil, nil, nil, nil, nil ] + +racc_reduce_table = [ + 0, 0, :racc_error, + 0, 64, :_reduce_1, + 2, 64, :_reduce_2, + 0, 65, :_reduce_3, + 2, 65, :_reduce_4, + 1, 66, :_reduce_5, + 2, 66, :_reduce_6, + 0, 67, :_reduce_none, + 1, 67, :_reduce_none, + 5, 59, :_reduce_none, + 0, 68, :_reduce_10, + 0, 69, :_reduce_11, + 5, 60, :_reduce_12, + 2, 60, :_reduce_13, + 0, 72, :_reduce_14, + 2, 72, :_reduce_15, + 2, 61, :_reduce_none, + 2, 61, :_reduce_none, + 1, 76, :_reduce_18, + 2, 76, :_reduce_19, + 2, 70, :_reduce_20, + 3, 70, :_reduce_21, + 5, 70, :_reduce_22, + 2, 70, :_reduce_none, + 2, 70, :_reduce_24, + 2, 70, :_reduce_25, + 3, 70, :_reduce_26, + 2, 70, :_reduce_27, + 1, 70, :_reduce_28, + 1, 70, :_reduce_29, + 1, 81, :_reduce_30, + 1, 81, :_reduce_31, + 1, 82, :_reduce_32, + 2, 82, :_reduce_33, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 2, 71, :_reduce_37, + 3, 71, :_reduce_38, + 3, 71, :_reduce_39, + 3, 71, :_reduce_40, + 2, 71, :_reduce_41, + 2, 71, :_reduce_42, + 2, 71, :_reduce_43, + 2, 71, :_reduce_44, + 2, 71, :_reduce_45, + 2, 77, :_reduce_none, + 2, 77, :_reduce_47, + 2, 77, :_reduce_48, + 2, 77, :_reduce_49, + 2, 77, :_reduce_50, + 2, 77, :_reduce_51, + 2, 77, :_reduce_52, + 2, 77, :_reduce_53, + 0, 87, :_reduce_none, + 1, 87, :_reduce_none, + 1, 88, :_reduce_56, + 2, 88, :_reduce_57, + 2, 83, :_reduce_58, + 3, 83, :_reduce_59, + 0, 91, :_reduce_none, + 1, 91, :_reduce_none, + 3, 86, :_reduce_62, + 8, 78, :_reduce_63, + 5, 79, :_reduce_64, + 8, 79, :_reduce_65, + 1, 92, :_reduce_66, + 3, 92, :_reduce_67, + 1, 93, :_reduce_68, + 3, 93, :_reduce_69, + 0, 99, :_reduce_none, + 1, 99, :_reduce_none, + 0, 100, :_reduce_none, + 1, 100, :_reduce_none, + 1, 94, :_reduce_74, + 3, 94, :_reduce_75, + 3, 94, :_reduce_76, + 6, 94, :_reduce_77, + 3, 94, :_reduce_78, + 3, 94, :_reduce_79, + 0, 102, :_reduce_none, + 1, 102, :_reduce_none, + 1, 90, :_reduce_82, + 1, 103, :_reduce_83, + 2, 103, :_reduce_84, + 2, 84, :_reduce_85, + 3, 84, :_reduce_86, + 1, 80, :_reduce_none, + 1, 80, :_reduce_none, + 0, 104, :_reduce_89, + 0, 105, :_reduce_90, + 5, 75, :_reduce_91, + 1, 106, :_reduce_92, + 2, 106, :_reduce_93, + 2, 107, :_reduce_94, + 1, 108, :_reduce_95, + 2, 108, :_reduce_96, + 1, 85, :_reduce_97, + 1, 85, :_reduce_98, + 3, 85, :_reduce_99, + 1, 89, :_reduce_none, + 1, 89, :_reduce_none, + 1, 110, :_reduce_102, + 2, 110, :_reduce_103, + 2, 62, :_reduce_none, + 2, 62, :_reduce_none, + 4, 109, :_reduce_106, + 1, 111, :_reduce_107, + 3, 111, :_reduce_108, + 0, 112, :_reduce_109, + 2, 112, :_reduce_110, + 3, 112, :_reduce_111, + 5, 112, :_reduce_112, + 7, 112, :_reduce_113, + 4, 112, :_reduce_114, + 3, 112, :_reduce_115, + 1, 96, :_reduce_116, + 1, 96, :_reduce_117, + 1, 96, :_reduce_118, + 0, 113, :_reduce_none, + 1, 113, :_reduce_none, + 2, 97, :_reduce_121, + 3, 97, :_reduce_122, + 4, 97, :_reduce_123, + 0, 114, :_reduce_124, + 0, 115, :_reduce_125, + 5, 98, :_reduce_126, + 3, 95, :_reduce_127, + 0, 116, :_reduce_128, + 3, 63, :_reduce_129, + 1, 73, :_reduce_none, + 0, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 101, :_reduce_135 ] + +racc_reduce_n = 136 + +racc_shift_n = 231 + +racc_token_table = { + false => 0, + :error => 1, + :C_DECLARATION => 2, + :CHARACTER => 3, + :IDENT_COLON => 4, + :IDENTIFIER => 5, + :INTEGER => 6, + :STRING => 7, + :TAG => 8, + "%%" => 9, + "%{" => 10, + "%}" => 11, + "%require" => 12, + ";" => 13, + "%expect" => 14, + "%define" => 15, + "{" => 16, + "}" => 17, + "%param" => 18, + "%lex-param" => 19, + "%parse-param" => 20, + "%code" => 21, + "%initial-action" => 22, + "%no-stdlib" => 23, + "%locations" => 24, + "%union" => 25, + "%destructor" => 26, + "%printer" => 27, + "%error-token" => 28, + "%after-shift" => 29, + "%before-reduce" => 30, + "%after-reduce" => 31, + "%after-shift-error-token" => 32, + "%after-pop-stack" => 33, + "-temp-group" => 34, + "%token" => 35, + "%type" => 36, + "%nterm" => 37, + "%left" => 38, + "%right" => 39, + "%precedence" => 40, + "%nonassoc" => 41, + "%start" => 42, + "%rule" => 43, + "(" => 44, + ")" => 45, + ":" => 46, + "%inline" => 47, + "," => 48, + "|" => 49, + "%empty" => 50, + "%prec" => 51, + "?" => 52, + "+" => 53, + "*" => 54, + "[" => 55, + "]" => 56, + "{...}" => 57 } + +racc_nt_base = 58 + +racc_use_result_var = true + +Racc_arg = [ + racc_action_table, + racc_action_check, + racc_action_default, + racc_action_pointer, + racc_goto_table, + racc_goto_check, + racc_goto_default, + racc_goto_pointer, + racc_nt_base, + racc_reduce_table, + racc_token_table, + racc_shift_n, + racc_reduce_n, + racc_use_result_var ] +Ractor.make_shareable(Racc_arg) if defined?(Ractor) + +Racc_token_to_s_table = [ + "$end", + "error", + "C_DECLARATION", + "CHARACTER", + "IDENT_COLON", + "IDENTIFIER", + "INTEGER", + "STRING", + "TAG", + "\"%%\"", + "\"%{\"", + "\"%}\"", + "\"%require\"", + "\";\"", + "\"%expect\"", + "\"%define\"", + "\"{\"", + "\"}\"", + "\"%param\"", + "\"%lex-param\"", + "\"%parse-param\"", + "\"%code\"", + "\"%initial-action\"", + "\"%no-stdlib\"", + "\"%locations\"", + "\"%union\"", + "\"%destructor\"", + "\"%printer\"", + "\"%error-token\"", + "\"%after-shift\"", + "\"%before-reduce\"", + "\"%after-reduce\"", + "\"%after-shift-error-token\"", + "\"%after-pop-stack\"", + "\"-temp-group\"", + "\"%token\"", + "\"%type\"", + "\"%nterm\"", + "\"%left\"", + "\"%right\"", + "\"%precedence\"", + "\"%nonassoc\"", + "\"%start\"", + "\"%rule\"", + "\"(\"", + "\")\"", + "\":\"", + "\"%inline\"", + "\",\"", + "\"|\"", + "\"%empty\"", + "\"%prec\"", + "\"?\"", + "\"+\"", + "\"*\"", + "\"[\"", + "\"]\"", + "\"{...}\"", + "$start", + "input", + "prologue_declaration", + "bison_declaration", + "rules_or_grammar_declaration", + "epilogue_declaration", + "\"-many@prologue_declaration\"", + "\"-many@bison_declaration\"", + "\"-many1@rules_or_grammar_declaration\"", + "\"-option@epilogue_declaration\"", + "@1", + "@2", + "parser_option", + "grammar_declaration", + "\"-many@;\"", + "variable", + "value", + "param", + "\"-many1@param\"", + "symbol_declaration", + "rule_declaration", + "inline_declaration", + "symbol", + "\"-group@symbol|TAG\"", + "\"-many1@-group@symbol|TAG\"", + "token_declarations", + "symbol_declarations", + "token_declarations_for_precedence", + "token_declaration", + "\"-option@TAG\"", + "\"-many1@token_declaration\"", + "id", + "alias", + "\"-option@INTEGER\"", + "rule_args", + "rule_rhs_list", + "rule_rhs", + "named_ref", + "parameterized_suffix", + "parameterized_args", + "action", + "\"-option@%empty\"", + "\"-option@named_ref\"", + "string_as_id", + "\"-option@string_as_id\"", + "\"-many1@symbol\"", + "@3", + "@4", + "\"-many1@id\"", + "\"-group@TAG-\\\"-many1@id\\\"\"", + "\"-many1@-group@TAG-\\\"-many1@id\\\"\"", + "rules", + "\"-many1@;\"", + "rhs_list", + "rhs", + "\"-option@parameterized_suffix\"", + "@5", + "@6", + "@7" ] +Ractor.make_shareable(Racc_token_to_s_table) if defined?(Ractor) + +Racc_debug_parser = true + +##### State transition tables end ##### + +# reduce 0 omitted + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_1(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_2(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_3(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_4(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_5(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_6(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 7 omitted + +# reduce 8 omitted + +# reduce 9 omitted + +module_eval(<<'.,.,', 'parser.y', 13) + def _reduce_10(val, _values, result) + begin_c_declaration("%}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 17) + def _reduce_11(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 21) + def _reduce_12(val, _values, result) + @grammar.prologue_first_lineno = val[0].first_line + @grammar.prologue = val[2].s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 26) + def _reduce_13(val, _values, result) + @grammar.required = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_14(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_15(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 16 omitted + +# reduce 17 omitted + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_18(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_19(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 36) + def _reduce_20(val, _values, result) + @grammar.expect = val[1].s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 40) + def _reduce_21(val, _values, result) + @grammar.define[val[1].s_value] = val[2]&.s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 44) + def _reduce_22(val, _values, result) + @grammar.define[val[1].s_value] = val[3]&.s_value + + result + end +.,., + +# reduce 23 omitted + +module_eval(<<'.,.,', 'parser.y', 49) + def _reduce_24(val, _values, result) + val[1].each {|token| + @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 55) + def _reduce_25(val, _values, result) + val[1].each {|token| + @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 61) + def _reduce_26(val, _values, result) + @grammar.add_percent_code(id: val[1], code: val[2]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 65) + def _reduce_27(val, _values, result) + @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 69) + def _reduce_28(val, _values, result) + @grammar.no_stdlib = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 73) + def _reduce_29(val, _values, result) + @grammar.locations = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_30(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_31(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_32(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_33(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 34 omitted + +# reduce 35 omitted + +# reduce 36 omitted + +module_eval(<<'.,.,', 'parser.y', 82) + def _reduce_37(val, _values, result) + @grammar.set_union( + Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), + val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 89) + def _reduce_38(val, _values, result) + @grammar.add_destructor( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 97) + def _reduce_39(val, _values, result) + @grammar.add_printer( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 105) + def _reduce_40(val, _values, result) + @grammar.add_error_token( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 113) + def _reduce_41(val, _values, result) + @grammar.after_shift = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 117) + def _reduce_42(val, _values, result) + @grammar.before_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 121) + def _reduce_43(val, _values, result) + @grammar.after_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 125) + def _reduce_44(val, _values, result) + @grammar.after_shift_error_token = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 129) + def _reduce_45(val, _values, result) + @grammar.after_pop_stack = val[1] + + result + end +.,., + +# reduce 46 omitted + +module_eval(<<'.,.,', 'parser.y', 136) + def _reduce_47(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + @grammar.add_type(id: id, tag: hash[:tag]) + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 144) + def _reduce_48(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + if @grammar.find_term_by_s_value(id.s_value) + on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) + else + @grammar.add_type(id: id, tag: hash[:tag]) + end + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 156) + def _reduce_49(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_left(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 166) + def _reduce_50(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_right(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 176) + def _reduce_51(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_precedence(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 186) + def _reduce_52(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_nonassoc(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 196) + def _reduce_53(val, _values, result) + @grammar.set_start_nterm(val[1]) + + result + end +.,., + +# reduce 54 omitted + +# reduce 55 omitted + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_56(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_57(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 202) + def _reduce_58(val, _values, result) + val[1].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[0], replace: true) + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 208) + def _reduce_59(val, _values, result) + val[2].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[1], replace: true) + } + + result + end +.,., + +# reduce 60 omitted + +# reduce 61 omitted + +module_eval(<<'.,.,', 'parser.y', 213) + def _reduce_62(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 218) + def _reduce_63(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 225) + def _reduce_64(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, [], val[4], is_inline: true) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 230) + def _reduce_65(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 235) + def _reduce_66(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 236) + def _reduce_67(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 241) + def _reduce_68(val, _values, result) + builder = val[0] + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 246) + def _reduce_69(val, _values, result) + builder = val[2] + result = val[0].append(builder) + + result + end +.,., + +# reduce 70 omitted + +# reduce 71 omitted + +# reduce 72 omitted + +# reduce 73 omitted + +module_eval(<<'.,.,', 'parser.y', 253) + def _reduce_74(val, _values, result) + reset_precs + result = Grammar::Parameterized::Rhs.new + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 258) + def _reduce_75(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.symbols << token + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 267) + def _reduce_76(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 274) + def _reduce_77(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 281) + def _reduce_78(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 289) + def _reduce_79(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +# reduce 80 omitted + +# reduce 81 omitted + +module_eval(<<'.,.,', 'parser.y', 301) + def _reduce_82(val, _values, result) + result = val[0].s_value if val[0] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_83(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_84(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 306) + def _reduce_85(val, _values, result) + result = if val[0] + [{tag: val[0], tokens: val[1]}] + else + [{tag: nil, tokens: val[1]}] + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 312) + def _reduce_86(val, _values, result) + result = val[0].append({tag: val[1], tokens: val[2]}) + result + end +.,., + +# reduce 87 omitted + +# reduce 88 omitted + +module_eval(<<'.,.,', 'parser.y', 321) + def _reduce_89(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 325) + def _reduce_90(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 329) + def _reduce_91(val, _values, result) + result = val[2] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_92(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_93(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_94(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_95(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_96(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 333) + def _reduce_97(val, _values, result) + result = [{tag: nil, tokens: val[0]}] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 334) + def _reduce_98(val, _values, result) + result = val[0].map {|tag, ids| {tag: tag, tokens: ids} } + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 335) + def _reduce_99(val, _values, result) + result = [{tag: nil, tokens: val[0]}, {tag: val[1], tokens: val[2]}] + result + end +.,., + +# reduce 100 omitted + +# reduce 101 omitted + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_102(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_103(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 104 omitted + +# reduce 105 omitted + +module_eval(<<'.,.,', 'parser.y', 348) + def _reduce_106(val, _values, result) + lhs = val[0] + lhs.alias_name = val[1] + val[3].each do |builder| + builder.lhs = lhs + builder.complete_input + @grammar.add_rule_builder(builder) + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 360) + def _reduce_107(val, _values, result) + if val[0].rhs.count > 1 + empties = val[0].rhs.select { |sym| sym.is_a?(Lrama::Lexer::Token::Empty) } + empties.each do |empty| + on_action_error("%empty on non-empty rule", empty) + end + end + builder = val[0] + if !builder.line + builder.line = @lexer.line - 1 + end + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 374) + def _reduce_108(val, _values, result) + builder = val[2] + if !builder.line + builder.line = @lexer.line - 1 + end + result = val[0].append(builder) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 384) + def _reduce_109(val, _values, result) + reset_precs + result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 389) + def _reduce_110(val, _values, result) + builder = val[0] + builder.add_rhs(Lrama::Lexer::Token::Empty.new(location: @lexer.location)) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 395) + def _reduce_111(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.add_rhs(token) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 404) + def _reduce_112(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 413) + def _reduce_113(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 422) + def _reduce_114(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + user_code.tag = val[3] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 431) + def _reduce_115(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 444) + def _reduce_116(val, _values, result) + result = "option" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 445) + def _reduce_117(val, _values, result) + result = "nonempty_list" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 446) + def _reduce_118(val, _values, result) + result = "list" + result + end +.,., + +# reduce 119 omitted + +# reduce 120 omitted + +module_eval(<<'.,.,', 'parser.y', 451) + def _reduce_121(val, _values, result) + result = if val[1] + [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] + else + [val[0]] + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 457) + def _reduce_122(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 458) + def _reduce_123(val, _values, result) + result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 463) + def _reduce_124(val, _values, result) + if prec_seen? + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 471) + def _reduce_125(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 475) + def _reduce_126(val, _values, result) + result = val[2] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 478) + def _reduce_127(val, _values, result) + result = val[1].s_value + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 483) + def _reduce_128(val, _values, result) + begin_c_declaration('\Z') + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 487) + def _reduce_129(val, _values, result) + end_c_declaration + @grammar.epilogue_first_lineno = val[0].first_line + 1 + @grammar.epilogue = val[2].s_value + + result + end +.,., + +# reduce 130 omitted + +# reduce 131 omitted + +# reduce 132 omitted + +# reduce 133 omitted + +# reduce 134 omitted + +module_eval(<<'.,.,', 'parser.y', 499) + def _reduce_135(val, _values, result) + result = Lrama::Lexer::Token::Ident.new(s_value: val[0].s_value) + result + end +.,., + +def _reduce_none(val, _values, result) + val[0] +end + + end # class Parser +end # module Lrama diff --git a/tool/lrama/lib/lrama/reporter.rb b/tool/lrama/lib/lrama/reporter.rb new file mode 100644 index 0000000000..ed25cc7f8f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'reporter/conflicts' +require_relative 'reporter/grammar' +require_relative 'reporter/precedences' +require_relative 'reporter/profile' +require_relative 'reporter/rules' +require_relative 'reporter/states' +require_relative 'reporter/terms' + +module Lrama + class Reporter + include Lrama::Tracer::Duration + + # @rbs (**bool options) -> void + def initialize(**options) + @options = options + @rules = Rules.new(**options) + @terms = Terms.new(**options) + @conflicts = Conflicts.new + @precedences = Precedences.new + @grammar = Grammar.new(**options) + @states = States.new(**options) + end + + # @rbs (File io, Lrama::States states) -> void + def report(io, states) + report_duration(:report) do + report_duration(:report_rules) { @rules.report(io, states) } + report_duration(:report_terms) { @terms.report(io, states) } + report_duration(:report_conflicts) { @conflicts.report(io, states) } + report_duration(:report_precedences) { @precedences.report(io, states) } + report_duration(:report_grammar) { @grammar.report(io, states) } + report_duration(:report_states) { @states.report(io, states, ielr: states.ielr_defined?) } + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/conflicts.rb b/tool/lrama/lib/lrama/reporter/conflicts.rb new file mode 100644 index 0000000000..f4d8c604c9 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/conflicts.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Conflicts + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_conflicts(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_conflicts(io, states) + has_conflict = false + + states.states.each do |state| + messages = format_conflict_messages(state.conflicts) + + unless messages.empty? + has_conflict = true + io << "State #{state.id} conflicts: #{messages.join(', ')}\n" + end + end + + io << "\n\n" if has_conflict + end + + # @rbs (Array[(Lrama::State::ShiftReduceConflict | Lrama::State::ReduceReduceConflict)] conflicts) -> Array[String] + def format_conflict_messages(conflicts) + conflict_types = { + shift_reduce: "shift/reduce", + reduce_reduce: "reduce/reduce" + } + + conflict_types.keys.map do |type| + type_conflicts = conflicts.select { |c| c.type == type } + "#{type_conflicts.count} #{conflict_types[type]}" unless type_conflicts.empty? + end.compact + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/grammar.rb b/tool/lrama/lib/lrama/reporter/grammar.rb new file mode 100644 index 0000000000..dc3f3f6bfd --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/grammar.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Grammar + # @rbs (?grammar: bool, **bool _) -> void + def initialize(grammar: false, **_) + @grammar = grammar + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @grammar + + io << "Grammar\n" + last_lhs = nil + + states.rules.each do |rule| + if rule.empty_rule? + r = "ε" + else + r = rule.rhs.map(&:display_name).join(" ") + end + + if rule.lhs == last_lhs + io << sprintf("%5d %s| %s", rule.id, " " * rule.lhs.display_name.length, r) << "\n" + else + io << "\n" + io << sprintf("%5d %s: %s", rule.id, rule.lhs.display_name, r) << "\n" + end + + last_lhs = rule.lhs + end + io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/precedences.rb b/tool/lrama/lib/lrama/reporter/precedences.rb new file mode 100644 index 0000000000..73c0888700 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/precedences.rb @@ -0,0 +1,54 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Precedences + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_precedences(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_precedences(io, states) + used_precedences = states.precedences.select(&:used_by?) + + return if used_precedences.empty? + + io << "Precedences\n\n" + + used_precedences.each do |precedence| + io << " precedence on #{precedence.symbol.display_name} is used to resolve conflict on\n" + + if precedence.used_by_lalr? + io << " LALR\n" + + precedence.used_by_lalr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + + if precedence.used_by_ielr? + io << " IELR\n" + + precedence.used_by_ielr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile.rb b/tool/lrama/lib/lrama/reporter/profile.rb new file mode 100644 index 0000000000..b569b94d4f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'profile/call_stack' +require_relative 'profile/memory' diff --git a/tool/lrama/lib/lrama/reporter/profile/call_stack.rb b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb new file mode 100644 index 0000000000..8a4d44b61c --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module CallStack + # See "Call-stack Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_stackprof + ex = nil #: Exception? + path = 'tmp/stackprof-cpu-myapp.dump' + + StackProf.run(mode: :cpu, raw: true, out: path) do + yield + rescue Exception => e + ex = e + end + + STDERR.puts("Call-stack Profiling result is generated on #{path}") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_stackprof + require "stackprof" + true + rescue LoadError + warn "stackprof is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile/memory.rb b/tool/lrama/lib/lrama/reporter/profile/memory.rb new file mode 100644 index 0000000000..a019581fdf --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/memory.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module Memory + # See "Memory Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_memory_profiler + ex = nil #: Exception? + + report = MemoryProfiler.report do # steep:ignore UnknownConstant + yield + rescue Exception => e + ex = e + end + + report.pretty_print(to_file: "tmp/memory_profiler.txt") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_memory_profiler + require "memory_profiler" + true + rescue LoadError + warn "memory_profiler is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/rules.rb b/tool/lrama/lib/lrama/reporter/rules.rb new file mode 100644 index 0000000000..3e8bf19a0a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/rules.rb @@ -0,0 +1,43 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Rules + # @rbs (?rules: bool, **bool _) -> void + def initialize(rules: false, **_) + @rules = rules + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @rules + + used_rules = states.rules.flat_map(&:rhs) + + unless used_rules.empty? + io << "Rule Usage Frequency\n\n" + frequency_counts = used_rules.each_with_object(Hash.new(0)) { |rule, counts| counts[rule] += 1 } + + frequency_counts + .select { |rule,| !rule.midrule? } + .sort_by { |rule, count| [-count, rule.name] } + .each_with_index { |(rule, count), i| io << sprintf("%5d %s (%d times)", i, rule.name, count) << "\n" } + io << "\n\n" + end + + unused_rules = states.rules.map(&:lhs).select do |rule| + !used_rules.include?(rule) && rule.token_id != 0 + end + + unless unused_rules.empty? + io << "#{unused_rules.count} Unused Rules\n\n" + unused_rules.each_with_index do |rule, index| + io << sprintf("%5d %s", index, rule.display_name) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/states.rb b/tool/lrama/lib/lrama/reporter/states.rb new file mode 100644 index 0000000000..d152d0511a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/states.rb @@ -0,0 +1,387 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class States + # @rbs (?itemsets: bool, ?lookaheads: bool, ?solved: bool, ?counterexamples: bool, ?verbose: bool, **bool _) -> void + def initialize(itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false, **_) + @itemsets = itemsets + @lookaheads = lookaheads + @solved = solved + @counterexamples = counterexamples + @verbose = verbose + end + + # @rbs (IO io, Lrama::States states, ielr: bool) -> void + def report(io, states, ielr: false) + cex = Counterexamples.new(states) if @counterexamples + + states.compute_la_sources_for_conflicted_states + report_split_states(io, states.states) if ielr + + states.states.each do |state| + report_state_header(io, state) + report_items(io, state) + report_conflicts(io, state) + report_shifts(io, state) + report_nonassoc_errors(io, state) + report_reduces(io, state) + report_nterm_transitions(io, state) + report_conflict_resolutions(io, state) if @solved + report_counterexamples(io, state, cex) if @counterexamples && state.has_conflicts? # @type var cex: Lrama::Counterexamples + report_verbose_info(io, state, states) if @verbose + # End of Report State + io << "\n" + end + end + + private + + # @rbs (IO io, Array[Lrama::State] states) -> void + def report_split_states(io, states) + ss = states.select(&:split_state?) + + return if ss.empty? + + io << "Split States\n\n" + + ss.each do |state| + io << " State #{state.id} is split from state #{state.lalr_isocore.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_state_header(io, state) + io << "State #{state.id}\n\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_items(io, state) + last_lhs = nil + list = @itemsets ? state.items : state.kernels + + list.sort_by {|i| [i.rule_id, i.position] }.each do |item| + r = item.empty_rule? ? "ε •" : item.rhs.map(&:display_name).insert(item.position, "•").join(" ") + + l = if item.lhs == last_lhs + " " * item.lhs.id.s_value.length + "|" + else + item.lhs.id.s_value + ":" + end + + la = "" + if @lookaheads && item.end_of_rule? + reduce = state.find_reduce_by_item!(item) + look_ahead = reduce.selected_look_ahead + unless look_ahead.empty? + la = " [#{look_ahead.compact.map(&:display_name).join(", ")}]" + end + end + + last_lhs = item.lhs + io << sprintf("%5i %s %s%s", item.rule_id, l, r, la) << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflicts(io, state) + return if state.conflicts.empty? + + state.conflicts.each do |conflict| + syms = conflict.symbols.map { |sym| sym.display_name } + io << " Conflict on #{syms.join(", ")}. " + + case conflict.type + when :shift_reduce + # @type var conflict: Lrama::State::ShiftReduceConflict + io << "shift/reduce(#{conflict.reduce.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + when :reduce_reduce + # @type var conflict: Lrama::State::ReduceReduceConflict + io << "reduce(#{conflict.reduce1.item.rule.lhs.display_name})/reduce(#{conflict.reduce2.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce1.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + + conflict.reduce2.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + else + raise "Unknown conflict type #{conflict.type}" + end + + io << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_shifts(io, state) + shifts = state.term_transitions.reject(&:not_selected) + + return if shifts.empty? + + next_syms = shifts.map(&:next_sym) + max_len = next_syms.map(&:display_name).map(&:length).max + shifts.each do |shift| + io << " #{shift.next_sym.display_name.ljust(max_len)} shift, and go to state #{shift.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nonassoc_errors(io, state) + error_symbols = state.resolved_conflicts.select { |resolved| resolved.which == :error }.map { |error| error.symbol.display_name } + + return if error_symbols.empty? + + max_len = error_symbols.map(&:length).max + error_symbols.each do |name| + io << " #{name.ljust(max_len)} error (nonassociative)\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_reduces(io, state) + reduce_pairs = [] #: Array[[Lrama::Grammar::Symbol, Lrama::State::Action::Reduce]] + + state.non_default_reduces.each do |reduce| + reduce.look_ahead&.each do |term| + reduce_pairs << [term, reduce] + end + end + + return if reduce_pairs.empty? && !state.default_reduction_rule + + max_len = [ + reduce_pairs.map(&:first).map(&:display_name).map(&:length).max || 0, + state.default_reduction_rule ? "$default".length : 0 + ].max + + reduce_pairs.sort_by { |term, _| term.number }.each do |term, reduce| + rule = reduce.item.rule + io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" + end + + if (r = state.default_reduction_rule) + s = "$default".ljust(max_len) + + if r.initial_rule? + io << " #{s} accept\n" + else + io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nterm_transitions(io, state) + return if state.nterm_transitions.empty? + + goto_transitions = state.nterm_transitions.sort_by do |goto| + goto.next_sym.number + end + + max_len = goto_transitions.map(&:next_sym).map do |nterm| + nterm.id.s_value.length + end.max + goto_transitions.each do |goto| + io << " #{goto.next_sym.id.s_value.ljust(max_len)} go to state #{goto.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflict_resolutions(io, state) + return if state.resolved_conflicts.empty? + + state.resolved_conflicts.each do |resolved| + io << " #{resolved.report_message}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::Counterexamples cex) -> void + def report_counterexamples(io, state, cex) + examples = cex.compute(state) + + examples.each do |example| + is_shift_reduce = example.type == :shift_reduce + label0 = is_shift_reduce ? "shift/reduce" : "reduce/reduce" + label1 = is_shift_reduce ? "Shift derivation" : "First Reduce derivation" + label2 = is_shift_reduce ? "Reduce derivation" : "Second Reduce derivation" + + io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" + io << " #{label1}\n" + + example.derivations1.render_strings_for_report.each do |str| + io << " #{str}\n" + end + + io << " #{label2}\n" + + example.derivations2.render_strings_for_report.each do |str| + io << " #{str}\n" + end + end + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_verbose_info(io, state, states) + report_direct_read_sets(io, state, states) + report_reads_relation(io, state, states) + report_read_sets(io, state, states) + report_includes_relation(io, state, states) + report_lookback_relation(io, state, states) + report_follow_sets(io, state, states) + report_look_ahead_sets(io, state, states) + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_direct_read_sets(io, state, states) + io << " [Direct Read sets]\n" + direct_read_sets = states.direct_read_sets + + state.nterm_transitions.each do |goto| + terms = direct_read_sets[goto] + next unless terms && !terms.empty? + + str = terms.map { |sym| sym.id.s_value }.join(", ") + io << " read #{goto.next_sym.id.s_value} shift #{str}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_reads_relation(io, state, states) + io << " [Reads Relation]\n" + + state.nterm_transitions.each do |goto| + goto2 = states.reads_relation[goto] + next unless goto2 + + goto2.each do |goto2| + io << " (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_read_sets(io, state, states) + io << " [Read sets]\n" + read_sets = states.read_sets + + state.nterm_transitions.each do |goto| + terms = read_sets[goto] + next unless terms && !terms.empty? + + terms.each do |sym| + io << " #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_includes_relation(io, state, states) + io << " [Includes Relation]\n" + + state.nterm_transitions.each do |goto| + gotos = states.includes_relation[goto] + next unless gotos + + gotos.each do |goto2| + io << " (State #{state.id}, #{goto.next_sym.id.s_value}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_lookback_relation(io, state, states) + io << " [Lookback Relation]\n" + + states.rules.each do |rule| + gotos = states.lookback_relation.dig(state.id, rule.id) + next unless gotos + + gotos.each do |goto2| + io << " (Rule: #{rule.display_name}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_follow_sets(io, state, states) + io << " [Follow sets]\n" + follow_sets = states.follow_sets + + state.nterm_transitions.each do |goto| + terms = follow_sets[goto] + next unless terms + + terms.each do |sym| + io << " #{goto.next_sym.id.s_value} -> #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_look_ahead_sets(io, state, states) + io << " [Look-Ahead Sets]\n" + look_ahead_rules = [] #: Array[[Lrama::Grammar::Rule, Array[Lrama::Grammar::Symbol]]] + + states.rules.each do |rule| + syms = states.la.dig(state.id, rule.id) + next unless syms + + look_ahead_rules << [rule, syms] + end + + return if look_ahead_rules.empty? + + max_len = look_ahead_rules.flat_map { |_, syms| syms.map { |s| s.id.s_value.length } }.max + + look_ahead_rules.each do |rule, syms| + syms.each do |sym| + io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/terms.rb b/tool/lrama/lib/lrama/reporter/terms.rb new file mode 100644 index 0000000000..f72d8b1a1a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/terms.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Terms + # @rbs (?terms: bool, **bool _) -> void + def initialize(terms: false, **_) + @terms = terms + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @terms + + look_aheads = states.states.each do |state| + state.reduces.flat_map do |reduce| + reduce.look_ahead unless reduce.look_ahead.nil? + end + end + + next_terms = states.states.flat_map do |state| + state.term_transitions.map {|shift| shift.next_sym } + end + + unused_symbols = states.terms.reject do |term| + (look_aheads + next_terms).include?(term) + end + + io << states.terms.count << " Terms\n\n" + + io << states.nterms.count << " Non-Terminals\n\n" + + unless unused_symbols.empty? + io << "#{unused_symbols.count} Unused Terms\n\n" + unused_symbols.each_with_index do |term, index| + io << sprintf("%5d %s", index, term.id.s_value) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb new file mode 100644 index 0000000000..50912e094e --- /dev/null +++ b/tool/lrama/lib/lrama/state.rb @@ -0,0 +1,534 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "state/action" +require_relative "state/inadequacy_annotation" +require_relative "state/item" +require_relative "state/reduce_reduce_conflict" +require_relative "state/resolved_conflict" +require_relative "state/shift_reduce_conflict" + +module Lrama + class State + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type conflict = State::ShiftReduceConflict | State::ReduceReduceConflict + # type transition = Action::Shift | Action::Goto + # type lookahead_set = Hash[Item, Array[Grammar::Symbol]] + # + # @id: Integer + # @accessing_symbol: Grammar::Symbol + # @kernels: Array[Item] + # @items: Array[Item] + # @items_to_state: Hash[Array[Item], State] + # @conflicts: Array[conflict] + # @resolved_conflicts: Array[ResolvedConflict] + # @default_reduction_rule: Grammar::Rule? + # @closure: Array[Item] + # @nterm_transitions: Array[Action::Goto] + # @term_transitions: Array[Action::Shift] + # @transitions: Array[transition] + # @internal_dependencies: Hash[Action::Goto, Array[Action::Goto]] + # @successor_dependencies: Hash[Action::Goto, Array[Action::Goto]] + + attr_reader :id #: Integer + attr_reader :accessing_symbol #: Grammar::Symbol + attr_reader :kernels #: Array[Item] + attr_reader :conflicts #: Array[conflict] + attr_reader :resolved_conflicts #: Array[ResolvedConflict] + attr_reader :default_reduction_rule #: Grammar::Rule? + attr_reader :closure #: Array[Item] + attr_reader :items #: Array[Item] + attr_reader :annotation_list #: Array[InadequacyAnnotation] + attr_reader :predecessors #: Array[State] + attr_reader :items_to_state #: Hash[Array[Item], State] + attr_reader :lane_items #: Hash[State, Array[[Item, Item]]] + + attr_accessor :_transitions #: Array[[Grammar::Symbol, Array[Item]]] + attr_accessor :reduces #: Array[Action::Reduce] + attr_accessor :ielr_isocores #: Array[State] + attr_accessor :lalr_isocore #: State + attr_accessor :lookaheads_recomputed #: bool + attr_accessor :follow_kernel_items #: Hash[Action::Goto, Hash[Item, bool]] + attr_accessor :always_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor :goto_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + + # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void + def initialize(id, accessing_symbol, kernels) + @id = id + @accessing_symbol = accessing_symbol + @kernels = kernels.freeze + @items = @kernels + # Manage relationships between items to state + # to resolve next state + @items_to_state = {} + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + @predecessors = [] + @lalr_isocore = self + @ielr_isocores = [self] + @internal_dependencies = {} + @successor_dependencies = {} + @annotation_list = [] + @lookaheads_recomputed = false + @follow_kernel_items = {} + @always_follows = {} + @goto_follows = {} + @lhs_contributions = {} + @lane_items = {} + end + + # @rbs (State other) -> bool + def ==(other) + self.id == other.id + end + + # @rbs (Array[Item] closure) -> void + def closure=(closure) + @closure = closure + @items = @kernels + @closure + end + + # @rbs () -> Array[Action::Reduce] + def non_default_reduces + reduces.reject do |reduce| + reduce.rule == @default_reduction_rule + end + end + + # @rbs () -> void + def compute_transitions_and_reduces + _transitions = {} + @_lane_items ||= {} + reduces = [] + items.each do |item| + # TODO: Consider what should be pushed + if item.end_of_rule? + reduces << Action::Reduce.new(item) + else + key = item.next_sym + _transitions[key] ||= [] + @_lane_items[key] ||= [] + next_item = item.new_by_next_position + _transitions[key] << next_item + @_lane_items[key] << [item, next_item] + end + end + + # It seems Bison 3.8.2 iterates transitions order by symbol number + transitions = _transitions.sort_by do |next_sym, to_items| + next_sym.number + end + + self._transitions = transitions.freeze + self.reduces = reduces.freeze + end + + # @rbs (Grammar::Symbol next_sym, State next_state) -> void + def set_lane_items(next_sym, next_state) + @lane_items[next_state] = @_lane_items[next_sym] + end + + # @rbs (Array[Item] items, State next_state) -> void + def set_items_to_state(items, next_state) + @items_to_state[items] = next_state + end + + # @rbs (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void + def set_look_ahead(rule, look_ahead) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead = look_ahead + end + + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void + def set_look_ahead_sources(rule, sources) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead_sources = sources + end + + # @rbs () -> Array[Action::Goto] + def nterm_transitions # steep:ignore + @nterm_transitions ||= transitions.select {|transition| transition.is_a?(Action::Goto) } + end + + # @rbs () -> Array[Action::Shift] + def term_transitions # steep:ignore + @term_transitions ||= transitions.select {|transition| transition.is_a?(Action::Shift) } + end + + # @rbs () -> Array[transition] + def transitions + @transitions ||= _transitions.map do |next_sym, to_items| + if next_sym.term? + Action::Shift.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + else + Action::Goto.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + end + end + end + + # @rbs (transition transition, State next_state) -> void + def update_transition(transition, next_state) + set_items_to_state(transition.to_items, next_state) + next_state.append_predecessor(self) + update_transitions_caches(transition) + end + + # @rbs () -> void + def update_transitions_caches(transition) + new_transition = + if transition.next_sym.term? + Action::Shift.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + else + Action::Goto.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + end + + @transitions.delete(transition) + @transitions << new_transition + @nterm_transitions = nil + @term_transitions = nil + + @follow_kernel_items[new_transition] = @follow_kernel_items.delete(transition) + @always_follows[new_transition] = @always_follows.delete(transition) + end + + # @rbs () -> Array[Action::Shift] + def selected_term_transitions + term_transitions.reject do |shift| + shift.not_selected + end + end + + # Move to next state by sym + # + # @rbs (Grammar::Symbol sym) -> State + def transition(sym) + result = nil + + if sym.term? + result = term_transitions.find {|shift| shift.next_sym == sym }.to_state + else + result = nterm_transitions.find {|goto| goto.next_sym == sym }.to_state + end + + raise "Can not transit by #{sym} #{self}" if result.nil? + + result + end + + # @rbs (Item item) -> Action::Reduce + def find_reduce_by_item!(item) + reduces.find do |r| + r.item == item + end || (raise "reduce is not found. #{item}") + end + + # @rbs (Grammar::Rule default_reduction_rule) -> void + def default_reduction_rule=(default_reduction_rule) + @default_reduction_rule = default_reduction_rule + + reduces.each do |r| + if r.rule == default_reduction_rule + r.default_reduction = true + end + end + end + + # @rbs () -> bool + def has_conflicts? + !@conflicts.empty? + end + + # @rbs () -> Array[conflict] + def sr_conflicts + @conflicts.select do |conflict| + conflict.type == :shift_reduce + end + end + + # @rbs () -> Array[conflict] + def rr_conflicts + @conflicts.select do |conflict| + conflict.type == :reduce_reduce + end + end + + # Clear information related to conflicts. + # IELR computation re-calculates conflicts and default reduction of states + # after LALR computation. + # Call this method before IELR computation to avoid duplicated conflicts information + # is stored. + # + # @rbs () -> void + def clear_conflicts + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + + term_transitions.each(&:clear_conflicts) + reduces.each(&:clear_conflicts) + end + + # @rbs () -> bool + def split_state? + @lalr_isocore != self + end + + # Definition 3.40 (propagate_lookaheads) + # + # @rbs (State next_state) -> lookahead_set + def propagate_lookaheads(next_state) + next_state.kernels.map {|next_kernel| + lookahead_sets = + if next_kernel.position > 1 + kernel = kernels.find {|k| k.predecessor_item_of?(next_kernel) } + item_lookahead_set[kernel] + else + goto_follow_set(next_kernel.lhs) + end + + [next_kernel, lookahead_sets & next_state.lookahead_set_filters[next_kernel]] + }.to_h + end + + # Definition 3.43 (is_compatible) + # + # @rbs (lookahead_set filtered_lookahead) -> bool + def is_compatible?(filtered_lookahead) + !lookaheads_recomputed || + @lalr_isocore.annotation_list.all? {|annotation| + a = annotation.dominant_contribution(item_lookahead_set) + b = annotation.dominant_contribution(filtered_lookahead) + a.nil? || b.nil? || a == b + } + end + + # Definition 3.38 (lookahead_set_filters) + # + # @rbs () -> lookahead_set + def lookahead_set_filters + @lookahead_set_filters ||= kernels.map {|kernel| + [kernel, @lalr_isocore.annotation_list.select {|annotation| annotation.contributed?(kernel) }.map(&:token)] + }.to_h + end + + # Definition 3.27 (inadequacy_lists) + # + # @rbs () -> Hash[Grammar::Symbol, Array[Action::Shift | Action::Reduce]] + def inadequacy_list + return @inadequacy_list if @inadequacy_list + + inadequacy_list = {} + + term_transitions.each do |shift| + inadequacy_list[shift.next_sym] ||= [] + inadequacy_list[shift.next_sym] << shift.dup + end + reduces.each do |reduce| + next if reduce.look_ahead.nil? + + reduce.look_ahead.each do |token| + inadequacy_list[token] ||= [] + inadequacy_list[token] << reduce.dup + end + end + + @inadequacy_list = inadequacy_list.select {|token, actions| actions.size > 1 } + end + + # Definition 3.30 (annotate_manifestation) + # + # @rbs () -> void + def annotate_manifestation + inadequacy_list.each {|token, actions| + contribution_matrix = actions.map {|action| + if action.is_a?(Action::Shift) + [action, nil] + else + [action, action.rule.empty_rule? ? lhs_contributions(action.rule.lhs, token) : kernels.map {|k| [k, k.rule == action.item.rule && k.end_of_rule?] }.to_h] + end + }.to_h + @annotation_list << InadequacyAnnotation.new(self, token, actions, contribution_matrix) + } + end + + # Definition 3.32 (annotate_predecessor) + # + # @rbs (State predecessor) -> void + def annotate_predecessor(predecessor) + propagating_list = annotation_list.map {|annotation| + contribution_matrix = annotation.contribution_matrix.map {|action, contributions| + if contributions.nil? + [action, nil] + elsif first_kernels.any? {|kernel| contributions[kernel] && predecessor.lhs_contributions(kernel.lhs, annotation.token).empty? } + [action, nil] + else + cs = predecessor.lane_items[self].map {|pred_kernel, kernel| + c = contributions[kernel] && ( + (kernel.position > 1 && predecessor.item_lookahead_set[pred_kernel].include?(annotation.token)) || + (kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, annotation.token)[pred_kernel]) + ) + [pred_kernel, c] + }.to_h + [action, cs] + end + }.to_h + + # Observation 3.33 (Simple Split-Stable Dominance) + # + # If all of contributions in the contribution_matrix are + # always contribution or never contribution, we can stop annotate propagations + # to the predecessor state. + next nil if contribution_matrix.all? {|_, contributions| contributions.nil? || contributions.all? {|_, contributed| !contributed } } + + InadequacyAnnotation.new(annotation.state, annotation.token, annotation.actions, contribution_matrix) + }.compact + predecessor.append_annotation_list(propagating_list) + end + + # @rbs () -> Array[Item] + def first_kernels + @first_kernels ||= kernels.select {|kernel| kernel.position == 1 } + end + + # @rbs (Array[InadequacyAnnotation] propagating_list) -> void + def append_annotation_list(propagating_list) + annotation_list.each do |annotation| + merging_list = propagating_list.select {|a| a.state == annotation.state && a.token == annotation.token && a.actions == annotation.actions } + annotation.merge_matrix(merging_list.map(&:contribution_matrix)) + propagating_list -= merging_list + end + + @annotation_list += propagating_list + end + + # Definition 3.31 (compute_lhs_contributions) + # + # @rbs (Grammar::Symbol sym, Grammar::Symbol token) -> (nil | Hash[Item, bool]) + def lhs_contributions(sym, token) + return @lhs_contributions[sym][token] unless @lhs_contributions.dig(sym, token).nil? + + transition = nterm_transitions.find {|goto| goto.next_sym == sym } + @lhs_contributions[sym] ||= {} + @lhs_contributions[sym][token] = + if always_follows[transition].include?(token) + {} + else + kernels.map {|kernel| [kernel, follow_kernel_items[transition][kernel] && item_lookahead_set[kernel].include?(token)] }.to_h + end + end + + # Definition 3.26 (item_lookahead_sets) + # + # @rbs () -> lookahead_set + def item_lookahead_set + return @item_lookahead_set if @item_lookahead_set + + @item_lookahead_set = kernels.map {|k| [k, []] }.to_h + @item_lookahead_set = kernels.map {|kernel| + value = + if kernel.lhs.accept_symbol? + [] + elsif kernel.position > 1 + prev_items = predecessors_with_item(kernel) + prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms } + elsif kernel.position == 1 + prev_state = @predecessors.find {|p| p.transitions.any? {|transition| transition.next_sym == kernel.lhs } } + goto = prev_state.nterm_transitions.find {|goto| goto.next_sym == kernel.lhs } + prev_state.goto_follows[goto] + end + [kernel, value] + }.to_h + end + + # @rbs (lookahead_set k) -> void + def item_lookahead_set=(k) + @item_lookahead_set = k + end + + # @rbs (Item item) -> Array[[State, Item]] + def predecessors_with_item(item) + result = [] + @predecessors.each do |pre| + pre.items.each do |i| + result << [pre, i] if i.predecessor_item_of?(item) + end + end + result + end + + # @rbs (State prev_state) -> void + def append_predecessor(prev_state) + @predecessors << prev_state + @predecessors.uniq! + end + + # Definition 3.39 (compute_goto_follow_set) + # + # @rbs (Grammar::Symbol nterm_token) -> Array[Grammar::Symbol] + def goto_follow_set(nterm_token) + return [] if nterm_token.accept_symbol? + goto = @lalr_isocore.nterm_transitions.find {|g| g.next_sym == nterm_token } + + @kernels + .select {|kernel| @lalr_isocore.follow_kernel_items[goto][kernel] } + .map {|kernel| item_lookahead_set[kernel] } + .reduce(@lalr_isocore.always_follows[goto]) {|result, terms| result |= terms } + end + + # Definition 3.8 (Goto Follows Internal Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def internal_dependencies(goto) + return @internal_dependencies[goto] if @internal_dependencies[goto] + + syms = @items.select {|i| + i.next_sym == goto.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0 + }.map(&:lhs).uniq + @internal_dependencies[goto] = nterm_transitions.select {|goto2| syms.include?(goto2.next_sym) } + end + + # Definition 3.5 (Goto Follows Successor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def successor_dependencies(goto) + return @successor_dependencies[goto] if @successor_dependencies[goto] + + @successor_dependencies[goto] = goto.to_state.nterm_transitions.select {|next_goto| next_goto.next_sym.nullable } + end + + # Definition 3.9 (Goto Follows Predecessor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def predecessor_dependencies(goto) + state_items = [] + @kernels.select {|kernel| + kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) + }.each do |item| + queue = predecessors_with_item(item) + until queue.empty? + st, i = queue.pop + if i.position == 0 + state_items << [st, i] + else + st.predecessors_with_item(i).each {|v| queue << v } + end + end + end + + state_items.map {|state, item| + state.nterm_transitions.find {|goto2| goto2.next_sym == item.lhs } + } + end + end +end diff --git a/tool/lrama/lib/lrama/state/action.rb b/tool/lrama/lib/lrama/state/action.rb new file mode 100644 index 0000000000..791685fc23 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "action/goto" +require_relative "action/reduce" +require_relative "action/shift" diff --git a/tool/lrama/lib/lrama/state/action/goto.rb b/tool/lrama/lib/lrama/state/action/goto.rb new file mode 100644 index 0000000000..4c2c82afdc --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/goto.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Goto + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/reduce.rb b/tool/lrama/lib/lrama/state/action/reduce.rb new file mode 100644 index 0000000000..9678ab0a98 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/reduce.rb @@ -0,0 +1,71 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Reduce + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @item: Item + # @look_ahead: Array[Grammar::Symbol]? + # @look_ahead_sources: Hash[Grammar::Symbol, Array[Action::Goto]]? + # @not_selected_symbols: Array[Grammar::Symbol] + + attr_reader :item #: Item + attr_reader :look_ahead #: Array[Grammar::Symbol]? + attr_reader :look_ahead_sources #: Hash[Grammar::Symbol, Array[Action::Goto]]? + attr_reader :not_selected_symbols #: Array[Grammar::Symbol] + + # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html + attr_accessor :default_reduction #: bool + + # @rbs (Item item) -> void + def initialize(item) + @item = item + @look_ahead = nil + @look_ahead_sources = nil + @not_selected_symbols = [] + end + + # @rbs () -> Grammar::Rule + def rule + @item.rule + end + + # @rbs (Array[Grammar::Symbol] look_ahead) -> Array[Grammar::Symbol] + def look_ahead=(look_ahead) + @look_ahead = look_ahead.freeze + end + + # @rbs (Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> Hash[Grammar::Symbol, Array[Action::Goto]] + def look_ahead_sources=(sources) + @look_ahead_sources = sources.freeze + end + + # @rbs (Grammar::Symbol sym) -> Array[Grammar::Symbol] + def add_not_selected_symbol(sym) + @not_selected_symbols << sym + end + + # @rbs () -> (::Array[Grammar::Symbol?]) + def selected_look_ahead + if look_ahead + look_ahead - @not_selected_symbols + else + [] + end + end + + # @rbs () -> void + def clear_conflicts + @not_selected_symbols = [] + @default_reduction = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/shift.rb b/tool/lrama/lib/lrama/state/action/shift.rb new file mode 100644 index 0000000000..52d9f8c4f0 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/shift.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Shift + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + attr_accessor :not_selected #: bool + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + + # @rbs () -> void + def clear_conflicts + @not_selected = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/inadequacy_annotation.rb b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb new file mode 100644 index 0000000000..3654fa4607 --- /dev/null +++ b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb @@ -0,0 +1,140 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class InadequacyAnnotation + # @rbs! + # type action = Action::Shift | Action::Reduce + + attr_accessor :state #: State + attr_accessor :token #: Grammar::Symbol + attr_accessor :actions #: Array[action] + attr_accessor :contribution_matrix #: Hash[action, Hash[Item, bool]] + + # @rbs (State state, Grammar::Symbol token, Array[action] actions, Hash[action, Hash[Item, bool]] contribution_matrix) -> void + def initialize(state, token, actions, contribution_matrix) + @state = state + @token = token + @actions = actions + @contribution_matrix = contribution_matrix + end + + # @rbs (Item item) -> bool + def contributed?(item) + @contribution_matrix.any? {|action, contributions| !contributions.nil? && contributions[item] } + end + + # @rbs (Array[Hash[action, Hash[Item, bool]]] another_matrixes) -> void + def merge_matrix(another_matrixes) + another_matrixes.each do |another_matrix| + @contribution_matrix.merge!(another_matrix) {|action, contributions, another_contributions| + next contributions if another_contributions.nil? + next another_contributions if contributions.nil? + + contributions.merge!(another_contributions) {|_, contributed, another_contributed| contributed || another_contributed } + } + end + end + + # Definition 3.42 (dominant_contribution) + # + # @rbs (State::lookahead_set lookaheads) -> Array[action]? + def dominant_contribution(lookaheads) + actions = @actions.select {|action| + contribution_matrix[action].nil? || contribution_matrix[action].any? {|item, contributed| contributed && lookaheads[item].include?(@token) } + } + return nil if actions.empty? + + resolve_conflict(actions) + end + + # @rbs (Array[action] actions) -> Array[action] + def resolve_conflict(actions) + # @type var shifts: Array[Action::Shift] + # @type var reduces: Array[Action::Reduce] + shifts = actions.select {|action| action.is_a?(Action::Shift)} + reduces = actions.select {|action| action.is_a?(Action::Reduce) } + + shifts.each do |shift| + reduces.each do |reduce| + sym = shift.next_sym + + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + actions.delete(shift) + next + when shift_prec > reduce_prec + # Shift is selected + actions.delete(reduce) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence&.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + next + when :right + # Shift is selected + actions.delete(reduce) + next + when :left + # Reduce is selected + actions.delete(shift) + next + when :nonassoc + # Can not resolve + # + # nonassoc creates "run-time" error, precedence creates "compile-time" error. + # Then omit both the shift and reduce. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + actions.delete(shift) + actions.delete(reduce) + else + raise "Unknown precedence type. #{sym}" + end + end + end + + actions + end + + # @rbs () -> String + def to_s + "State: #{@state.id}, Token: #{@token.id.s_value}, Actions: #{actions_to_s}, Contributions: #{contribution_matrix_to_s}" + end + + private + + # @rbs () -> String + def actions_to_s + '[' + @actions.map {|action| + if action.is_a?(Action::Shift) || action.is_a?(Action::Goto) + action.class.name + elsif action.is_a?(Action::Reduce) + "#{action.class.name}: (#{action.item})" + end + }.join(', ') + ']' + end + + # @rbs () -> String + def contribution_matrix_to_s + '[' + @contribution_matrix.map {|action, contributions| + "#{(action.is_a?(Action::Shift) || action.is_a?(Action::Goto)) ? action.class.name : "#{action.class.name}: (#{action.item})"}: " + contributions&.transform_keys(&:to_s).to_s + }.join(', ') + ']' + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/item.rb b/tool/lrama/lib/lrama/state/item.rb new file mode 100644 index 0000000000..3ecdd70b76 --- /dev/null +++ b/tool/lrama/lib/lrama/state/item.rb @@ -0,0 +1,120 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# TODO: Validate position is not over rule rhs + +require "forwardable" + +module Lrama + class State + class Item < Struct.new(:rule, :position, keyword_init: true) + # @rbs! + # include Grammar::Rule::_DelegatedMethods + # + # attr_accessor rule: Grammar::Rule + # attr_accessor position: Integer + # + # def initialize: (?rule: Grammar::Rule, ?position: Integer) -> void + + extend Forwardable + + def_delegators "rule", :lhs, :rhs + + # Optimization for States#setup_state + # + # @rbs () -> Integer + def hash + [rule_id, position].hash + end + + # @rbs () -> Integer + def rule_id + rule.id + end + + # @rbs () -> bool + def empty_rule? + rule.empty_rule? + end + + # @rbs () -> Integer + def number_of_rest_symbols + @number_of_rest_symbols ||= rhs.count - position + end + + # @rbs () -> Grammar::Symbol + def next_sym + rhs[position] + end + + # @rbs () -> Grammar::Symbol + def next_next_sym + @next_next_sym ||= rhs[position + 1] + end + + # @rbs () -> Grammar::Symbol + def previous_sym + rhs[position - 1] + end + + # @rbs () -> bool + def end_of_rule? + rhs.count == position + end + + # @rbs () -> bool + def beginning_of_rule? + position == 0 + end + + # @rbs () -> bool + def start_item? + rule.initial_rule? && beginning_of_rule? + end + + # @rbs () -> State::Item + def new_by_next_position + Item.new(rule: rule, position: position + 1) + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_before_dot # steep:ignore + rhs[0...position] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_dot # steep:ignore + rhs[position..-1] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_transition # steep:ignore + rhs[position+1..-1] + end + + # @rbs () -> ::String + def to_s + "#{lhs.id.s_value}: #{display_name}" + end + + # @rbs () -> ::String + def display_name + r = rhs.map(&:display_name).insert(position, "•").join(" ") + "#{r} (rule #{rule_id})" + end + + # Right after position + # + # @rbs () -> ::String + def display_rest + r = symbols_after_dot.map(&:display_name).join(" ") + ". #{r} (rule #{rule_id})" + end + + # @rbs (State::Item other_item) -> bool + def predecessor_item_of?(other_item) + rule == other_item.rule && position == other_item.position - 1 + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb new file mode 100644 index 0000000000..55ecad40bd --- /dev/null +++ b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class ReduceReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :reduce1 #: State::Action::Reduce + attr_reader :reduce2 #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], reduce1: State::Action::Reduce, reduce2: State::Action::Reduce) -> void + def initialize(symbols:, reduce1:, reduce2:) + @symbols = symbols + @reduce1 = reduce1 + @reduce2 = reduce2 + end + + # @rbs () -> :reduce_reduce + def type + :reduce_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/resolved_conflict.rb b/tool/lrama/lib/lrama/state/resolved_conflict.rb new file mode 100644 index 0000000000..014533c233 --- /dev/null +++ b/tool/lrama/lib/lrama/state/resolved_conflict.rb @@ -0,0 +1,65 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + # * state: A state on which the conflct is resolved + # * symbol: A symbol under discussion + # * reduce: A reduce under discussion + # * which: For which a conflict is resolved. :shift, :reduce or :error (for nonassociative) + # * resolved_by_precedence: If the conflict is resolved by precedence definition or not + class ResolvedConflict + # @rbs! + # type which_enum = :reduce | :shift | :error + + attr_reader :state #: State + attr_reader :symbol #: Grammar::Symbol + attr_reader :reduce #: State::Action::Reduce + attr_reader :which #: which_enum + attr_reader :resolved_by_precedence #: bool + + # @rbs (state: State, symbol: Grammar::Symbol, reduce: State::Action::Reduce, which: which_enum, resolved_by_precedence: bool) -> void + def initialize(state:, symbol:, reduce:, which:, resolved_by_precedence:) + @state = state + @symbol = symbol + @reduce = reduce + @which = which + @resolved_by_precedence = resolved_by_precedence + end + + # @rbs () -> (::String | bot) + def report_message + "Conflict between rule #{reduce.rule.id} and token #{symbol.display_name} #{how_resolved}." + end + + # @rbs () -> (::String | bot) + def report_precedences_message + "Conflict between reduce by \"#{reduce.rule.display_name}\" and shift #{symbol.display_name} #{how_resolved}." + end + + private + + # @rbs () -> (::String | bot) + def how_resolved + s = symbol.display_name + r = reduce.rule.precedence_sym&.display_name + case + when which == :shift && resolved_by_precedence + msg = "resolved as #{which} (%right #{s})" + when which == :shift + msg = "resolved as #{which} (#{r} < #{s})" + when which == :reduce && resolved_by_precedence + msg = "resolved as #{which} (%left #{s})" + when which == :reduce + msg = "resolved as #{which} (#{s} < #{r})" + when which == :error + msg = "resolved as an #{which} (%nonassoc #{s})" + else + raise "Unknown direction. #{self}" + end + + msg + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb new file mode 100644 index 0000000000..548f2de614 --- /dev/null +++ b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class ShiftReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :shift #: State::Action::Shift + attr_reader :reduce #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], shift: State::Action::Shift, reduce: State::Action::Reduce) -> void + def initialize(symbols:, shift:, reduce:) + @symbols = symbols + @shift = shift + @reduce = reduce + end + + # @rbs () -> :shift_reduce + def type + :shift_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb new file mode 100644 index 0000000000..ddce627df4 --- /dev/null +++ b/tool/lrama/lib/lrama/states.rb @@ -0,0 +1,867 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "tracer/duration" +require_relative "state/item" + +module Lrama + # States is passed to a template file + # + # "Efficient Computation of LALR(1) Look-Ahead Sets" + # https://dl.acm.org/doi/pdf/10.1145/69622.357187 + class States + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type state_id = Integer + # type rule_id = Integer + # + # include Grammar::_DelegatedMethods + # + # @grammar: Grammar + # @tracer: Tracer + # @states: Array[State] + # @direct_read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @reads_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @includes_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @lookback_relation: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + # @follow_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @la: Hash[state_id, Hash[rule_id, Bitmap::bitmap]] + + extend Forwardable + include Lrama::Tracer::Duration + + def_delegators "@grammar", :symbols, :terms, :nterms, :rules, :precedences, + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined? + + attr_reader :states #: Array[State] + attr_reader :reads_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :includes_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :lookback_relation #: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + + # @rbs (Grammar grammar, Tracer tracer) -> void + def initialize(grammar, tracer) + @grammar = grammar + @tracer = tracer + + @states = [] + + # `DR(p, A) = {t ∈ T | p -(A)-> r -(t)-> }` + # where p is state, A is nterm, t is term. + # + # `@direct_read_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @direct_read_sets = {} + + # Reads relation on nonterminal transitions (pair of state and nterm) + # `(p, A) reads (r, C) iff p -(A)-> r -(C)-> and C =>* ε` + # where p, r are state, A, C are nterm. + # + # `@reads_relation` is a hash whose + # key is goto, + # value is array of goto. + @reads_relation = {} + + # `Read(p, A) =s DR(p, A) ∪ ∪{Read(r, C) | (p, A) reads (r, C)}` + # + # `@read_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @read_sets = {} + + # `(p, A) includes (p', B) iff B -> βAγ, γ =>* ε, p' -(β)-> p` + # where p, p' are state, A, B are nterm, β, γ is sequence of symbol. + # + # `@includes_relation` is a hash whose + # key is goto, + # value is array of goto. + @includes_relation = {} + + # `(q, A -> ω) lookback (p, A) iff p -(ω)-> q` + # where p, q are state, A -> ω is rule, A is nterm, ω is sequence of symbol. + # + # `@lookback_relation` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is array of goto. + @lookback_relation = {} + + # `Follow(p, A) =s Read(p, A) ∪ ∪{Follow(p', B) | (p, A) includes (p', B)}` + # + # `@follow_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @follow_sets = {} + + # `LA(q, A -> ω) = ∪{Follow(p, A) | (q, A -> ω) lookback (p, A)` + # + # `@la` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is bitmap of term. + @la = {} + end + + # @rbs () -> void + def compute + report_duration(:compute_lr0_states) { compute_lr0_states } + + # Look Ahead Sets + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + + # Conflicts + report_duration(:compute_conflicts) { compute_conflicts(:lalr) } + + report_duration(:compute_default_reduction) { compute_default_reduction } + end + + # @rbs () -> void + def compute_ielr + # Preparation + report_duration(:clear_conflicts) { clear_conflicts } + # Phase 1 + report_duration(:compute_predecessors) { compute_predecessors } + report_duration(:compute_follow_kernel_items) { compute_follow_kernel_items } + report_duration(:compute_always_follows) { compute_always_follows } + report_duration(:compute_goto_follows) { compute_goto_follows } + # Phase 2 + report_duration(:compute_inadequacy_annotations) { compute_inadequacy_annotations } + # Phase 3 + report_duration(:split_states) { split_states } + # Phase 4 + report_duration(:clear_look_ahead_sets) { clear_look_ahead_sets } + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + # Phase 5 + report_duration(:compute_conflicts) { compute_conflicts(:ielr) } + report_duration(:compute_default_reduction) { compute_default_reduction } + end + + # @rbs () -> Integer + def states_count + @states.count + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def direct_read_sets + @_direct_read_sets ||= @direct_read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def read_sets + @_read_sets ||= @read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def follow_sets + @_follow_sets ||= @follow_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[state_id, Hash[rule_id, Array[Grammar::Symbol]]] + def la + @_la ||= @la.transform_values do |second_hash| + second_hash.transform_values do |v| + bitmap_to_terms(v) + end + end + end + + # @rbs () -> Integer + def sr_conflicts_count + @sr_conflicts_count ||= @states.flat_map(&:sr_conflicts).count + end + + # @rbs () -> Integer + def rr_conflicts_count + @rr_conflicts_count ||= @states.flat_map(&:rr_conflicts).count + end + + # @rbs (Logger logger) -> void + def validate!(logger) + validate_conflicts_within_threshold!(logger) + end + + def compute_la_sources_for_conflicted_states + reflexive = {} + @states.each do |state| + state.nterm_transitions.each do |goto| + reflexive[goto] = [goto] + end + end + + # compute_read_sets + read_sets = Digraph.new(nterm_transitions, @reads_relation, reflexive).compute + # compute_follow_sets + follow_sets = Digraph.new(nterm_transitions, @includes_relation, read_sets).compute + + @states.select(&:has_conflicts?).each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + sources = {} + + ary.each do |goto| + source = follow_sets[goto] + + next unless source + + source.each do |goto2| + tokens = direct_read_sets[goto2] + tokens.each do |token| + sources[token] ||= [] + sources[token] |= [goto2] + end + end + end + + state.set_look_ahead_sources(rule, sources) + end + end + end + + private + + # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] + def create_state(accessing_symbol, kernels, states_created) + # A item can appear in some states, + # so need to use `kernels` (not `kernels.first`) as a key. + # + # For example... + # + # %% + # program: '+' strings_1 + # | '-' strings_2 + # ; + # + # strings_1: string_1 + # ; + # + # strings_2: string_1 + # | string_2 + # ; + # + # string_1: string + # ; + # + # string_2: string '+' + # ; + # + # string: tSTRING + # ; + # %% + # + # For these grammar, there are 2 states + # + # State A + # string_1: string • + # + # State B + # string_1: string • + # string_2: string • '+' + # + return [states_created[kernels], false] if states_created[kernels] + + state = State.new(@states.count, accessing_symbol, kernels) + @states << state + states_created[kernels] = state + + return [state, true] + end + + # @rbs (State state) -> void + def setup_state(state) + # closure + closure = [] + queued = {} + items = state.kernels.dup + + items.each do |item| + queued[item.rule_id] = true if item.position == 0 + end + + while (item = items.shift) do + if (sym = item.next_sym) && sym.nterm? + @grammar.find_rules_by_symbol!(sym).each do |rule| + next if queued[rule.id] + i = State::Item.new(rule: rule, position: 0) + closure << i + items << i + queued[i.rule_id] = true + end + end + end + + state.closure = closure.sort_by {|i| i.rule.id } + + # Trace + @tracer.trace_closure(state) + + # shift & reduce + state.compute_transitions_and_reduces + end + + # @rbs (Array[State] states, State state) -> void + def enqueue_state(states, state) + # Trace + @tracer.trace_state_list_append(@states.count, state) + + states << state + end + + # @rbs () -> void + def compute_lr0_states + # State queue + states = [] + states_created = {} + + state, _ = create_state(symbols.first, [State::Item.new(rule: @grammar.rules.first, position: 0)], states_created) + enqueue_state(states, state) + + while (state = states.shift) do + # Trace + @tracer.trace_state(state) + + setup_state(state) + + # `State#transitions` can not be used here + # because `items_to_state` of the `state` is not set yet. + state._transitions.each do |next_sym, to_items| + new_state, created = create_state(next_sym, to_items, states_created) + state.set_items_to_state(to_items, new_state) + state.set_lane_items(next_sym, new_state) + enqueue_state(states, new_state) if created + end + end + end + + # @rbs () -> Array[State::Action::Goto] + def nterm_transitions + a = [] + + @states.each do |state| + state.nterm_transitions.each do |goto| + a << goto + end + end + + a + end + + # @rbs () -> void + def compute_look_ahead_sets + report_duration(:compute_direct_read_sets) { compute_direct_read_sets } + report_duration(:compute_reads_relation) { compute_reads_relation } + report_duration(:compute_read_sets) { compute_read_sets } + report_duration(:compute_includes_relation) { compute_includes_relation } + report_duration(:compute_lookback_relation) { compute_lookback_relation } + report_duration(:compute_follow_sets) { compute_follow_sets } + report_duration(:compute_la) { compute_la } + end + + # @rbs () -> void + def compute_direct_read_sets + @states.each do |state| + state.nterm_transitions.each do |goto| + ary = goto.to_state.term_transitions.map do |shift| + shift.next_sym.number + end + + @direct_read_sets[goto] = Bitmap.from_array(ary) + end + end + end + + # @rbs () -> void + def compute_reads_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + goto.to_state.nterm_transitions.each do |goto2| + nterm2 = goto2.next_sym + if nterm2.nullable + @reads_relation[goto] ||= [] + @reads_relation[goto] << goto2 + end + end + end + end + end + + # @rbs () -> void + def compute_read_sets + @read_sets = Digraph.new(nterm_transitions, @reads_relation, @direct_read_sets).compute + end + + # Execute transition of state by symbols + # then return final state. + # + # @rbs (State state, Array[Grammar::Symbol] symbols) -> State + def transition(state, symbols) + symbols.each do |sym| + state = state.transition(sym) + end + + state + end + + # @rbs () -> void + def compute_includes_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + nterm = goto.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + i = rule.rhs.count - 1 + + while (i > -1) do + sym = rule.rhs[i] + + break if sym.term? + state2 = transition(state, rule.rhs[0...i]) + # p' = state, B = nterm, p = state2, A = sym + key = state2.nterm_transitions.find do |goto2| + goto2.next_sym.token_id == sym.token_id + end || (raise "Goto by #{sym.name} on state #{state2.id} is not found") + # TODO: need to omit if state == state2 ? + @includes_relation[key] ||= [] + @includes_relation[key] << goto + break unless sym.nullable + i -= 1 + end + end + end + end + end + + # @rbs () -> void + def compute_lookback_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + nterm = goto.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + state2 = transition(state, rule.rhs) + # p = state, A = nterm, q = state2, A -> ω = rule + @lookback_relation[state2.id] ||= {} + @lookback_relation[state2.id][rule.id] ||= [] + @lookback_relation[state2.id][rule.id] << goto + end + end + end + end + + # @rbs () -> void + def compute_follow_sets + @follow_sets = Digraph.new(nterm_transitions, @includes_relation, @read_sets).compute + end + + # @rbs () -> void + def compute_la + @states.each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + ary.each do |goto| + # q = state, A -> ω = rule, p = state2, A = nterm + follows = @follow_sets[goto] + + next if follows == 0 + + @la[state.id] ||= {} + @la[state.id][rule.id] ||= 0 + look_ahead = @la[state.id][rule.id] | follows + @la[state.id][rule.id] |= look_ahead + + # No risk of conflict when + # * the state only has single reduce + # * the state only has nterm_transitions (GOTO) + next if state.reduces.count == 1 && state.term_transitions.count == 0 + + state.set_look_ahead(rule, bitmap_to_terms(look_ahead)) + end + end + end + end + + # @rbs (Bitmap::bitmap bit) -> Array[Grammar::Symbol] + def bitmap_to_terms(bit) + ary = Bitmap.to_array(bit) + ary.map do |i| + @grammar.find_symbol_by_number!(i) + end + end + + # @rbs () -> void + def compute_conflicts(lr_type) + compute_shift_reduce_conflicts(lr_type) + compute_reduce_reduce_conflicts + end + + # @rbs () -> void + def compute_shift_reduce_conflicts(lr_type) + states.each do |state| + state.term_transitions.each do |shift| + state.reduces.each do |reduce| + sym = shift.next_sym + + next unless reduce.look_ahead + next unless reduce.look_ahead.include?(sym) + + # Shift/Reduce conflict + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when shift_prec > reduce_prec + # Shift is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence.type + when :precedence + # Can not resolve the conflict + # + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + when :right + # Shift is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when :left + # Reduce is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when :nonassoc + # The conflict is resolved + # + # %nonassoc creates "run-time" error by removing both shift and reduce from + # the state. This makes the state to get syntax error if the conflicted token appears. + # On the other hand, %precedence creates "compile-time" error by keeping both + # shift and reduce on the state. This makes the state to be conflicted on the token. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :error, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + else + raise "Unknown precedence type. #{sym}" + end + end + end + end + end + + # @rbs (Grammar::Precedence shift_prec, Grammar::Precedence reduce_prec, State::ResolvedConflict resolved_conflict) -> void + def mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + case lr_type + when :lalr + shift_prec.mark_used_by_lalr(resolved_conflict) + reduce_prec.mark_used_by_lalr(resolved_conflict) + when :ielr + shift_prec.mark_used_by_ielr(resolved_conflict) + reduce_prec.mark_used_by_ielr(resolved_conflict) + end + end + + # @rbs () -> void + def compute_reduce_reduce_conflicts + states.each do |state| + state.reduces.combination(2) do |reduce1, reduce2| + next if reduce1.look_ahead.nil? || reduce2.look_ahead.nil? + + intersection = reduce1.look_ahead & reduce2.look_ahead + + unless intersection.empty? + state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) + end + end + end + end + + # @rbs () -> void + def compute_default_reduction + states.each do |state| + next if state.reduces.empty? + # Do not set, if conflict exist + next unless state.conflicts.empty? + # Do not set, if shift with `error` exists. + next if state.term_transitions.map {|shift| shift.next_sym }.include?(@grammar.error_symbol) + + state.default_reduction_rule = state.reduces.map do |r| + [r.rule, r.rule.id, (r.look_ahead || []).count] + end.min_by do |rule, rule_id, count| + [-count, rule_id] + end.first + end + end + + # @rbs () -> void + def clear_conflicts + states.each(&:clear_conflicts) + end + + # Definition 3.15 (Predecessors) + # + # @rbs () -> void + def compute_predecessors + @states.each do |state| + state.transitions.each do |transition| + transition.to_state.append_predecessor(state) + end + end + end + + # Definition 3.16 (follow_kernel_items) + # + # @rbs () -> void + def compute_follow_kernel_items + set = nterm_transitions + relation = compute_goto_internal_relation + base_function = compute_goto_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, follow_kernel_items| + state = goto.from_state + state.follow_kernel_items[goto] = state.kernels.map {|kernel| + [kernel, Bitmap.to_bool_array(follow_kernel_items, state.kernels.count)] + }.to_h + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_goto_bitmaps + nterm_transitions.map {|goto| + bools = goto.from_state.kernels.map.with_index {|kernel, i| i if kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) }.compact + [goto, Bitmap.from_array(bools)] + }.to_h + end + + # Definition 3.20 (always_follows, one closure) + # + # @rbs () -> void + def compute_always_follows + set = nterm_transitions + relation = compute_goto_successor_or_internal_relation + base_function = compute_transition_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, always_follows_bitmap| + goto.from_state.always_follows[goto] = bitmap_to_terms(always_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_successor_or_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.successor_dependencies(goto) + state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_transition_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.to_state.term_transitions.map {|shift| shift.next_sym.number })] + }.to_h + end + + # Definition 3.24 (goto_follows, via always_follows) + # + # @rbs () -> void + def compute_goto_follows + set = nterm_transitions + relation = compute_goto_internal_or_predecessor_dependencies + base_function = compute_always_follows_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, goto_follows_bitmap| + goto.from_state.goto_follows[goto] = bitmap_to_terms(goto_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_or_predecessor_dependencies + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + state.predecessor_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_always_follows_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.from_state.always_follows[goto].map(&:number))] + }.to_h + end + + # @rbs () -> void + def split_states + @states.each do |state| + state.transitions.each do |transition| + compute_state(state, transition, transition.to_state) + end + end + end + + # @rbs () -> void + def compute_inadequacy_annotations + @states.each do |state| + state.annotate_manifestation + end + + queue = @states.reject {|state| state.annotation_list.empty? } + + while (curr = queue.shift) do + curr.predecessors.each do |pred| + cache = pred.annotation_list.dup + curr.annotate_predecessor(pred) + queue << pred if cache != pred.annotation_list && !queue.include?(pred) + end + end + end + + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> void + def merge_lookaheads(state, filtered_lookaheads) + return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? } + + state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 } + state.transitions.each do |transition| + next if transition.to_state.lookaheads_recomputed + compute_state(state, transition, transition.to_state) + end + end + + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void + def compute_state(state, transition, next_state) + propagating_lookaheads = state.propagate_lookaheads(next_state) + s = next_state.ielr_isocores.find {|st| st.is_compatible?(propagating_lookaheads) } + + if s.nil? + s = next_state.lalr_isocore + new_state = State.new(@states.count, s.accessing_symbol, s.kernels) + new_state.closure = s.closure + new_state.compute_transitions_and_reduces + s.transitions.each do |transition| + new_state.set_items_to_state(transition.to_items, transition.to_state) + end + @states << new_state + new_state.lalr_isocore = s + s.ielr_isocores << new_state + s.ielr_isocores.each do |st| + st.ielr_isocores = s.ielr_isocores + end + new_state.lookaheads_recomputed = true + new_state.item_lookahead_set = propagating_lookaheads + state.update_transition(transition, new_state) + elsif(!s.lookaheads_recomputed) + s.lookaheads_recomputed = true + s.item_lookahead_set = propagating_lookaheads + else + merge_lookaheads(s, propagating_lookaheads) + state.update_transition(transition, s) if state.items_to_state[transition.to_items].id != s.id + end + end + + # @rbs (Logger logger) -> void + def validate_conflicts_within_threshold!(logger) + exit false unless conflicts_within_threshold?(logger) + end + + # @rbs (Logger logger) -> bool + def conflicts_within_threshold?(logger) + return true unless @grammar.expect + + [sr_conflicts_within_threshold?(logger), rr_conflicts_within_threshold?(logger)].all? + end + + # @rbs (Logger logger) -> bool + def sr_conflicts_within_threshold?(logger) + return true if @grammar.expect == sr_conflicts_count + + logger.error("shift/reduce conflicts: #{sr_conflicts_count} found, #{@grammar.expect} expected") + false + end + + # @rbs (Logger logger) -> bool + def rr_conflicts_within_threshold?(logger, expected: 0) + return true if expected == rr_conflicts_count + + logger.error("reduce/reduce conflicts: #{rr_conflicts_count} found, #{expected} expected") + false + end + + # @rbs () -> void + def clear_look_ahead_sets + @direct_read_sets.clear + @reads_relation.clear + @read_sets.clear + @includes_relation.clear + @lookback_relation.clear + @follow_sets.clear + @la.clear + + @_direct_read_sets = nil + @_read_sets = nil + @_follow_sets = nil + @_la = nil + end + end +end diff --git a/tool/lrama/lib/lrama/tracer.rb b/tool/lrama/lib/lrama/tracer.rb new file mode 100644 index 0000000000..fda699a665 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer.rb @@ -0,0 +1,51 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "tracer/actions" +require_relative "tracer/closure" +require_relative "tracer/duration" +require_relative "tracer/only_explicit_rules" +require_relative "tracer/rules" +require_relative "tracer/state" + +module Lrama + class Tracer + # @rbs (IO io, **bool options) -> void + def initialize(io, **options) + @io = io + @options = options + @only_explicit_rules = OnlyExplicitRules.new(io, **options) + @rules = Rules.new(io, **options) + @actions = Actions.new(io, **options) + @closure = Closure.new(io, **options) + @state = State.new(io, **options) + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + @only_explicit_rules.trace(grammar) + @rules.trace(grammar) + @actions.trace(grammar) + end + + # @rbs (Lrama::State state) -> void + def trace_closure(state) + @closure.trace(state) + end + + # @rbs (Lrama::State state) -> void + def trace_state(state) + @state.trace(state) + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_state_list_append(state_count, state) + @state.trace_list_append(state_count, state) + end + + # @rbs () -> void + def enable_duration + Duration.enable if @options[:time] + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/actions.rb b/tool/lrama/lib/lrama/tracer/actions.rb new file mode 100644 index 0000000000..7b9c9b9f53 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/actions.rb @@ -0,0 +1,22 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Actions + # @rbs (IO io, ?actions: bool, **bool options) -> void + def initialize(io, actions: false, **options) + @io = io + @actions = actions + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @actions + + @io << "Grammar rules with actions:" << "\n" + grammar.rules.each { |rule| @io << rule.with_actions << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/closure.rb b/tool/lrama/lib/lrama/tracer/closure.rb new file mode 100644 index 0000000000..5b2f0b27e6 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/closure.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Closure + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @closure = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @closure + + @io << "Closure: input" << "\n" + state.kernels.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + @io << "Closure: output" << "\n" + state.items.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/duration.rb b/tool/lrama/lib/lrama/tracer/duration.rb new file mode 100644 index 0000000000..91c49625b2 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/duration.rb @@ -0,0 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + module Duration + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @_report_duration_enabled: bool + + # @rbs () -> void + def self.enable + @_report_duration_enabled = true + end + + # @rbs () -> bool + def self.enabled? + !!@_report_duration_enabled + end + + # @rbs [T] (_ToS message) { -> T } -> T + def report_duration(message) + time1 = Time.now.to_f + result = yield + time2 = Time.now.to_f + + if Duration.enabled? + STDERR.puts sprintf("%s %10.5f s", message, time2 - time1) + end + + return result + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb new file mode 100644 index 0000000000..4f64e7d2f4 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class OnlyExplicitRules + # @rbs (IO io, ?only_explicit: bool, **bool) -> void + def initialize(io, only_explicit: false, **_) + @io = io + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each do |rule| + @io << rule.display_name_without_action << "\n" if rule.lhs.first_set.any? + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/rules.rb b/tool/lrama/lib/lrama/tracer/rules.rb new file mode 100644 index 0000000000..d6e85b8432 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Rules + # @rbs (IO io, ?rules: bool, ?only_explicit: bool, **bool) -> void + def initialize(io, rules: false, only_explicit: false, **_) + @io = io + @rules = rules + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return if !@rules || @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each { |rule| @io << rule.display_name << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/state.rb b/tool/lrama/lib/lrama/tracer/state.rb new file mode 100644 index 0000000000..21c0047f8e --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/state.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class State + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @state = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @state + + # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but + # I think it is not correct... + previous = state.kernels.first.previous_sym + @io << "Processing state #{state.id} (reached by #{previous.display_name})" << "\n" + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_list_append(state_count, state) + return unless @state + + previous = state.kernels.first.previous_sym + @io << sprintf("state_list_append (state = %d, symbol = %d (%s))", + state_count, previous.number, previous.display_name) << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb new file mode 100644 index 0000000000..d649b74939 --- /dev/null +++ b/tool/lrama/lib/lrama/version.rb @@ -0,0 +1,6 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + VERSION = "0.7.1".freeze #: String +end diff --git a/tool/lrama/lib/lrama/warnings.rb b/tool/lrama/lib/lrama/warnings.rb new file mode 100644 index 0000000000..52f09144ef --- /dev/null +++ b/tool/lrama/lib/lrama/warnings.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'warnings/conflicts' +require_relative 'warnings/implicit_empty' +require_relative 'warnings/name_conflicts' +require_relative 'warnings/redefined_rules' +require_relative 'warnings/required' +require_relative 'warnings/useless_precedence' + +module Lrama + class Warnings + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @conflicts = Conflicts.new(logger, warnings) + @implicit_empty = ImplicitEmpty.new(logger, warnings) + @name_conflicts = NameConflicts.new(logger, warnings) + @redefined_rules = RedefinedRules.new(logger, warnings) + @required = Required.new(logger, warnings) + @useless_precedence = UselessPrecedence.new(logger, warnings) + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + @conflicts.warn(states) + @implicit_empty.warn(grammar) + @name_conflicts.warn(grammar) + @redefined_rules.warn(grammar) + @required.warn(grammar) + @useless_precedence.warn(grammar, states) + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/conflicts.rb b/tool/lrama/lib/lrama/warnings/conflicts.rb new file mode 100644 index 0000000000..6ba0de6f9c --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/conflicts.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Conflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::States states) -> void + def warn(states) + return unless @warnings + + if states.sr_conflicts_count != 0 + @logger.warn("shift/reduce conflicts: #{states.sr_conflicts_count} found") + end + + if states.rr_conflicts_count != 0 + @logger.warn("reduce/reduce conflicts: #{states.rr_conflicts_count} found") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/implicit_empty.rb b/tool/lrama/lib/lrama/warnings/implicit_empty.rb new file mode 100644 index 0000000000..ba81adca01 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/implicit_empty.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Empty rules are easily overlooked and ambiguous + # - Empty alternatives like `rule: | "token";` can be missed during code reading + # - Difficult to distinguish between intentional empty rules vs. omissions + # - Explicit marking with %empty directive comment improves clarity + class ImplicitEmpty + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.rule_builders.each do |builder| + if builder.rhs.empty? + @logger.warn("warning: empty rule without %empty") + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/name_conflicts.rb b/tool/lrama/lib/lrama/warnings/name_conflicts.rb new file mode 100644 index 0000000000..c0754ab551 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/name_conflicts.rb @@ -0,0 +1,63 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Parameterized rule names conflicting with symbol names + # - When a %rule name is identical to a terminal or non-terminal symbol name, + # it reduces grammar readability and may cause unintended behavior + # - Detecting these conflicts helps improve grammar definition quality + class NameConflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return if grammar.parameterized_rules.empty? + + symbol_names = collect_symbol_names(grammar) + check_conflicts(grammar.parameterized_rules, symbol_names) + end + + private + + # @rbs (Lrama::Grammar grammar) -> Set[String] + def collect_symbol_names(grammar) + symbol_names = Set.new + + collect_term_names(grammar.terms, symbol_names) + collect_nterm_names(grammar.nterms, symbol_names) + + symbol_names + end + + # @rbs (Array[untyped] terms, Set[String] symbol_names) -> void + def collect_term_names(terms, symbol_names) + terms.each do |term| + symbol_names.add(term.id.s_value) + symbol_names.add(term.alias_name) if term.alias_name + end + end + + # @rbs (Array[untyped] nterms, Set[String] symbol_names) -> void + def collect_nterm_names(nterms, symbol_names) + nterms.each do |nterm| + symbol_names.add(nterm.id.s_value) + end + end + + # @rbs (Array[untyped] parameterized_rules, Set[String] symbol_names) -> void + def check_conflicts(parameterized_rules, symbol_names) + parameterized_rules.each do |param_rule| + next unless symbol_names.include?(param_rule.name) + + @logger.warn("warning: parameterized rule name \"#{param_rule.name}\" conflicts with symbol name") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/redefined_rules.rb b/tool/lrama/lib/lrama/warnings/redefined_rules.rb new file mode 100644 index 0000000000..8ac2f1f103 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/redefined_rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class RedefinedRules + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.parameterized_resolver.redefined_rules.each do |rule| + @logger.warn("parameterized rule redefined: #{rule}") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/required.rb b/tool/lrama/lib/lrama/warnings/required.rb new file mode 100644 index 0000000000..4ab1ed787e --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/required.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Required + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings = false, **_) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + if grammar.required + @logger.warn("currently, %require is simply valid as a grammar but does nothing") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/useless_precedence.rb b/tool/lrama/lib/lrama/warnings/useless_precedence.rb new file mode 100644 index 0000000000..2913d6d7e5 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/useless_precedence.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class UselessPrecedence + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + return unless @warnings + + grammar.precedences.each do |precedence| + unless precedence.used_by? + @logger.warn("Precedence #{precedence.s_value} (line: #{precedence.lineno}) is defined but not used in any rule.") + end + end + end + end + end +end diff --git a/tool/lrama/template/bison/_yacc.h b/tool/lrama/template/bison/_yacc.h new file mode 100644 index 0000000000..3e270c9171 --- /dev/null +++ b/tool/lrama/template/bison/_yacc.h @@ -0,0 +1,79 @@ +<%# b4_shared_declarations -%> + <%-# b4_cpp_guard_open([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#ifndef <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> +# define <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> + <%- end -%> + <%-# b4_declare_yydebug & b4_YYDEBUG_define -%> +/* Debug traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif +#if YYDEBUG && !defined(yydebug) +extern int yydebug; +#endif +<%= output.percent_code("requires") %> + + <%-# b4_token_enums_defines -%> +/* Token kinds. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + enum yytokentype + { +<%= output.token_enums -%> + }; + typedef enum yytokentype yytoken_kind_t; +#endif + + <%-# b4_declare_yylstype -%> + <%-# b4_value_type_define -%> +/* Value type. */ +<% if output.grammar.union %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +union YYSTYPE +{ +#line <%= output.grammar.union.lineno %> "<%= output.grammar_file_path %>" +<%= output.grammar.union.braces_less_code %> +#line [@oline@] [@ofile@] + +}; +typedef union YYSTYPE YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% else %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef int YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% end %> + + <%-# b4_location_type_define -%> +/* Location type. */ +#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED +typedef struct YYLTYPE YYLTYPE; +struct YYLTYPE +{ + int first_line; + int first_column; + int last_line; + int last_column; +}; +# define YYLTYPE_IS_DECLARED 1 +# define YYLTYPE_IS_TRIVIAL 1 +#endif + + + + + <%-# b4_declare_yyerror_and_yylex. Not supported -%> + <%-# b4_declare_yyparse -%> +int yyparse (<%= output.parse_param %>); + + +<%= output.percent_code("provides") %> + <%-# b4_cpp_guard_close([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#endif /* !<%= output.b4_cpp_guard__b4_spec_mapped_header_file %> */ + <%- end -%> diff --git a/tool/lrama/template/bison/yacc.c b/tool/lrama/template/bison/yacc.c new file mode 100644 index 0000000000..6edd59a0d5 --- /dev/null +++ b/tool/lrama/template/bison/yacc.c @@ -0,0 +1,2068 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison implementation for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* C LALR(1) parser skeleton written by Richard Stallman, by + simplifying the original so-called "semantic" parser. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + +/* All symbols defined below should begin with yy or YY, to avoid + infringing on user name space. This should be done even for local + variables, as they might otherwise be expanded by user macros. + There are some unavoidable exceptions within include files to + define necessary library symbols; they are noted "INFRINGES ON + USER NAME SPACE" below. */ + +<%# b4_identification -%> +/* Identify Bison output, and Bison version. */ +#define YYBISON 30802 + +/* Bison version string. */ +#define YYBISON_VERSION "3.8.2" + +/* Skeleton name. */ +#define YYSKELETON_NAME "<%= output.template_basename %>" + +/* Pure parsers. */ +#define YYPURE 1 + +/* Push parsers. */ +#define YYPUSH 0 + +/* Pull parsers. */ +#define YYPULL 1 + + +<%# b4_user_pre_prologue -%> +<%- if output.aux.prologue -%> +/* First part of user prologue. */ +#line <%= output.aux.prologue_first_lineno %> "<%= output.grammar_file_path %>" +<%= output.aux.prologue %> +#line [@oline@] [@ofile@] +<%- end -%> + +<%# b4_cast_define -%> +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast<Type> (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast<Type> (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif +<%# b4_null_define -%> +# ifndef YY_NULLPTR +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif +# else +# define YY_NULLPTR ((void*)0) +# endif +# endif + +<%# b4_header_include_if -%> +<%- if output.include_header -%> +#include "<%= output.include_header %>" +<%- else -%> +/* Use api.header.include to #include this header + instead of duplicating it here. */ +<%= output.render_partial("bison/_yacc.h") %> +<%- end -%> +<%# b4_declare_symbol_enum -%> +/* Symbol kind. */ +enum yysymbol_kind_t +{ +<%= output.symbol_enum -%> +}; +typedef enum yysymbol_kind_t yysymbol_kind_t; + + + + +<%# b4_user_post_prologue -%> +<%# b4_c99_int_type_define -%> +#ifdef short +# undef short +#endif + +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + <limits.h> and (if available) <stdint.h> are included + so that the code can choose integer types of a good width. */ + +#ifndef __PTRDIFF_MAX__ +# include <limits.h> /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stdint.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif + +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ + +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; +#endif + +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; +#else +typedef short yytype_int16; +#endif + +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + <https://h20195.www2.hpe.com/V2/getpdf.aspx/4AA4-7673ENW.pdf>. */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 +#endif + +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; +#else +typedef short yytype_uint8; +#endif + +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; +#else +typedef int yytype_uint16; +#endif + +<%# b4_sizes_types_define -%> +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif +#endif + +#ifndef YYSIZE_T +# ifdef __SIZE_TYPE__ +# define YYSIZE_T __SIZE_TYPE__ +# elif defined size_t +# define YYSIZE_T size_t +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# define YYSIZE_T size_t +# else +# define YYSIZE_T unsigned +# endif +#endif + +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef <%= output.int_type_for([output.yynstates - 1]) %> yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include <libintl.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_(Msgid) dgettext ("bison-runtime", Msgid) +# endif +# endif +# ifndef YY_ +# define YY_(Msgid) Msgid +# endif +#endif + + +<%# b4_attribute_define -%> +#ifndef YY_ATTRIBUTE_PURE +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif +#endif + +#ifndef YY_ATTRIBUTE_UNUSED +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) +# else +# define YY_ATTRIBUTE_UNUSED +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YY_USE(E) ((void) (E)) +#else +# define YY_USE(E) /* empty */ +#endif + +/* Suppress an incorrect diagnostic about yylval being uninitialized. */ +#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__ +# if __GNUC__ * 100 + __GNUC_MINOR__ < 407 +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") +# else +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ + _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") +# endif +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ + _Pragma ("GCC diagnostic pop") +#else +# define YY_INITIAL_VALUE(Value) Value +#endif +#ifndef YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_END +#endif +#ifndef YY_INITIAL_VALUE +# define YY_INITIAL_VALUE(Value) /* Nothing. */ +#endif + +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + + +#define YY_ASSERT(E) ((void) (0 && (E))) + +#if 1 + +/* The parser invokes alloca or malloc; define the necessary symbols. */ + +# ifdef YYSTACK_USE_ALLOCA +# if YYSTACK_USE_ALLOCA +# ifdef __GNUC__ +# define YYSTACK_ALLOC __builtin_alloca +# elif defined __BUILTIN_VA_ARG_INCR +# include <alloca.h> /* INFRINGES ON USER NAME SPACE */ +# elif defined _AIX +# define YYSTACK_ALLOC __alloca +# elif defined _MSC_VER +# include <malloc.h> /* INFRINGES ON USER NAME SPACE */ +# define alloca _alloca +# else +# define YYSTACK_ALLOC alloca +# if ! defined _ALLOCA_H && ! defined EXIT_SUCCESS +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ + /* Use EXIT_SUCCESS as a witness for stdlib.h. */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# endif +# endif +# endif + +# ifdef YYSTACK_ALLOC + /* Pacify GCC's 'empty if-body' warning. */ +# define YYSTACK_FREE(Ptr) do { /* empty */; } while (0) +# ifndef YYSTACK_ALLOC_MAXIMUM + /* The OS might guarantee only one guard page at the bottom of the stack, + and a page size can be as small as 4096 bytes. So we cannot safely + invoke alloca (N) if N exceeds 4096. Use a slightly smaller number + to allow for a few compiler-allocated temporary stack slots. */ +# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +# endif +# else +# define YYSTACK_ALLOC YYMALLOC +# define YYSTACK_FREE YYFREE +# ifndef YYSTACK_ALLOC_MAXIMUM +# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +# endif +# if (defined __cplusplus && ! defined EXIT_SUCCESS \ + && ! ((defined YYMALLOC || defined malloc) \ + && (defined YYFREE || defined free))) +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# ifndef YYMALLOC +# define YYMALLOC malloc +# if ! defined malloc && ! defined EXIT_SUCCESS +void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# ifndef YYFREE +# define YYFREE free +# if ! defined free && ! defined EXIT_SUCCESS +void free (void *); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# endif +#endif /* 1 */ + +#if (! defined yyoverflow \ + && (! defined __cplusplus \ + || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL \ + && defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) + +/* A type that is properly aligned for any stack member. */ +union yyalloc +{ + yy_state_t yyss_alloc; + YYSTYPE yyvs_alloc; + YYLTYPE yyls_alloc; +}; + +/* The size of the maximum gap between one aligned stack and the next. */ +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) + +/* The size of an array large to enough to hold all stacks, each with + N elements. */ +# define YYSTACK_BYTES(N) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE) \ + + YYSIZEOF (YYLTYPE)) \ + + 2 * YYSTACK_GAP_MAXIMUM) + +# define YYCOPY_NEEDED 1 + +/* Relocate STACK from its old location to the new one. The + local variables YYSIZE and YYSTACKSIZE give the old and new number of + elements in the stack, and YYPTR gives the new location of the + stack. Advance YYPTR to a properly aligned location for the next + stack. */ +# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYPTRDIFF_T yynewbytes; \ + YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ + } \ + while (0) + +#endif + +#if defined YYCOPY_NEEDED && YYCOPY_NEEDED +/* Copy COUNT objects from SRC to DST. The source and destination do + not overlap. */ +# ifndef YYCOPY +# if defined __GNUC__ && 1 < __GNUC__ +# define YYCOPY(Dst, Src, Count) \ + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) +# else +# define YYCOPY(Dst, Src, Count) \ + do \ + { \ + YYPTRDIFF_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (Dst)[yyi] = (Src)[yyi]; \ + } \ + while (0) +# endif +# endif +#endif /* !YYCOPY_NEEDED */ + +/* YYFINAL -- State number of the termination state. */ +#define YYFINAL <%= output.yyfinal %> +/* YYLAST -- Last index in YYTABLE. */ +#define YYLAST <%= output.yylast %> + +/* YYNTOKENS -- Number of terminals. */ +#define YYNTOKENS <%= output.yyntokens %> +/* YYNNTS -- Number of nonterminals. */ +#define YYNNTS <%= output.yynnts %> +/* YYNRULES -- Number of rules. */ +#define YYNRULES <%= output.yynrules %> +/* YYNSTATES -- Number of states. */ +#define YYNSTATES <%= output.yynstates %> + +/* YYMAXUTOK -- Last valid token kind. */ +#define YYMAXUTOK <%= output.yymaxutok %> + + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) + +/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM + as returned by yylex. */ +static const <%= output.int_type_for(output.context.yytranslate) %> yytranslate[] = +{ +<%= output.yytranslate %> +}; + +<%- if output.error_recovery -%> +/* YYTRANSLATE_INVERTED[SYMBOL-NUM] -- Token number corresponding to SYMBOL-NUM */ +static const <%= output.int_type_for(output.context.yytranslate_inverted) %> yytranslate_inverted[] = +{ +<%= output.yytranslate_inverted %> +}; +<%- end -%> +#if YYDEBUG +/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ +static const <%= output.int_type_for(output.context.yyrline) %> yyrline[] = +{ +<%= output.yyrline %> +}; +#endif + +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if 1 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + +/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. + First, the terminals, then, starting at YYNTOKENS, nonterminals. */ +static const char *const yytname[] = +{ +<%= output.yytname %> +}; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} +#endif + +#define YYPACT_NINF (<%= output.yypact_ninf %>) + +#define yypact_value_is_default(Yyn) \ + <%= output.table_value_equals(output.context.yypact, "Yyn", output.yypact_ninf, "YYPACT_NINF") %> + +#define YYTABLE_NINF (<%= output.yytable_ninf %>) + +#define yytable_value_is_error(Yyn) \ + <%= output.table_value_equals(output.context.yytable, "Yyn", output.yytable_ninf, "YYTABLE_NINF") %> + +<%# b4_parser_tables_define -%> +/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + STATE-NUM. */ +static const <%= output.int_type_for(output.context.yypact) %> yypact[] = +{ +<%= output.int_array_to_string(output.context.yypact) %> +}; + +/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. + Performed when YYTABLE does not specify something else to do. Zero + means the default is an error. */ +static const <%= output.int_type_for(output.context.yydefact) %> yydefact[] = +{ +<%= output.int_array_to_string(output.context.yydefact) %> +}; + +/* YYPGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yypgoto) %> yypgoto[] = +{ +<%= output.int_array_to_string(output.context.yypgoto) %> +}; + +/* YYDEFGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yydefgoto) %> yydefgoto[] = +{ +<%= output.int_array_to_string(output.context.yydefgoto) %> +}; + +/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If + positive, shift that token. If negative, reduce the rule whose + number is the opposite. If YYTABLE_NINF, syntax error. */ +static const <%= output.int_type_for(output.context.yytable) %> yytable[] = +{ +<%= output.int_array_to_string(output.context.yytable) %> +}; + +static const <%= output.int_type_for(output.context.yycheck) %> yycheck[] = +{ +<%= output.int_array_to_string(output.context.yycheck) %> +}; + +/* YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of + state STATE-NUM. */ +static const <%= output.int_type_for(output.context.yystos) %> yystos[] = +{ +<%= output.int_array_to_string(output.context.yystos) %> +}; + +/* YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr1) %> yyr1[] = +{ +<%= output.int_array_to_string(output.context.yyr1) %> +}; + +/* YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr2) %> yyr2[] = +{ +<%= output.int_array_to_string(output.context.yyr2) %> +}; + + +enum { YYENOMEM = -2 }; + +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab +#define YYNOMEM goto yyexhaustedlab + + +#define YYRECOVERING() (!!yyerrstatus) + +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (<%= output.yyerror_args %>, YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF + +<%# b4_yylloc_default_define -%> +/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. + If N is 0, then set CURRENT to the empty location which ends + the previous symbol: RHS[0] (always defined). */ + +#ifndef YYLLOC_DEFAULT +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = \ + YYRHSLOC (Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC (Rhs, 0).last_column; \ + } \ + while (0) +#endif + +#define YYRHSLOC(Rhs, K) ((Rhs)[K]) + + +/* Enable debugging if requested. */ +#if YYDEBUG + +# ifndef YYFPRINTF +# include <stdio.h> /* INFRINGES ON USER NAME SPACE */ +# define YYFPRINTF fprintf +# endif + +# define YYDPRINTF(Args) \ +do { \ + if (yydebug) \ + YYFPRINTF Args; \ +} while (0) + + +<%# b4_yylocation_print_define -%> +/* YYLOCATION_PRINT -- Print the location on the stream. + This macro was not mandated originally: define only if we know + we won't break user code: when these are the locations we know. */ + +# ifndef YYLOCATION_PRINT + +# if defined YY_LOCATION_PRINT + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) YY_LOCATION_PRINT(File, *(Loc)<%= output.user_args %>) + +# elif defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + +/* Print *YYLOCP on YYO. Private, do not rely on its existence. */ + +YY_ATTRIBUTE_UNUSED +static int +yy_location_print_ (FILE *yyo, YYLTYPE const * const yylocp) +{ + int res = 0; + int end_col = 0 != yylocp->last_column ? yylocp->last_column - 1 : 0; + if (0 <= yylocp->first_line) + { + res += YYFPRINTF (yyo, "%d", yylocp->first_line); + if (0 <= yylocp->first_column) + res += YYFPRINTF (yyo, ".%d", yylocp->first_column); + } + if (0 <= yylocp->last_line) + { + if (yylocp->first_line < yylocp->last_line) + { + res += YYFPRINTF (yyo, "-%d", yylocp->last_line); + if (0 <= end_col) + res += YYFPRINTF (yyo, ".%d", end_col); + } + else if (0 <= end_col && yylocp->first_column < end_col) + res += YYFPRINTF (yyo, "-%d", end_col); + } + return res; +} + +# define YYLOCATION_PRINT yy_location_print_ + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT(File, Loc<%= output.user_args %>) YYLOCATION_PRINT(File, &(Loc)<%= output.user_args %>) + +# else + +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) ((void) 0) + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT YYLOCATION_PRINT + +# endif +# endif /* !defined YYLOCATION_PRINT */ + + +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) \ +do { \ + if (yydebug) \ + { \ + YYFPRINTF (stderr, "%s ", Title); \ + yy_symbol_print (stderr, \ + Kind, Value, Location<%= output.user_args %>); \ + YYFPRINTF (stderr, "\n"); \ + } \ +} while (0) + + +<%# b4_yy_symbol_print_define -%> +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ + +static void +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + FILE *yyoutput = yyo; +<%= output.parse_param_use("yyoutput", "yylocationp") %> + if (!yyvaluep) + return; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +<%# b4_symbol_actions(printer) -%> +switch (yykind) + { +<%= output.symbol_actions_for_printer -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ + +static void +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); + + YYLOCATION_PRINT (yyo, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ": "); + yy_symbol_value_print (yyo, yykind, yyvaluep, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ")"); +} + +/*------------------------------------------------------------------. +| yy_stack_print -- Print the state stack from its BOTTOM up to its | +| TOP (included). | +`------------------------------------------------------------------*/ + +static void +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop<%= output.user_formals %>) +{ + YYFPRINTF (stderr, "Stack now"); + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF (stderr, " %d", yybot); + } + YYFPRINTF (stderr, "\n"); +} + +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_stack_print ((Bottom), (Top)<%= output.user_args %>); \ +} while (0) + + +/*------------------------------------------------. +| Report that the YYRULE is going to be reduced. | +`------------------------------------------------*/ + +static void +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, YYLTYPE *yylsp, + int yyrule<%= output.user_formals %>) +{ + int yylno = yyrline[yyrule]; + int yynrhs = yyr2[yyrule]; + int yyi; + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", + yyrule - 1, yylno); + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF (stderr, " $%d = ", yyi + 1); + yy_symbol_print (stderr, + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)], + &(yylsp[(yyi + 1) - (yynrhs)])<%= output.user_args %>); + YYFPRINTF (stderr, "\n"); + } +} + +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_reduce_print (yyssp, yyvsp, yylsp, Rule<%= output.user_args %>); \ +} while (0) + +/* Nonzero means print parse trace. It is left uninitialized so that + multiple parsers can coexist. */ +#ifndef yydebug +int yydebug; +#endif +#else /* !YYDEBUG */ +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) +#endif /* !YYDEBUG */ + + +/* YYINITDEPTH -- initial size of the parser's stacks. */ +#ifndef YYINITDEPTH +# define YYINITDEPTH 200 +#endif + +/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only + if the built-in stack extension method is used). + + Do not make this value too large; the results are undefined if + YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) + evaluated with infinite-precision integer arithmetic. */ + +#ifndef YYMAXDEPTH +# define YYMAXDEPTH 10000 +#endif + + +/* Context of a parse error. */ +typedef struct +{ + yy_state_t *yyssp; + yysymbol_kind_t yytoken; + YYLTYPE *yylloc; +} yypcontext_t; + +/* Put in YYARG at most YYARGN of the expected tokens given the + current YYCTX, and return the number of tokens stored in YYARG. If + YYARG is null, return the number of expected tokens (guaranteed to + be less than YYNTOKENS). Return YYENOMEM on memory exhaustion. + Return 0 if there are more than YYARGN expected tokens, yet fill + YYARG up to YYARGN. */ +static int +yypcontext_expected_tokens (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + int yyn = yypact[+*yyctx->yyssp]; + if (!yypact_value_is_default (yyn)) + { + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. In other words, skip the first -YYN actions for + this state because they are default actions. */ + int yyxbegin = yyn < 0 ? -yyn : 0; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYSYMBOL_YYerror + && !yytable_value_is_error (yytable[yyx + yyn])) + { + if (!yyarg) + ++yycount; + else if (yycount == yyargn) + return 0; + else + yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx); + } + } + if (yyarg && yycount == 0 && 0 < yyargn) + yyarg[0] = YYSYMBOL_YYEMPTY; + return yycount; +} + + + + +#ifndef yystrlen +# if defined __GLIBC__ && defined _STRING_H +# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S))) +# else +/* Return the length of YYSTR. */ +static YYPTRDIFF_T +yystrlen (const char *yystr) +{ + YYPTRDIFF_T yylen; + for (yylen = 0; yystr[yylen]; yylen++) + continue; + return yylen; +} +# endif +#endif + +#ifndef yystpcpy +# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +# define yystpcpy stpcpy +# else +/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in + YYDEST. */ +static char * +yystpcpy (char *yydest, const char *yysrc) +{ + char *yyd = yydest; + const char *yys = yysrc; + + while ((*yyd++ = *yys++) != '\0') + continue; + + return yyd - 1; +} +# endif +#endif + +#ifndef yytnamerr +/* Copy to YYRES the contents of YYSTR after stripping away unnecessary + quotes and backslashes, so that it's suitable for yyerror. The + heuristic is that double-quoting is unnecessary unless the string + contains an apostrophe, a comma, or backslash (other than + backslash-backslash). YYSTR is taken from yytname. If YYRES is + null, do not copy; instead, return the length of what the result + would have been. */ +static YYPTRDIFF_T +yytnamerr (char *yyres, const char *yystr) +{ + if (*yystr == '"') + { + YYPTRDIFF_T yyn = 0; + char const *yyp = yystr; + for (;;) + switch (*++yyp) + { + case '\'': + case ',': + goto do_not_strip_quotes; + + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; + else + goto append; + + append: + default: + if (yyres) + yyres[yyn] = *yyp; + yyn++; + break; + + case '"': + if (yyres) + yyres[yyn] = '\0'; + return yyn; + } + do_not_strip_quotes: ; + } + + if (yyres) + return yystpcpy (yyres, yystr) - yyres; + else + return yystrlen (yystr); +} +#endif + + +static int +yy_syntax_error_arguments (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + /* There are many possibilities here to consider: + - If this state is a consistent state with a default action, then + the only way this function was invoked is if the default action + is an error action. In that case, don't check for expected + tokens because there are none. + - The only way there can be no lookahead present (in yychar) is if + this state is a consistent state with a default action. Thus, + detecting the absence of a lookahead is sufficient to determine + that there is no unexpected or expected token to report. In that + case, just report a simple "syntax error". + - Don't assume there isn't a lookahead just because this state is a + consistent state with a default action. There might have been a + previous inconsistent state, consistent state with a non-default + action, or user semantic action that manipulated yychar. + - Of course, the expected token list depends on states to have + correct lookahead information, and it depends on the parser not + to perform extra reductions after fetching a lookahead from the + scanner and before detecting a syntax error. Thus, state merging + (from LALR or IELR) and default reductions corrupt the expected + token list. However, the list is correct for canonical LR with + one exception: it will still contain any token that will not be + accepted due to an error action in a later state. + */ + if (yyctx->yytoken != YYSYMBOL_YYEMPTY) + { + int yyn; + if (yyarg) + yyarg[yycount] = yyctx->yytoken; + ++yycount; + yyn = yypcontext_expected_tokens (yyctx, + yyarg ? yyarg + 1 : yyarg, yyargn - 1); + if (yyn == YYENOMEM) + return YYENOMEM; + else + yycount += yyn; + } + return yycount; +} + +/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message + about the unexpected token YYTOKEN for the state stack whose top is + YYSSP. + + Return 0 if *YYMSG was successfully written. Return -1 if *YYMSG is + not large enough to hold the message. In that case, also set + *YYMSG_ALLOC to the required number of bytes. Return YYENOMEM if the + required number of bytes is too large to store. */ +static int +yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg, + const yypcontext_t *yyctx<%= output.user_formals %>) +{ + enum { YYARGS_MAX = 5 }; + /* Internationalized format string. */ + const char *yyformat = YY_NULLPTR; + /* Arguments of yyformat: reported tokens (one for the "unexpected", + one per "expected"). */ + yysymbol_kind_t yyarg[YYARGS_MAX]; + /* Cumulated lengths of YYARG. */ + YYPTRDIFF_T yysize = 0; + + /* Actual size of YYARG. */ + int yycount = yy_syntax_error_arguments (yyctx, yyarg, YYARGS_MAX); + if (yycount == YYENOMEM) + return YYENOMEM; + + switch (yycount) + { +#define YYCASE_(N, S) \ + case N: \ + yyformat = S; \ + break + default: /* Avoid compiler warnings. */ + YYCASE_(0, YY_("syntax error")); + YYCASE_(1, YY_("syntax error, unexpected %s")); + YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); + YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s")); + YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); + YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); +#undef YYCASE_ + } + + /* Compute error message size. Don't count the "%s"s, but reserve + room for the terminator. */ + yysize = yystrlen (yyformat) - 2 * yycount + 1; + { + int yyi; + for (yyi = 0; yyi < yycount; ++yyi) + { + YYPTRDIFF_T yysize1 + = yysize + yytnamerr (YY_NULLPTR, yytname[yyarg[yyi]]); + if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM) + yysize = yysize1; + else + return YYENOMEM; + } + } + + if (*yymsg_alloc < yysize) + { + *yymsg_alloc = 2 * yysize; + if (! (yysize <= *yymsg_alloc + && *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM)) + *yymsg_alloc = YYSTACK_ALLOC_MAXIMUM; + return -1; + } + + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + { + char *yyp = *yymsg; + int yyi = 0; + while ((*yyp = *yyformat) != '\0') + if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount) + { + yyp += yytnamerr (yyp, yytname[yyarg[yyi++]]); + yyformat += 2; + } + else + { + ++yyp; + ++yyformat; + } + } + return 0; +} + +<%# b4_yydestruct_define %> +/*-----------------------------------------------. +| Release the memory associated to this symbol. | +`-----------------------------------------------*/ + +static void +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep, YYLTYPE *yylocationp<%= output.user_formals %>) +{ +<%= output.parse_param_use("yyvaluep", "yylocationp") %> + if (!yymsg) + yymsg = "Deleting"; + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp<%= output.user_args %>); + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + switch (yykind) + { +<%= output.symbol_actions_for_destructor -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + + +<%- if output.error_recovery -%> +#ifndef YYMAXREPAIR +# define YYMAXREPAIR(<%= output.parse_param_name %>) (3) +#endif + +#ifndef YYERROR_RECOVERY_ENABLED +# define YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>) (1) +#endif + +enum yy_repair_type { + inserting, + deleting, + shifting, +}; + +struct yy_repair { + enum yy_repair_type type; + yysymbol_kind_t term; +}; +typedef struct yy_repair yy_repair; + +struct yy_repairs { + /* For debug */ + int id; + /* For breadth-first traversing */ + struct yy_repairs *next; + YYPTRDIFF_T stack_length; + /* Bottom of states */ + yy_state_t *states; + /* Top of states */ + yy_state_t *state; + /* repair length */ + int repair_length; + /* */ + struct yy_repairs *prev_repair; + struct yy_repair repair; +}; +typedef struct yy_repairs yy_repairs; + +struct yy_term { + yysymbol_kind_t kind; + YYSTYPE value; + YYLTYPE location; +}; +typedef struct yy_term yy_term; + +struct yy_repair_terms { + int id; + int length; + yy_term terms[]; +}; +typedef struct yy_repair_terms yy_repair_terms; + +static void +yy_error_token_initialize (yysymbol_kind_t yykind, YYSTYPE * const yyvaluep, YYLTYPE * const yylocationp<%= output.user_formals %>) +{ + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +switch (yykind) + { +<%= output.symbol_actions_for_error_token -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + +static yy_repair_terms * +yy_create_repair_terms(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + yy_repair_terms *rep_terms; + int count = 0; + + while (r->prev_repair) + { + count++; + r = r->prev_repair; + } + + rep_terms = (yy_repair_terms *) YYMALLOC (sizeof (yy_repair_terms) + sizeof (yy_term) * count); + rep_terms->id = reps->id; + rep_terms->length = count; + + r = reps; + while (r->prev_repair) + { + rep_terms->terms[count-1].kind = r->repair.term; + count--; + r = r->prev_repair; + } + + return rep_terms; +} + +static void +yy_print_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + + YYDPRINTF ((stderr, + "id: %d, repair_length: %d, repair_state: %d, prev_repair_id: %d\n", + reps->id, reps->repair_length, *reps->state, reps->prev_repair->id)); + + while (r->prev_repair) + { + YYDPRINTF ((stderr, "%s ", yysymbol_name (r->repair.term))); + r = r->prev_repair; + } + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_print_repair_terms(yy_repair_terms *rep_terms<%= output.user_formals %>) +{ + for (int i = 0; i < rep_terms->length; i++) + YYDPRINTF ((stderr, "%s ", yysymbol_name (rep_terms->terms[i].kind))); + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_free_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + while (reps) + { + yy_repairs *r = reps; + reps = reps->next; + YYFREE (r->states); + YYFREE (r); + } +} + +static int +yy_process_repairs(yy_repairs *reps, yysymbol_kind_t token) +{ + int yyn; + int yystate = *reps->state; + int yylen = 0; + yysymbol_kind_t yytoken = token; + + goto yyrecover_backup; + +yyrecover_newstate: + // TODO: check reps->stack_length + reps->state += 1; + *reps->state = (yy_state_t) yystate; + + +yyrecover_backup: + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yyrecover_default; + + /* "Reading a token" */ + if (yytoken == YYSYMBOL_YYEMPTY) + return 1; + + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yyrecover_default; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyrecover_errlab; + yyn = -yyn; + goto yyrecover_reduce; + } + + /* shift */ + yystate = yyn; + yytoken = YYSYMBOL_YYEMPTY; + goto yyrecover_newstate; + + +yyrecover_default: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyrecover_errlab; + goto yyrecover_reduce; + + +yyrecover_reduce: + yylen = yyr2[yyn]; + /* YYPOPSTACK */ + reps->state -= yylen; + yylen = 0; + + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *reps->state; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *reps->state + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yyrecover_newstate; + +yyrecover_errlab: + return 0; +} + +static yy_repair_terms * +yyrecover(yy_state_t *yyss, yy_state_t *yyssp, int yychar<%= output.user_formals %>) +{ + yysymbol_kind_t yytoken = YYTRANSLATE (yychar); + yy_repair_terms *rep_terms = YY_NULLPTR; + int count = 0; + + yy_repairs *head = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + yy_repairs *current = head; + yy_repairs *tail = head; + YYPTRDIFF_T stack_length = yyssp - yyss + 1; + + head->id = count; + head->next = 0; + head->stack_length = stack_length; + head->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + head->state = head->states + (yyssp - yyss); + YYCOPY (head->states, yyss, stack_length); + head->repair_length = 0; + head->prev_repair = 0; + + stack_length = (stack_length * 2 > 100) ? (stack_length * 2) : 100; + count++; + + while (current) + { + int yystate = *current->state; + int yyn = yypact[yystate]; + /* See also: yypcontext_expected_tokens */ + if (!yypact_value_is_default (yyn)) + { + int yyxbegin = yyn < 0 ? -yyn : 0; + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + { + if (yyx != YYSYMBOL_YYerror) + { + if (current->repair_length + 1 > YYMAXREPAIR(<%= output.parse_param_name %>)) + continue; + + yy_repairs *reps = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + reps->id = count; + reps->next = 0; + reps->stack_length = stack_length; + reps->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + reps->state = reps->states + (current->state - current->states); + YYCOPY (reps->states, current->states, current->state - current->states + 1); + reps->repair_length = current->repair_length + 1; + reps->prev_repair = current; + reps->repair.type = inserting; + reps->repair.term = (yysymbol_kind_t) yyx; + + /* Process PDA assuming next token is yyx */ + if (! yy_process_repairs (reps, (yysymbol_kind_t)yyx)) + { + YYFREE (reps); + continue; + } + + tail->next = reps; + tail = reps; + count++; + + if (yyx == yytoken) + { + rep_terms = yy_create_repair_terms (current<%= output.user_args %>); + YYDPRINTF ((stderr, "repair_terms found. id: %d, length: %d\n", rep_terms->id, rep_terms->length)); + yy_print_repairs (current<%= output.user_args %>); + yy_print_repair_terms (rep_terms<%= output.user_args %>); + + goto done; + } + + YYDPRINTF ((stderr, + "New repairs is enqueued. count: %d, yystate: %d, yyx: %d\n", + count, yystate, yyx)); + yy_print_repairs (reps<%= output.user_args %>); + } + } + } + + current = current->next; + } + +done: + + yy_free_repairs(head<%= output.user_args %>); + + if (!rep_terms) + { + YYDPRINTF ((stderr, "repair_terms not found\n")); + } + + return rep_terms; +} +<%- end -%> + + + +/*----------. +| yyparse. | +`----------*/ + +int +yyparse (<%= output.parse_param %>) +{ +<%# b4_declare_scanner_communication_variables -%> +/* Lookahead token kind. */ +int yychar; + + +/* The semantic value of the lookahead symbol. */ +/* Default value used for initialization, for pacifying older GCCs + or non-GCC compilers. */ +#ifdef __cplusplus +static const YYSTYPE yyval_default = {}; +(void) yyval_default; +#else +YY_INITIAL_VALUE (static const YYSTYPE yyval_default;) +#endif +YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); + +/* Location data for the lookahead symbol. */ +static const YYLTYPE yyloc_default +# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + = { 1, 1, 1, 1 } +# endif +; +YYLTYPE yylloc = yyloc_default; + +<%# b4_declare_parser_state_variables -%> + /* Number of syntax errors so far. */ + int yynerrs = 0; + YY_USE (yynerrs); /* Silence compiler warning. */ + + yy_state_fast_t yystate = 0; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus = 0; + + /* Refer to the stacks through separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ + + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; + + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; + + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; + + /* The location stack: array, bottom, top. */ + YYLTYPE yylsa[YYINITDEPTH]; + YYLTYPE *yyls = yylsa; + YYLTYPE *yylsp = yyls; + + int yyn; + /* The return value of yyparse. */ + int yyresult; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; + YYLTYPE yyloc; + + /* The locations where the error started and ended. */ + YYLTYPE yyerror_range[3]; +<%- if output.error_recovery -%> + yy_repair_terms *rep_terms = 0; + yy_term term_backup; + int rep_terms_index; + int yychar_backup; +<%- end -%> + + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char *yymsg = yymsgbuf; + YYPTRDIFF_T yymsg_alloc = sizeof yymsgbuf; + +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N), yylsp -= (N)) + + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; + + YYDPRINTF ((stderr, "Starting parse\n")); + + yychar = YYEMPTY; /* Cause a token to be read. */ + + +<%# b4_user_initial_action -%> +<%= output.user_initial_action("/* User initialization code. */") %> +#line [@oline@] [@ofile@] + + yylsp[0] = yylloc; + goto yysetstate; + + +/*------------------------------------------------------------. +| yynewstate -- push a new state, which is found in yystate. | +`------------------------------------------------------------*/ +yynewstate: + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; + + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + + if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + YYNOMEM; +#else + { + /* Get the current used size of the three stacks, in elements. */ + YYPTRDIFF_T yysize = yyssp - yyss + 1; + +# if defined yyoverflow + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + yy_state_t *yyss1 = yyss; + YYSTYPE *yyvs1 = yyvs; + YYLTYPE *yyls1 = yyls; + + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow (YY_("memory exhausted"), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), + &yyls1, yysize * YYSIZEOF (*yylsp), + &yystacksize); + yyss = yyss1; + yyvs = yyvs1; + yyls = yyls1; + } +# else /* defined YYSTACK_RELOCATE */ + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + YYNOMEM; + yystacksize *= 2; + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yy_state_t *yyss1 = yyss; + union yyalloc *yyptr = + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); + if (! yyptr) + YYNOMEM; + YYSTACK_RELOCATE (yyss_alloc, yyss); + YYSTACK_RELOCATE (yyvs_alloc, yyvs); + YYSTACK_RELOCATE (yyls_alloc, yyls); +# undef YYSTACK_RELOCATE + if (yyss1 != yyssa) + YYSTACK_FREE (yyss1); + } +# endif + + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; + yylsp = yyls + yysize - 1; + + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END + + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ + + + if (yystate == YYFINAL) + YYACCEPT; + + goto yybackup; + + +/*-----------. +| yybackup. | +`-----------*/ +yybackup: + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ + + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yydefault; + + /* Not known => get a lookahead token if don't already have one. */ + +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + if (yychar == YYEMPTY && rep_terms) + { + + if (rep_terms_index < rep_terms->length) + { + YYDPRINTF ((stderr, "An error recovery token is used\n")); + yy_term term = rep_terms->terms[rep_terms_index]; + yytoken = term.kind; + yylval = term.value; + yylloc = term.location; + yychar = yytranslate_inverted[yytoken]; + YY_SYMBOL_PRINT ("Next error recovery token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + rep_terms_index++; + } + else + { + YYDPRINTF ((stderr, "Error recovery is completed\n")); + yytoken = term_backup.kind; + yylval = term_backup.value; + yylloc = term_backup.location; + yychar = yychar_backup; + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + + YYFREE (rep_terms); + rep_terms = 0; + yychar_backup = 0; + } + } + } +<%- end -%> + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token\n")); + yychar = yylex <%= output.yylex_formals %>; + } + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + yychar = <%= output.eof_symbol.id.s_value %>; + yytoken = <%= output.eof_symbol.enum_name %>; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else if (yychar == <%= output.error_symbol.id.s_value %>) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = <%= output.undef_symbol.id.s_value %>; + yytoken = <%= output.error_symbol.enum_name %>; + yyerror_range[1] = yylloc; + goto yyerrlab1; + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyerrlab; + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc<%= output.user_args %>); + yystate = yyn; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + *++yylsp = yylloc; +<%= output.after_shift_function("/* %after-shift code. */") %> + + /* Discard the shifted token. */ + yychar = YYEMPTY; + goto yynewstate; + + +/*-----------------------------------------------------------. +| yydefault -- do the default action for the current state. | +`-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + +/*-----------------------------. +| yyreduce -- do a reduction. | +`-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + '$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1-yylen]; +<%= output.before_reduce_function("/* %before-reduce function. */") %> + + /* Default location. */ + YYLLOC_DEFAULT (yyloc, (yylsp - yylen), yylen); + yyerror_range[1] = yyloc; + YY_REDUCE_PRINT (yyn<%= output.user_args %>); + switch (yyn) + { +<%= output.user_actions -%> + + default: break; + } + /* User semantic actions sometimes alter yychar, and that requires + that yytoken be updated with the new translation. We take the + approach of translating immediately before every use of yytoken. + One alternative is translating here after every semantic action, + but that translation would be missed if the semantic action invokes + YYABORT, YYACCEPT, or YYERROR immediately after altering yychar or + if it invokes YYBACKUP. In the case of YYABORT or YYACCEPT, an + incorrect destructor might then be invoked immediately. In the + case of YYERROR or YYBACKUP, subsequent parser actions might lead + to an incorrect destructor call or verbose syntax error message + before the lookahead is translated. */ + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc<%= output.user_args %>); + + YYPOPSTACK (yylen); +<%= output.after_reduce_function("/* %after-reduce function. */") %> + yylen = 0; + + *++yyvsp = yyval; + *++yylsp = yyloc; + + /* Now 'shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yynewstate; + + +/*--------------------------------------. +| yyerrlab -- here on detecting error. | +`--------------------------------------*/ +yyerrlab: + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; + { + yypcontext_t yyctx + = {yyssp, yytoken, &yylloc}; + char const *yymsgp = YY_("syntax error"); + int yysyntax_error_status; + yysyntax_error_status = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + if (yysyntax_error_status == 0) + yymsgp = yymsg; + else if (yysyntax_error_status == -1) + { + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + yymsg = YY_CAST (char *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, yymsg_alloc))); + if (yymsg) + { + yysyntax_error_status + = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + yymsgp = yymsg; + } + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + yysyntax_error_status = YYENOMEM; + } + } + yyerror (<%= output.yyerror_args %>, yymsgp); + if (yysyntax_error_status == YYENOMEM) + YYNOMEM; + } + } + + yyerror_range[1] = yylloc; + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + /* Return failure if at end of input. */ + if (yychar == <%= output.eof_symbol.id.s_value %>) + YYABORT; + } + else + { + yydestruct ("Error: discarding", + yytoken, &yylval, &yylloc<%= output.user_args %>); + yychar = YYEMPTY; + } + } + + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; + + +/*---------------------------------------------------. +| yyerrorlab -- error raised explicitly by YYERROR. | +`---------------------------------------------------*/ +yyerrorlab: + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; + ++yynerrs; + + /* Do not reclaim the symbols of the rule whose action triggered + this YYERROR. */ + YYPOPSTACK (yylen); +<%= output.after_pop_stack_function("yylen", "/* %after-pop-stack function. */") %> + yylen = 0; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + yystate = *yyssp; + goto yyerrlab1; + + +/*-------------------------------------------------------------. +| yyerrlab1 -- common code for both syntax error and YYERROR. | +`-------------------------------------------------------------*/ +yyerrlab1: +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + rep_terms = yyrecover (yyss, yyssp, yychar<%= output.user_args %>); + if (rep_terms) + { + for (int i = 0; i < rep_terms->length; i++) + { + yy_term *term = &rep_terms->terms[i]; + yy_error_token_initialize (term->kind, &term->value, &term->location<%= output.user_args %>); + } + + yychar_backup = yychar; + /* Can be packed into (the tail of) rep_terms? */ + term_backup.kind = yytoken; + term_backup.value = yylval; + term_backup.location = yylloc; + rep_terms_index = 0; + yychar = YYEMPTY; + + goto yybackup; + } + } +<%- end -%> + yyerrstatus = 3; /* Each real token shifted decrements this. */ + + /* Pop stack until we find a state that shifts the error token. */ + for (;;) + { + yyn = yypact[yystate]; + if (!yypact_value_is_default (yyn)) + { + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) + { + yyn = yytable[yyn]; + if (0 < yyn) + break; + } + } + + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + + yyerror_range[1] = *yylsp; + yydestruct ("Error: popping", + YY_ACCESSING_SYMBOL (yystate), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); +<%= output.after_pop_stack_function(1, "/* %after-pop-stack function. */") %> + yystate = *yyssp; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + } + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + + yyerror_range[2] = yylloc; + ++yylsp; + YYLLOC_DEFAULT (*yylsp, yyerror_range, 2); + + /* Shift the error token. */ + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp<%= output.user_args %>); +<%= output.after_shift_error_token_function("/* %after-shift-error-token code. */") %> + + yystate = yyn; + goto yynewstate; + + +/*-------------------------------------. +| yyacceptlab -- YYACCEPT comes here. | +`-------------------------------------*/ +yyacceptlab: + yyresult = 0; + goto yyreturnlab; + + +/*-----------------------------------. +| yyabortlab -- YYABORT comes here. | +`-----------------------------------*/ +yyabortlab: + yyresult = 1; + goto yyreturnlab; + + +/*-----------------------------------------------------------. +| yyexhaustedlab -- YYNOMEM (memory exhaustion) comes here. | +`-----------------------------------------------------------*/ +yyexhaustedlab: + yyerror (<%= output.yyerror_args %>, YY_("memory exhausted")); + yyresult = 2; + goto yyreturnlab; + + +/*----------------------------------------------------------. +| yyreturnlab -- parsing is finished, clean up and return. | +`----------------------------------------------------------*/ +yyreturnlab: + if (yychar != YYEMPTY) + { + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = YYTRANSLATE (yychar); + yydestruct ("Cleanup: discarding lookahead", + yytoken, &yylval, &yylloc<%= output.user_args %>); + } + /* Do not reclaim the symbols of the rule whose action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK (yylen); + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + while (yyssp != yyss) + { + yydestruct ("Cleanup: popping", + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); + } +#ifndef yyoverflow + if (yyss != yyssa) + YYSTACK_FREE (yyss); +#endif + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + return yyresult; +} + +<%# b4_percent_code_get([[epilogue]]) -%> +<%- if output.aux.epilogue -%> +#line <%= output.aux.epilogue_first_lineno - 1 %> "<%= output.grammar_file_path %>" +<%= output.aux.epilogue -%> +<%- end -%> + diff --git a/tool/lrama/template/bison/yacc.h b/tool/lrama/template/bison/yacc.h new file mode 100644 index 0000000000..848dbf5961 --- /dev/null +++ b/tool/lrama/template/bison/yacc.h @@ -0,0 +1,40 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison interface for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ +<%= output.render_partial("bison/_yacc.h") %> diff --git a/tool/lrama/template/diagram/diagram.html b/tool/lrama/template/diagram/diagram.html new file mode 100644 index 0000000000..3e87e6e519 --- /dev/null +++ b/tool/lrama/template/diagram/diagram.html @@ -0,0 +1,102 @@ +<!DOCTYPE html> +<html> +<head> + <title>Lrama syntax diagrams</title> + + <style> + <%= output.default_style %> + .diagram-header { + display: inline-block; + font-weight: bold; + font-size: 18px; + margin-bottom: -8px; + text-align: center; + } + + svg { + width: 100%; + } + + svg.railroad-diagram g.non-terminal text { + cursor: pointer; + } + + h2.hover-header { + background-color: #90ee90; + } + + svg.railroad-diagram g.non-terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + + svg.railroad-diagram g.terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + </style> +</head> + +<body align="center"> + <%= output.diagrams %> + <script> + document.addEventListener("DOMContentLoaded", () => { + function addHoverEffect(selector, hoverClass, relatedSelector, relatedHoverClass, getTextElements) { + document.querySelectorAll(selector).forEach(element => { + element.addEventListener("mouseenter", () => { + element.classList.add(hoverClass); + getTextElements(element).forEach(textEl => { + if (!relatedSelector) return; + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.add(relatedHoverClass); + }); + }); + }); + + element.addEventListener("mouseleave", () => { + element.classList.remove(hoverClass); + if (!relatedSelector) return; + getTextElements(element).forEach(textEl => { + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.remove(relatedHoverClass); + }); + }); + }); + }); + } + + function getElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => el.textContent.trim() === text.trim()); + } + + function getParentElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => + [...el.querySelectorAll("text")].some(textEl => textEl.textContent.trim() === text.trim()) + ); + } + + function scrollToMatchingHeader() { + document.querySelectorAll("g.non-terminal").forEach(element => { + element.addEventListener("click", () => { + const textElements = [...element.querySelectorAll("text")]; + for (const textEl of textElements) { + const targetHeader = getElementsByText("h2", textEl.textContent)[0]; + if (targetHeader) { + targetHeader.scrollIntoView({ behavior: "smooth", block: "start" }); + break; + } + } + }); + }); + } + + addHoverEffect("h2", "hover-header", "g.non-terminal", "hover-g", element => [element]); + addHoverEffect("g.non-terminal", "hover-g", "h2", "hover-header", + element => [...element.querySelectorAll("text")] + ); + addHoverEffect("g.terminal", "hover-g", "", "", element => [element]); + scrollToMatchingHeader(); + }); + </script> +</body> +</html> diff --git a/tool/m4/ruby_append_option.m4 b/tool/m4/ruby_append_option.m4 index ff828d2162..8cd2741ae8 100644 --- a/tool/m4/ruby_append_option.m4 +++ b/tool/m4/ruby_append_option.m4 @@ -3,3 +3,7 @@ AC_DEFUN([RUBY_APPEND_OPTION], [# RUBY_APPEND_OPTION($1) AS_CASE([" [$]{$1-} "], [*" $2 "*], [], [' '], [ $1="$2"], [ $1="[$]$1 $2"])])dnl +AC_DEFUN([RUBY_PREPEND_OPTION], + [# RUBY_PREPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_check_builtin_overflow.m4 b/tool/m4/ruby_check_builtin_overflow.m4 new file mode 100644 index 0000000000..8568d2c6d9 --- /dev/null +++ b/tool/m4/ruby_check_builtin_overflow.m4 @@ -0,0 +1,28 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_OVERFLOW], [dnl +{ # $0($1) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow, [int x;__builtin_[$1]_overflow(0,0,&x)]) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow_p, [__builtin_[$1]_overflow_p(0,0,(int)0)]) + + AS_IF([test "$rb_cv_builtin___builtin_[$1]_overflow" != no], [ + AC_CACHE_CHECK(for __builtin_[$1]_overflow with long long arguments, rb_cv_use___builtin_[$1]_overflow_long_long, [ + AC_LINK_IFELSE([AC_LANG_SOURCE([[ +@%:@pragma clang optimize off + +int +main(void) +{ + long long x = 0, y; + __builtin_$1_overflow(x, x, &y); + + return 0; +} +]])], + rb_cv_use___builtin_[$1]_overflow_long_long=yes, + rb_cv_use___builtin_[$1]_overflow_long_long=no)]) + ]) + AS_IF([test "$rb_cv_use___builtin_[$1]_overflow_long_long" = yes], [ + AC_DEFINE(USE___BUILTIN_[]AS_TR_CPP($1)_OVERFLOW_LONG_LONG, 1) + ]) +} +])dnl diff --git a/tool/m4/ruby_check_builtin_setjmp.m4 b/tool/m4/ruby_check_builtin_setjmp.m4 index 05118e2243..1e5d9b3028 100644 --- a/tool/m4/ruby_check_builtin_setjmp.m4 +++ b/tool/m4/ruby_check_builtin_setjmp.m4 @@ -20,7 +20,7 @@ AC_CACHE_CHECK(for __builtin_setjmp, ac_cv_func___builtin_setjmp, void (*volatile f)(void) = t; if (!jump()) printf("%d\n", f != 0); ]])], - [ac_cv_func___builtin_setjmp="yes with cast ($cast)"]) + [ac_cv_func___builtin_setjmp="yes${cast:+ with cast ($cast)}"]) ]) test "$ac_cv_func___builtin_setjmp" = no || break done]) diff --git a/tool/m4/ruby_check_header.m4 b/tool/m4/ruby_check_header.m4 new file mode 100644 index 0000000000..6fec9d16c5 --- /dev/null +++ b/tool/m4/ruby_check_header.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_HEADER], + [# RUBY_CHECK_HEADER($@) + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS m4_if([$5], [], [$INCFLAGS], [$5])" + AC_CHECK_HEADERS([$1], [$2], [$3], [$4]) + CPPFLAGS="$save_CPPFLAGS" + unset save_CPPFLAGS]) diff --git a/tool/m4/ruby_default_arch.m4 b/tool/m4/ruby_default_arch.m4 index 35eb8112f6..2f25ba81ee 100644 --- a/tool/m4/ruby_default_arch.m4 +++ b/tool/m4/ruby_default_arch.m4 @@ -1,12 +1,21 @@ dnl -*- Autoconf -*- AC_DEFUN([RUBY_DEFAULT_ARCH], [ +# Set ARCH_FLAG for different width but family CPU AC_MSG_CHECKING([arch option]) -AS_CASE([$1], - [arm64], [], - [*64], [ARCH_FLAG=-m64], - [[i[3-6]86]], [ARCH_FLAG=-m32], - [ppc], [ARCH_FLAG=-m32], - [AC_MSG_ERROR(unknown target architecture: $target_archs)] - ) -AC_MSG_RESULT([$ARCH_FLAG]) +AS_CASE([$1:"$host_cpu"], + [arm64:arm*], [ARCH_FLAG=-m64], + [arm*:arm*], [ARCH_FLAG=-m32], + [x86_64:[i[3-6]86]], [ARCH_FLAG=-m64], + [x64:x86_64], [], + [[i[3-6]86]:x86_64], [ARCH_FLAG=-m32], + [ppc64:ppc*], [ARCH_FLAG=-m64], + [ppc*:ppc64], [ARCH_FLAG=-m32], + [ + ARCH_FLAG= + for flag in "-arch "$1 -march=$1; do + _RUBY_TRY_CFLAGS([$]flag, [ARCH_FLAG="[$]flag"]) + test x"$ARCH_FLAG" = x || break + done] +) +AC_MSG_RESULT([${ARCH_FLAG:-'(none)'}]) ])dnl diff --git a/tool/m4/ruby_defint.m4 b/tool/m4/ruby_defint.m4 index e9ed68e5b8..7f262a73fc 100644 --- a/tool/m4/ruby_defint.m4 +++ b/tool/m4/ruby_defint.m4 @@ -17,7 +17,8 @@ typedef $1 t; int s = sizeof(t) == 42;])], ["${ac_cv_sizeof___int128@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int128"], [ rb_cv_type_$1=no])])]) AS_IF([test "${rb_cv_type_$1}" != no], [ - type="${rb_cv_type_$1@%:@@%:@unsigned }" + type="${rb_cv_type_$1@%:@@%:@*signed }" + AS_IF([test "$type" = "long long"], [type=long_long]) AS_IF([test "$type" != yes && eval 'test -n "${ac_cv_sizeof_'$type'+set}"'], [ eval cond='"${ac_cv_sizeof_'$type'}"' AS_CASE([$cond], [*:*], [ diff --git a/tool/m4/ruby_modular_gc.m4 b/tool/m4/ruby_modular_gc.m4 new file mode 100644 index 0000000000..661fce2e60 --- /dev/null +++ b/tool/m4/ruby_modular_gc.m4 @@ -0,0 +1,41 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_MODULAR_GC],[ +AC_ARG_WITH(modular-gc, + AS_HELP_STRING([--with-modular-gc=DIR], + [Enable replacement of Ruby's GC from a modular library in the specified directory.]), + [modular_gc_dir=$withval], [unset modular_gc_dir] +) + +AS_IF([test "$modular_gc_dir" = yes], [ + AC_MSG_ERROR(you must specify a directory when using --with-modular-gc) +]) + +AC_MSG_CHECKING([if building with modular GC support]) +AS_IF([test x"$modular_gc_dir" != x], [ + AC_MSG_RESULT([yes]) + + # Ensure that modular_gc_dir is always an absolute path so that Ruby + # never loads a modular GC from a relative path + AS_CASE(["$modular_gc_dir"], + [/*], [], + [test "$load_relative" = yes || modular_gc_dir="$prefix/$modular_gc_dir"] + ) + + # Ensure that modular_gc_dir always terminates with a / + AS_CASE(["$modular_gc_dir"], + [*/], [], + [modular_gc_dir="$modular_gc_dir/"] + ) + + AC_DEFINE([USE_MODULAR_GC], [1]) + + modular_gc_summary="yes (in $modular_gc_dir)" +], [ + AC_MSG_RESULT([no]) + AC_DEFINE([USE_MODULAR_GC], [0]) + + modular_gc_summary="no" +]) + +AC_SUBST(modular_gc_dir, "${modular_gc_dir}") +])dnl diff --git a/tool/m4/ruby_setjmp_type.m4 b/tool/m4/ruby_setjmp_type.m4 index 4ae26fe5cd..d26af34ea0 100644 --- a/tool/m4/ruby_setjmp_type.m4 +++ b/tool/m4/ruby_setjmp_type.m4 @@ -3,18 +3,15 @@ AC_DEFUN([RUBY_SETJMP_TYPE], [ RUBY_CHECK_BUILTIN_SETJMP RUBY_CHECK_SETJMP(_setjmpex, [], [@%:@include <setjmpex.h>]) RUBY_CHECK_SETJMP(_setjmp) -RUBY_CHECK_SETJMP(sigsetjmp, [sigjmp_buf]) AC_MSG_CHECKING(for setjmp type) setjmp_suffix= -unset setjmp_sigmask AC_ARG_WITH(setjmp-type, - AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), + AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), [ AS_CASE([$withval], [__builtin_setjmp], [setjmp=__builtin_setjmp], [_setjmp], [ setjmp_prefix=_], - [sigsetjmp,*], [ setjmp_prefix=sig setjmp_sigmask=`expr "$withval" : 'sigsetjmp\(,.*\)'`], - [sigsetjmp], [ setjmp_prefix=sig], + [sigsetjmp*], [ AC_MSG_WARN(No longer use sigsetjmp; use setjmp instead); setjmp_prefix=], [setjmp], [ setjmp_prefix=], [setjmpex], [ setjmp_prefix= setjmp_suffix=ex], [''], [ unset setjmp_prefix], @@ -34,19 +31,13 @@ AS_IF([test ${setjmp_prefix+set}], [ ], [test "$ac_cv_func__setjmp" = yes], [ setjmp_prefix=_ setjmp_suffix= -], [test "$ac_cv_func_sigsetjmp" = yes], [ - AS_CASE([$target_os],[solaris*|cygwin*],[setjmp_prefix=],[setjmp_prefix=sig]) - setjmp_suffix= ], [ setjmp_prefix= setjmp_suffix= ]) -AS_IF([test x$setjmp_prefix:$setjmp_sigmask = xsig:], [ - setjmp_sigmask=,0 -]) -AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}${setjmp_sigmask}) -AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env)${setjmp_sigmask})]) +AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}) +AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env))]) AC_DEFINE_UNQUOTED([RUBY_LONGJMP(env,val)], [${setjmp_prefix}longjmp($setjmp_cast(env),val)]) -AS_IF([test "(" "$GCC" != yes ")" -o x$setjmp_prefix != x__builtin_], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, ${setjmp_sigmask+${setjmp_prefix}}jmp_buf)) +AS_CASE(["$GCC:$setjmp_prefix"], [yes:__builtin_], [], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, jmp_buf)) AS_IF([test x$setjmp_suffix = xex], [AC_DEFINE_UNQUOTED(RUBY_USE_SETJMPEX, 1)]) ])dnl diff --git a/tool/m4/ruby_stack_grow_direction.m4 b/tool/m4/ruby_stack_grow_direction.m4 index a4d205cc3c..8c6fdd5722 100644 --- a/tool/m4/ruby_stack_grow_direction.m4 +++ b/tool/m4/ruby_stack_grow_direction.m4 @@ -3,7 +3,7 @@ AC_DEFUN([RUBY_STACK_GROW_DIRECTION], [ AS_VAR_PUSHDEF([stack_grow_dir], [rb_cv_stack_grow_dir_$1]) AC_CACHE_CHECK(stack growing direction on $1, stack_grow_dir, [ AS_CASE(["$1"], -[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*], [ $2=-1], +[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*|arm*|aarch*], [ $2=-1], [hppa*], [ $2=+1], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ diff --git a/tool/m4/ruby_try_cflags.m4 b/tool/m4/ruby_try_cflags.m4 index 672f4f8e51..b397642aad 100644 --- a/tool/m4/ruby_try_cflags.m4 +++ b/tool/m4/ruby_try_cflags.m4 @@ -6,14 +6,36 @@ m4_version_prereq([2.70], [], [ m4_defun([AC_LANG_PROGRAM(C)], m4_bpatsubst(m4_defn([AC_LANG_PROGRAM(C)]), [main ()], [main (void)])) ])dnl dnl -AC_DEFUN([RUBY_TRY_CFLAGS], [ - AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS]) +AC_DEFUN([_RUBY_TRY_CFLAGS], [ RUBY_WERROR_FLAG([ CFLAGS="[$]CFLAGS $1" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS([$1], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)], + [$4], [$5]) +])dnl + +AC_DEFUN([_RUBY_TRY_CFLAGS_PREPEND], [ + RUBY_WERROR_FLAG([ + CFLAGS="$1 [$]CFLAGS" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS_PREPEND], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS_PREPEND([$1], [$2 AC_MSG_RESULT(yes)], [$3 - AC_MSG_RESULT(no)]) - ]) + AC_MSG_RESULT(no)], + [$4], [$5]) ])dnl diff --git a/tool/m4/ruby_universal_arch.m4 b/tool/m4/ruby_universal_arch.m4 index c8914c88d9..d3e0dd0b47 100644 --- a/tool/m4/ruby_universal_arch.m4 +++ b/tool/m4/ruby_universal_arch.m4 @@ -17,7 +17,7 @@ AS_IF([test ${target_archs+set}], [ cpu=$archs cpu=`echo $cpu | sed 's/-.*-.*//'` universal_binary="${universal_binary+$universal_binary,}$cpu" - universal_archnames="${universal_archnames} ${archs}=${cpu}" + universal_archnames="${universal_archnames:+$universal_archnames }${archs}=${cpu}" ARCH_FLAG="${ARCH_FLAG+$ARCH_FLAG }-arch $archs" ]) done @@ -40,7 +40,7 @@ AS_IF([test ${target_archs+set}], [ AS_IF([$CC $CFLAGS $ARCH_FLAG -o conftest conftest.c > /dev/null 2>&1], [ rm -fr conftest.* ], [test -z "$ARCH_FLAG"], [ - RUBY_DEFAULT_ARCH("$target_archs") + RUBY_DEFAULT_ARCH($target_archs) ]) ]) target_cpu=${target_archs} @@ -73,7 +73,7 @@ EOF sed -n 's/^"processor-name=\(.*\)"/\1/p'` target="$target_cpu${target}" AC_MSG_RESULT([$target_cpu]) - ]) + ]) ]) target_archs="$target_cpu" ]) diff --git a/tool/m4/ruby_wasm_tools.m4 b/tool/m4/ruby_wasm_tools.m4 index a6d8c34ebc..efc017e771 100644 --- a/tool/m4/ruby_wasm_tools.m4 +++ b/tool/m4/ruby_wasm_tools.m4 @@ -9,7 +9,7 @@ AC_DEFUN([RUBY_WASM_TOOLS], AC_SUBST(wasmoptflags) : ${wasmoptflags=-O3} - AC_MSG_CHECKING([wheather \$WASI_SDK_PATH is set]) + AC_MSG_CHECKING([whether \$WASI_SDK_PATH is set]) AS_IF([test x"${WASI_SDK_PATH}" = x], [ AC_MSG_RESULT([no]) AC_MSG_ERROR([WASI_SDK_PATH environment variable is required]) @@ -19,6 +19,7 @@ AC_DEFUN([RUBY_WASM_TOOLS], LD="${LD:-${WASI_SDK_PATH}/bin/clang}" AR="${AR:-${WASI_SDK_PATH}/bin/llvm-ar}" RANLIB="${RANLIB:-${WASI_SDK_PATH}/bin/llvm-ranlib}" + OBJCOPY="${OBJCOPY:-${WASI_SDK_PATH}/bin/llvm-objcopy}" ]) ]) ])dnl diff --git a/tool/make-snapshot b/tool/make-snapshot index d9c0b125fa..4af6a855eb 100755 --- a/tool/make-snapshot +++ b/tool/make-snapshot @@ -10,6 +10,7 @@ require 'fileutils' require 'shellwords' require 'tmpdir' require 'pathname' +require 'date' require 'yaml' require 'json' require File.expand_path("../lib/vcs", __FILE__) @@ -38,8 +39,6 @@ options: -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")}) -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")}) -unicode_version=VER Unicode version to generate encodings - -svn[=URL] make snapshot from SVN repository - (#{SVNURL}) -help, --help show this message version: master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL @@ -58,7 +57,7 @@ PACKAGES = { DEFAULT_PACKAGES = PACKAGES.keys - ["tar"] if !$no7z and system("7z", out: IO::NULL) PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" - PACKAGES["zip"] = %w".zip 7z a -tzip -l -mx -mtc=off" << {out: IO::NULL} + PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL} elsif gzip = ENV.delete("GZIP") PACKAGES["gzip"].concat(gzip.shellsplit) end @@ -69,21 +68,10 @@ if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?) end ENV["LC_ALL"] = ENV["LANG"] = "C" -SVNURL = URI.parse("https://svn.ruby-lang.org/repos/ruby/") # https git clone is disabled at git.ruby-lang.org/ruby.git. GITURL = URI.parse("https://github.com/ruby/ruby.git") RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ -ENV["VPATH"] ||= "include/ruby" -YACC = ENV["YACC"] ||= "bison" -ENV["BASERUBY"] ||= "ruby" -ENV["RUBY"] ||= "ruby" -ENV["MV"] ||= "mv" -ENV["RM"] ||= "rm -f" -ENV["MINIRUBY"] ||= "ruby" -ENV["PROGRAM"] ||= "ruby" -ENV["AUTOCONF"] ||= "autoconf" -ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" ENV["TZ"] = "UTC" class String @@ -120,22 +108,32 @@ $digests &&= $digests.split(/[, ]+/).tap {|dig| $digests ||= DIGESTS $patch_file &&= File.expand_path($patch_file) -path = ENV["PATH"].split(File::PATH_SEPARATOR) -%w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| - cmd, = ENV[var].shellsplit - unless path.any? {|dir| +PATH = ENV["PATH"].split(File::PATH_SEPARATOR) +def PATH.executable_env(var, command = nil) + command = ENV[var] ||= (command or return) + cmd, = command.shellsplit + unless any? {|dir| file = File.expand_path(cmd, dir) File.file?(file) and File.executable?(file) } abort "#{File.basename $0}: #{var} command not found - #{cmd}" end + command end +PATH.executable_env("MV", "mv") +PATH.executable_env("RM", "rm -f") +PATH.executable_env("AUTOCONF", "autoconf") + %w[BASERUBY RUBY MINIRUBY].each do |var| - %x[#{ENV[var]} --disable-gem -e1 2>&1] - if $?.success? - ENV[var] += ' --disable-gem' + cmd = PATH.executable_env(var, "ruby") + help = IO.popen("#{cmd} --help", err: %i[child out], &:read) + unless $?.success? and /ruby/ =~ help + abort "#{File.basename $0}: #{var} ruby not found - #{cmd}" end + IO.popen("#{cmd} --disable-gem -eexit", err: %i[child out], &:read) + cmd += ' --disable-gem' if $?.success? + ENV[var] = cmd end if defined?($help) or defined?($_help) @@ -255,7 +253,6 @@ end def package(vcs, rev, destdir, tmp = nil) pwd = Dir.pwd - patchlevel = false prerelease = false if rev and revision = rev[/@(\h+)\z/, 1] rev = $` @@ -272,22 +269,23 @@ def package(vcs, rev, destdir, tmp = nil) when /\Astable\z/ vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} url &&= vcs.branch(url) - when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + when /\A(\d+)\.(\d+)\.(\d+)-(preview|rc)(\d+)/ prerelease = true tag = "#{$4}#{$5}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") - when /\A(.*)\.(.*)\.(.*)-p(\d+)/ - patchlevel = true - tag = "p#{$4}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") - when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ - if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") - patchlevel = true - tag = "" - url = vcs.tag("v#{$1}_#{$2}_#{$3}") + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") + else + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") + end + when /\A(\d+)\.(\d+)\.(\d+)\z/ + tag = "" + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") else - url = vcs.branch("ruby_#{rev.tr('.', '_')}") + url = vcs.tag("v#{$1}_#{$2}_#{$3}") end + when /\A(\d+)\.(\d+)\z/ + url = vcs.branch("ruby_#{rev.tr('.', '_')}") else warn "#{$0}: unknown version - #{rev}" return @@ -337,7 +335,7 @@ def package(vcs, rev, destdir, tmp = nil) FileUtils.rm(file, verbose: $VERBOSE) end - status = IO.read(File.dirname(__FILE__) + "/prereq.status") + status = File.read(File.dirname(__FILE__) + "/prereq.status") Dir.chdir(tmp) if tmp if !File.directory?(v) @@ -349,26 +347,20 @@ def package(vcs, rev, destdir, tmp = nil) File.open("#{v}/revision.h", "wb") {|f| f.puts vcs.revision_header(revision, modified) } - version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version ||= (versionhdr = File.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] version ||= begin - include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h") + include_ruby_versionhdr = File.read("#{v}/include/ruby/version.h") api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] [api_major_version, api_minor_version, version_teeny].join('.') end version or return - if patchlevel - unless tag.empty? - versionhdr ||= IO.read("#{v}/version.h") - patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] - tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision)) - end - elsif prerelease - versionhdr ||= IO.read("#{v}/version.h") - versionhdr.sub!(/^\#define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) - IO.write("#{v}/version.h", versionhdr) + if prerelease + versionhdr ||= File.read("#{v}/version.h") + versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace" + File.write("#{v}/version.h", versionhdr) else tag ||= vcs.revision_name(revision) end @@ -387,8 +379,21 @@ def package(vcs, rev, destdir, tmp = nil) puts $colorize.fail("patching failed") return end - def (clean = []).add(n) push(n); n end - def clean.create(file, content = "") File.binwrite(add(file), content) end + + class << (clean = []) + def add(n) push(n) + n + end + def create(file, content = "", &block) + add(file) + if block + File.open(file, "wb", &block) + else + File.binwrite(file, content) + end + end + end + Dir.chdir(v) do unless File.exist?("ChangeLog") vcs.export_changelog(url, nil, revision, "ChangeLog") @@ -409,7 +414,7 @@ def package(vcs, rev, destdir, tmp = nil) puts end - File.open(clean.add("cross.rb"), "w") do |f| + clean.create("cross.rb") do |f| f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)" f.puts "CROSS_COMPILING=true" f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)" @@ -420,7 +425,7 @@ def package(vcs, rev, destdir, tmp = nil) puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE unless File.exist?("configure") print "creating configure..." - unless system([ENV["AUTOCONF"]]*2) + unless system(File.exist?(gen = "./autogen.sh") ? gen : [ENV["AUTOCONF"]]*2) puts $colorize.fail(" failed") return end @@ -429,11 +434,11 @@ def package(vcs, rev, destdir, tmp = nil) clean.add("autom4te.cache") clean.add("enc/unicode/data") print "creating prerequisites..." - if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + if File.file?("common.mk") && /^prereq/ =~ commonmk = File.read("common.mk") puts extout = clean.add('tmp') begin - status = IO.read("tool/prereq.status") + status = File.read("tool/prereq.status") rescue Errno::ENOENT # use fallback file end @@ -446,7 +451,7 @@ def package(vcs, rev, destdir, tmp = nil) File.binwrite("#{defaults}/ruby.rb", "") miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] - mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). + mk = (File.read("template/Makefile.in") rescue File.read("Makefile.in")). gsub(/^@.*\n/, '') vars = { "EXTOUT"=>extout, @@ -459,6 +464,9 @@ def package(vcs, rev, destdir, tmp = nil) "MAJOR"=>api_major_version, "MINOR"=>api_minor_version, "TEENY"=>version_teeny, + "VPATH"=>(ENV["VPATH"] || "include/ruby"), + "PROGRAM"=>(ENV["PROGRAM"] || "ruby"), + "BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"), } status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do vars[$2] ||= $3 @@ -467,6 +475,13 @@ def package(vcs, rev, destdir, tmp = nil) vars["UNICODE_VERSION"] = $unicode_version if $unicode_version args = vars.dup mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} + commonmk.gsub!(/^!(?:include \$\(srcdir\)\/(.*))?/) do + if inc = $1 and File.exist?(inc) + File.binread(inc).gsub(/^!/, '# !') + else + "#" + end + end mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') mk << <<-'APPEND' @@ -504,12 +519,11 @@ touch-unicode-files: end print "prerequisites" else - system(*%W"#{YACC} -o parse.c parse.y") + system(*%W[#{PATH.executable_env("YACC", "bison")} -o parse.c parse.y]) end vcs.after_export(".") if exported clean.concat(Dir.glob("ext/**/autom4te.cache")) - FileUtils.rm_rf(clean) unless $keep_temp - FileUtils.rm_rf(".downloaded-cache") + clean.add(".downloaded-cache") if File.exist?("gems/bundled_gems") gems = Dir.glob("gems/*.gem") gems -= File.readlines("gems/bundled_gems").map {|line| @@ -517,10 +531,11 @@ touch-unicode-files: name, version, _ = line.split(' ') "gems/#{name}-#{version}.gem" } - FileUtils.rm_f(gems) + clean.concat(gems) else - FileUtils.rm_rf("gems") + clean.add("gems") end + FileUtils.rm_rf(clean) if modified touch_all(modified, "**/*/", 0) do |name, stat| stat.mtime > modified @@ -589,20 +604,18 @@ ensure Dir.chdir(pwd) end -if [$srcdir, ($svn||=nil), ($git||=nil)].compact.size > 1 - abort "#{File.basename $0}: -srcdir, -svn, and -git are exclusive" +if [$srcdir, ($git||=nil)].compact.size > 1 + abort "#{File.basename $0}: -srcdir and -git are exclusive" end if $srcdir vcs = VCS.detect($srcdir) -elsif $svn - vcs = VCS::SVN.new($svn == true ? SVNURL : URI.parse($svn)) elsif $git abort "#{File.basename $0}: use -srcdir with cloned local repository" else begin vcs = VCS.detect(File.expand_path("../..", __FILE__)) rescue VCS::NotFoundError - vcs = VCS::SVN.new(SVNURL) + abort "#{File.expand_path("../..", __FILE__)}: cannot find git repository" end end @@ -623,7 +636,7 @@ revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| key = basename[/\A(.*)\.(?:tar|zip)/, 1] info[key] ||= Hash.new{|h,k|h[k]={}} info[key]['version'] = version if version - info[key]['date'] = release_date.strftime('%Y-%m-%d') + info[key]['date'] = release_date.to_date if version info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" diff --git a/tool/merger.rb b/tool/merger.rb index d38f00b0fd..4c096087fc 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -2,9 +2,8 @@ # -*- ruby -*- exec "${RUBY-ruby}" "-x" "$0" "$@" && [ ] if false #!ruby -# This needs ruby 2.0, Subversion and Git. -# As a Ruby committer, run this in an SVN repository -# to commit a change. +# This needs ruby 2.0 and Git. +# As a Ruby committer, run this in a git repository to commit a change. require 'tempfile' require 'net/http' @@ -15,20 +14,11 @@ ENV['LC_ALL'] = 'C' ORIGIN = 'git@git.ruby-lang.org:ruby.git' GITHUB = 'git@github.com:ruby/ruby.git' -module Merger - REPOS = 'svn+ssh://svn@ci.ruby-lang.org/ruby/' -end - -class << Merger - include Merger - +class << Merger = Object.new def help puts <<-HELP \e[1msimple backport\e[0m - ruby #$0 1234 - -\e[1mbackport from other branch\e[0m - ruby #$0 17502 mvm + ruby #$0 1234abc \e[1mrevision increment\e[0m ruby #$0 revisionup @@ -37,16 +27,16 @@ class << Merger ruby #$0 teenyup \e[1mtagging major release\e[0m - ruby #$0 tag 2.2.0 + ruby #$0 tag 3.2.0 -\e[1mtagging patch release\e[0m (about 2.1.0 or later, it means X.Y.Z (Z > 0) release) +\e[1mtagging patch release\e[0m (for 2.1.0 or later, it means X.Y.Z (Z > 0) release) ruby #$0 tag \e[1mtagging preview/RC\e[0m - ruby #$0 tag 2.2.0-preview1 + ruby #$0 tag 3.2.0-preview1 \e[1mremove tag\e[0m - ruby #$0 removetag 2.2.9 + ruby #$0 removetag 3.2.9 \e[33;1m* all operations shall be applied to the working directory.\e[0m HELP @@ -57,11 +47,11 @@ class << Merger yield if block_given? STDERR.puts "\e[1;33m#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})\e[0m" case STDIN.gets - when /\Aa/i then exit + when /\Aa/i then exit 1 when /\Ar/i then redo when /\Ay/i then break when /\Ae/i then system(ENV['EDITOR'], editfile) - else exit + else exit 1 end end end @@ -69,17 +59,14 @@ class << Merger def version_up(teeny: false) now = Time.now now = now.localtime(9*60*60) # server is Japan Standard Time +09:00 - if svn_mode? - system('svn', 'revert', 'version.h') - else - system('git', 'checkout', 'HEAD', 'version.h') - end + system('git', 'checkout', 'HEAD', 'version.h') v, pl = version if teeny v[2].succ! end - if pl != '-1' # trunk does not have patchlevel + # We stopped bumping RUBY_PATCHLEVEL at Ruby 4.0.0. + if Integer(v[0]) <= 3 pl.succ! end @@ -127,33 +114,21 @@ class << Merger abort 'no relname is given and not in a release branch even if this is patch release' end end - tagname = "v#{v.join('_')}#{("_#{pl}" if v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl)}" - - if svn_mode? - if relname - branch_url = `svn info`[/URL: (.*)/, 1] - else - branch_url = "#{REPOS}branches/ruby_" - if v[0] < '2' || (v[0] == '2' && v[1] < '1') - abort 'patchlevel must be greater than 0 for patch release' if pl == '0' - branch_url << v.join('_') - else - abort 'teeny must be greater than 0 for patch release' if v[2] == '0' - branch_url << v.join('_').sub(/_\d+\z/, '') - end - end - tag_url = "#{REPOS}tags/#{tagname}" - system('svn', 'info', tag_url, out: IO::NULL, err: IO::NULL) - if $?.success? - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('svn', 'cp', '-m', "add tag #{tagname}", branch_url, tag_url, interactive: true) + if /^(?:preview|rc)/ =~ pl + tagname = "v#{v.join('.')}-#{pl}" + elsif Integer(v[0]) >= 4 + tagname = "v#{v.join('.')}" else - unless execute('git', 'tag', tagname) - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('git', 'push', ORIGIN, tagname, interactive: true) + tagname = "v#{v.join('_')}" end + + unless execute('git', 'diff', '--exit-code') + abort 'uncommitted changes' + end + unless execute('git', 'tag', tagname) + abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' + end + execute('git', 'push', ORIGIN, tagname, interactive: true) end def remove_tag(relname) @@ -167,70 +142,52 @@ class << Merger unless relname raise ArgumentError, 'relname is not specified' end - if /^v/ !~ relname - tagname = "v#{relname.gsub(/[.-]/, '_')}" - else + if relname.start_with?('v') tagname = relname - end - - if svn_mode? - tag_url = "#{REPOS}tags/#{tagname}" - execute('svn', 'rm', '-m', "remove tag #{tagname}", tag_url, interactive: true) + elsif Integer(relname.split('.', 2).first) >= 4 + tagname = "v#{relname}" else - execute('git', 'tag', '-d', tagname) - execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) - execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) + tagname = "v#{relname.gsub(/[.-]/, '_')}" end + + execute('git', 'tag', '-d', tagname) + execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) + execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) end def update_revision_h - if svn_mode? - execute('svn', 'up') - end execute('ruby tool/file2lastrev.rb --revision.h . > revision.tmp') execute('tool/ifchange', '--timestamp=.revision.time', 'revision.h', 'revision.tmp') execute('rm', '-f', 'revision.tmp') end def stat - if svn_mode? - `svn stat` - else - `git status --short` - end + `git status --short` end def diff(file = nil) - if svn_mode? - command = %w[svn diff --diff-cmd=diff -x -upw] - else - command = %w[git diff --color] - end + command = %w[git diff --color HEAD] IO.popen(command + [file].compact, &:read) end def commit(file) - if svn_mode? - begin - execute('svn', 'ci', '-F', file) - execute('svn', 'update') # svn ci doesn't update revision info on working copy - ensure - execute('rm', '-f', 'subversion.commitlog') - end - else - current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip - execute('git', 'add', '.') && - execute('git', 'commit', '-F', file) - end + current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip + execute('git', 'add', '.') && execute('git', 'commit', '-F', file) end - private - - def svn_mode? - return @svn_mode if defined?(@svn_mode) - @svn_mode = system("svn info", %i(out err) => IO::NULL) + def has_conflicts? + changes = IO.popen(%w[git status --porcelain -z]) { |io| io.readlines("\0", chomp: true) } + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + !conflict.empty? end + private + # Prints the version of Ruby found in version.h def version v = p = nil @@ -289,21 +246,21 @@ else case ARGV[0] when /--ticket=(.*)/ - tickets = $1.split(/,/).map{|num| " [Backport ##{num}]"}.join + tickets = $1.split(/,/) ARGV.shift else - tickets = '' + tickets = [] + detect_ticket = true end - revstr = ARGV[0].delete('^, :\-0-9a-fA-F') + revstr = ARGV[0].gsub(%r!https://github\.com/ruby/ruby/commit/|https://bugs\.ruby-lang\.org/projects/ruby-master/repository/git/revisions/!, '') + revstr = revstr.delete('^, :\-0-9a-fA-F') revs = revstr.split(/[,\s]+/) commit_message = '' revs.each do |rev| git_rev = nil case rev - when /\A\d{1,6}\z/ - svn_rev = rev when /\A\h{7,40}\z/ git_rev = rev when nil then @@ -314,28 +271,25 @@ else exit end - # Merge revision from Git patch or SVN - if git_rev - git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" - resp = Net::HTTP.get_response(URI(git_uri)) - if resp.code != '200' - abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" - end - patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - - message = "\n\n#{(patch[/^Subject: (.*)\n\ndiff --git/m, 1] || "Message not found for revision: #{git_rev}\n")}" - puts '+ git apply' - IO.popen(['git', 'apply'], 'wb') { |f| f.write(patch) } - else - default_merge_branch = (%r{^URL: .*/branches/ruby_1_8_} =~ `svn info` ? 'branches/ruby_1_8' : 'trunk') - svn_src = "#{Merger::REPOS}#{ARGV[1] || default_merge_branch}" - message = IO.popen(['svn', 'log', '-c', svn_rev, svn_src], &:read) + # Merge revision from Git patch + git_uri = "https://github.com/ruby/ruby/commit/#{git_rev}.patch" + resp = Net::HTTP.get_response(URI(git_uri)) + if resp.code != '200' + abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" + end + patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - cmd = ['svn', 'merge', '--accept=postpone', '-c', svn_rev, svn_src] - puts "+ #{cmd.join(' ')}" - system(*cmd) + if detect_ticket + tickets += patch.scan(/\[(?:Bug|Feature|Misc) #(\d+)\]/i).map(&:first) end + message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" + message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") + message = "\n\n#{message}" + + puts '+ git apply' + IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } + commit_message << message.sub(/\A-+\nr.*/, '').sub(/\n-+\n\z/, '').gsub(/^./, "\t\\&") end @@ -345,20 +299,22 @@ else Merger.version_up f = Tempfile.new 'merger.rb' - f.printf "merge revision(s) %s:%s", revstr, tickets + f.printf "merge revision(s) %s:%s", revs.join(', '), tickets.map{|num| " [Backport ##{num}]"}.join f.write commit_message f.flush f.close - Merger.interactive('conflicts resolved?', f.path) do - IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| - g << Merger.stat - g << "\n\n" - f.open - g << f.read - f.close - g << "\n\n" - g << Merger.diff + if Merger.has_conflicts? + Merger.interactive('conflicts resolved?', f.path) do + IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| + g << Merger.stat + g << "\n\n" + f.open + g << f.read + f.close + g << "\n\n" + g << Merger.diff + end end end diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat new file mode 100755 index 0000000000..d39568fe86 --- /dev/null +++ b/tool/missing-baseruby.bat @@ -0,0 +1,30 @@ +:"" == " +@echo off || ( + :warn + echo>&2.%~1 + goto :eof + :abort + exit /b 1 +)||( +:)"||( + # necessary libraries + require 'erb' + require 'fileutils' + require 'tempfile' + s = %^# +) +: ; call() { local call=${1#:}; shift; $call "$@"; } +: ; warn() { echo "$1" >&2; } +: ; abort () { exit 1; } + +call :warn "executable host ruby is required. use --with-baseruby option." +call :warn "Note that BASERUBY must be Ruby 3.1.0 or later." +call :abort +(goto :eof ^;) +verbose = true if ARGV[0] == "--verbose" +case +when !defined?(RubyVM::InstructionSequence) + abort(*(["BASERUBY must be CRuby"] if verbose)) +when RUBY_VERSION < s[%r[warn .*\KBASERUBY .*Ruby ([\d.]+)(?:\.0)?.*(?=\")],1] + abort(*(["#{$&}. Found: #{RUBY_VERSION}"] if verbose)) +end diff --git a/tool/mjit/bindgen.rb b/tool/mjit/bindgen.rb deleted file mode 100755 index a966a46944..0000000000 --- a/tool/mjit/bindgen.rb +++ /dev/null @@ -1,435 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -ENV['GEM_HOME'] = File.expand_path('./.bundle', __dir__) -require 'rubygems/source' -require 'bundler/inline' -gemfile(true) do - source 'https://rubygems.org' - gem 'ffi-clang', '0.7.0', require: false -end - -# Help ffi-clang find libclang -# Hint: apt install libclang1 -ENV['LIBCLANG'] ||= Dir.glob("/usr/lib/llvm-*/lib/libclang.so.1").grep_v(/-cpp/).sort.last -require 'ffi/clang' - -require 'etc' -require 'fiddle/import' -require 'set' - -unless build_dir = ARGV.first - abort "Usage: #{$0} BUILD_DIR" -end - -class Node < Struct.new( - :kind, - :spelling, - :type, - :typedef_type, - :bitwidth, - :sizeof_type, - :offsetof, - :enum_value, - :children, - keyword_init: true, -) -end - -# Parse a C header with ffi-clang and return Node objects. -# To ease the maintenance, ffi-clang should be used only inside this class. -class HeaderParser - def initialize(header, cflags:) - @translation_unit = FFI::Clang::Index.new.parse_translation_unit(header, cflags, [], {}) - end - - def parse - parse_children(@translation_unit.cursor) - end - - private - - def parse_children(cursor) - children = [] - cursor.visit_children do |cursor, _parent| - children << parse_cursor(cursor) - next :continue - end - children - end - - def parse_cursor(cursor) - unless cursor.kind.start_with?('cursor_') - raise "unexpected cursor kind: #{cursor.kind}" - end - kind = cursor.kind.to_s.delete_prefix('cursor_').to_sym - children = parse_children(cursor) - - offsetof = {} - if kind == :struct - children.select { |c| c.kind == :field_decl }.each do |child| - offsetof[child.spelling] = cursor.type.offsetof(child.spelling) - end - end - - sizeof_type = nil - if %i[struct union].include?(kind) - sizeof_type = cursor.type.sizeof - end - - enum_value = nil - if kind == :enum_constant_decl - enum_value = cursor.enum_value - end - - Node.new( - kind: kind, - spelling: cursor.spelling, - type: cursor.type.spelling, - typedef_type: cursor.typedef_type.spelling, - bitwidth: cursor.bitwidth, - sizeof_type: sizeof_type, - offsetof: offsetof, - enum_value: enum_value, - children: children, - ) - end -end - -# Convert Node objects to a Ruby binding source. -class BindingGenerator - BINDGEN_BEG = '### MJIT bindgen begin ###' - BINDGEN_END = '### MJIT bindgen end ###' - DEFAULTS = { '_Bool' => 'CType::Bool.new' } - DEFAULTS.default_proc = proc { |_h, k| "CType::Stub.new(:#{k})" } - - attr_reader :src - - # @param src_path [String] - # @param uses [Array<String>] - # @param values [Hash{ Symbol => Array<String> }] - # @param types [Array<String>] - # @param dynamic_types [Array<String>] #ifdef-dependent immediate types, which need Primitive.cexpr! for type detection - # @param skip_fields [Hash{ Symbol => Array<String> }] Struct fields that are skipped from bindgen - # @param ruby_fields [Hash{ Symbol => Array<String> }] Struct VALUE fields that are considered Ruby objects - def initialize(src_path:, uses:, values:, types:, dynamic_types:, skip_fields:, ruby_fields:) - @preamble, @postamble = split_ambles(src_path) - @src = String.new - @uses = uses.sort - @values = values.transform_values(&:sort) - @types = types.sort - @dynamic_types = dynamic_types.sort - @skip_fields = skip_fields.transform_keys(&:to_s) - @ruby_fields = ruby_fields.transform_keys(&:to_s) - @references = Set.new - end - - def generate(nodes) - println @preamble - - # Define USE_* macros - @uses.each do |use| - println " def C.#{use}" - println " Primitive.cexpr! %q{ RBOOL(#{use} != 0) }" - println " end" - println - end - - # Define macros/enums - @values.each do |type, values| - values.each do |value| - println " def C.#{value}" - println " Primitive.cexpr! %q{ #{type}2NUM(#{value}) }" - println " end" - println - end - end - - # TODO: Support nested declarations - nodes_index = nodes.group_by(&:spelling).transform_values(&:last) - - # Define types - @types.each do |type| - unless definition = generate_node(nodes_index[type]) - raise "Failed to generate type: #{type}" - end - println " def C.#{type}" - println "@#{type} ||= #{definition}".gsub(/^/, " ").chomp - println " end" - println - end - - # Define dynamic types - @dynamic_types.each do |type| - unless generate_node(nodes_index[type])&.start_with?('CType::Immediate') - raise "Non-immediate type is given to dynamic_types: #{type}" - end - println " def C.#{type}" - println " @#{type} ||= CType::Immediate.find(Primitive.cexpr!(\"SIZEOF(#{type})\"), Primitive.cexpr!(\"SIGNED_TYPE_P(#{type})\"))" - println " end" - println - end - - # Leave a stub for types that are referenced but not targeted - (@references - @types - @dynamic_types).each do |type| - println " def C.#{type}" - println " #{DEFAULTS[type]}" - println " end" - println - end - - print @postamble - end - - private - - # Return code before BINDGEN_BEG and code after BINDGEN_END - def split_ambles(src_path) - lines = File.read(src_path).lines - - preamble_end = lines.index { |l| l.include?(BINDGEN_BEG) } - raise "`#{BINDGEN_BEG}` was not found in '#{src_path}'" if preamble_end.nil? - - postamble_beg = lines.index { |l| l.include?(BINDGEN_END) } - raise "`#{BINDGEN_END}` was not found in '#{src_path}'" if postamble_beg.nil? - raise "`#{BINDGEN_BEG}` was found after `#{BINDGEN_END}`" if preamble_end >= postamble_beg - - return lines[0..preamble_end].join, lines[postamble_beg..-1].join - end - - # Generate code from a node. Used for constructing a complex nested node. - # @param node [Node] - def generate_node(node, sizeof_type: nil) - case node&.kind - when :struct, :union - # node.spelling is often empty for union, but we'd like to give it a name when it has one. - buf = +"CType::#{node.kind.to_s.sub(/\A[a-z]/, &:upcase)}.new(\n" - buf << " \"#{node.spelling}\", Primitive.cexpr!(\"SIZEOF(#{sizeof_type || node.type})\"),\n" - bit_fields_end = node.children.index { |c| c.bitwidth == -1 } || node.children.size # first non-bit field index - node.children.each_with_index do |child, i| - skip_type = sizeof_type&.gsub(/\(\(struct ([^\)]+) \*\)NULL\)->/, '\1.') || node.spelling - next if @skip_fields.fetch(skip_type, []).include?(child.spelling) - field_builder = proc do |field, type| - if node.kind == :struct - to_ruby = @ruby_fields.fetch(node.spelling, []).include?(field) - if child.bitwidth > 0 - if bit_fields_end <= i # give up offsetof calculation for non-leading bit fields - raise "non-leading bit fields are not supported. consider including '#{field}' in skip_fields." - end - offsetof = node.offsetof.fetch(field) - else - off_type = sizeof_type || "(*((#{node.type} *)NULL))" - offsetof = "Primitive.cexpr!(\"OFFSETOF(#{off_type}, #{field})\")" - end - " #{field}: [#{type}, #{offsetof}#{', true' if to_ruby}],\n" - else - " #{field}: #{type},\n" - end - end - - case child - # BitField is struct-specific. So it must be handled here. - in Node[kind: :field_decl, spelling:, bitwidth:, children: [_grandchild, *]] if bitwidth > 0 - buf << field_builder.call(spelling, "CType::BitField.new(#{bitwidth}, #{node.offsetof.fetch(spelling) % 8})") - # "(unnamed ...)" struct and union are handled here, which are also struct-specific. - in Node[kind: :field_decl, spelling:, type:, children: [grandchild]] if type.match?(/\((unnamed|anonymous) [^)]+\)\z/) - if sizeof_type - child_type = "#{sizeof_type}.#{child.spelling}" - else - child_type = "((#{node.type} *)NULL)->#{child.spelling}" - end - buf << field_builder.call(spelling, generate_node(grandchild, sizeof_type: child_type).gsub(/^/, ' ').sub(/\A +/, '')) - # In most cases, we'd like to let generate_type handle the type unless it's "(unnamed ...)". - in Node[kind: :field_decl, spelling:, type:] if !type.empty? - buf << field_builder.call(spelling, generate_type(type)) - else # forward declarations are ignored - end - end - buf << ")" - when :typedef_decl - case node.children - in [child] - generate_node(child) - in [child, Node[kind: :integer_literal]] - generate_node(child) - in _ unless node.typedef_type.empty? - generate_type(node.typedef_type) - end - when :enum_decl - generate_type('int') - when :type_ref - generate_type(node.spelling) - end - end - - # Generate code from a type name. Used for resolving the name of a simple leaf node. - # @param type [String] - def generate_type(type) - if type.match?(/\[\d+\]\z/) - return "CType::Pointer.new { #{generate_type(type.sub!(/\[\d+\]\z/, ''))} }" - end - type = type.delete_suffix('const') - if type.end_with?('*') - return "CType::Pointer.new { #{generate_type(type.delete_suffix('*').rstrip)} }" - end - - type = type.gsub(/((const|volatile) )+/, '').rstrip - if type.start_with?(/(struct|union|enum) /) - target = type.split(' ', 2).last - push_target(target) - "self.#{target}" - else - begin - ctype = Fiddle::Importer.parse_ctype(type) - rescue Fiddle::DLError - push_target(type) - "self.#{type}" - else - # Convert any function pointers to void* to workaround FILE* vs int* - if ctype == Fiddle::TYPE_VOIDP - "CType::Immediate.parse(\"void *\")" - else - "CType::Immediate.parse(#{type.dump})" - end - end - end - end - - def print(str) - @src << str - end - - def println(str = "") - @src << str << "\n" - end - - def chomp - @src.delete_suffix!("\n") - end - - def rstrip! - @src.rstrip! - end - - def push_target(target) - unless target.match?(/\A\w+\z/) - raise "invalid target: #{target}" - end - @references << target - end -end - -src_dir = File.expand_path('../..', __dir__) -src_path = File.join(src_dir, 'mjit_c.rb') -build_dir = File.expand_path(build_dir) -cflags = [ - src_dir, - build_dir, - File.join(src_dir, 'include'), - File.join(build_dir, ".ext/include/#{RUBY_PLATFORM}"), -].map { |dir| "-I#{dir}" } - -# Clear .cache/clangd created by the language server, which could break this bindgen -clangd_cache = File.join(src_dir, '.cache/clangd') -if Dir.exist?(clangd_cache) - system('rm', '-rf', clangd_cache, exception: true) -end - -# Parse mjit_c.h and generate mjit_c.rb -nodes = HeaderParser.new(File.join(src_dir, 'mjit_c.h'), cflags: cflags).parse -generator = BindingGenerator.new( - src_path: src_path, - uses: %w[ - USE_LAZY_LOAD - ], - values: { - INT: %w[ - NOT_COMPILED_STACK_SIZE - VM_CALL_KW_SPLAT - VM_CALL_KW_SPLAT_bit - VM_CALL_TAILCALL - VM_CALL_TAILCALL_bit - VM_METHOD_TYPE_CFUNC - VM_METHOD_TYPE_ISEQ - ], - UINT: %w[ - RUBY_EVENT_CLASS - SHAPE_CAPACITY_CHANGE - SHAPE_FLAG_SHIFT - SHAPE_FROZEN - SHAPE_ID_NUM_BITS - SHAPE_INITIAL_CAPACITY - SHAPE_IVAR - SHAPE_ROOT - ], - ULONG: %w[ - INVALID_SHAPE_ID - SHAPE_MASK - ], - PTR: %w[ - rb_cFalseClass - rb_cFloat - rb_cInteger - rb_cNilClass - rb_cSymbol - rb_cTrueClass - ], - }, - types: %w[ - CALL_DATA - IC - IVC - RB_BUILTIN - attr_index_t - compile_branch - compile_status - inlined_call_context - iseq_inline_constant_cache - iseq_inline_constant_cache_entry - iseq_inline_iv_cache_entry - iseq_inline_storage_entry - mjit_options - rb_builtin_function - rb_call_data - rb_callable_method_entry_struct - rb_callcache - rb_callinfo - rb_control_frame_t - rb_cref_t - rb_execution_context_struct - rb_execution_context_t - rb_iseq_constant_body - rb_iseq_location_t - rb_iseq_struct - rb_iseq_t - rb_method_definition_struct - rb_method_iseq_t - rb_method_type_t - rb_mjit_compile_info - rb_mjit_unit - rb_serial_t - rb_shape - rb_shape_t - ], - dynamic_types: %w[ - VALUE - shape_id_t - ], - skip_fields: { - 'rb_execution_context_struct.machine': %w[regs], # differs between macOS and Linux - rb_execution_context_struct: %w[method_missing_reason], # non-leading bit fields not supported - rb_iseq_constant_body: %w[yjit_payload], # conditionally defined - }, - ruby_fields: { - rb_iseq_location_struct: %w[ - base_label - first_lineno - label - pathobj - ] - }, -) -generator.generate(nodes) - -# Write mjit_c.rb -File.write(src_path, generator.src) diff --git a/tool/mjit_archflag.sh b/tool/mjit_archflag.sh deleted file mode 100644 index 082fb4bcd0..0000000000 --- a/tool/mjit_archflag.sh +++ /dev/null @@ -1,40 +0,0 @@ -# -*- sh -*- - -quote() { - printf "#${indent}define $1" - shift - ${1+printf} ${1+' "%s"'$sep} ${1+"$@"} - echo -} - -archs="" -arch_flag="" - -parse_arch_flags() { - for arch in $1; do - archs="${archs:+$archs }${arch%=*}" - done - - while shift && [ "$#" -gt 0 ]; do - case "$1" in - -arch) - shift - archs="${archs:+$archs }$1" - ;; - *) - arch_flag="${arch_flag:+${arch_flag} }$1" - ;; - esac - done -} - -define_arch_flags() { - ${archs:+echo} ${archs:+'#if 0'} - for arch in $archs; do - echo "#elif defined __${arch}__" - quote "MJIT_ARCHFLAG " -arch "${arch}" - done - ${archs:+echo} ${archs:+'#else'} - quote "MJIT_ARCHFLAG /* ${arch_flag:-no flag} */" ${arch_flag} - ${archs:+echo} ${archs:+'#endif'} -} diff --git a/tool/mjit_tabs.rb b/tool/mjit_tabs.rb deleted file mode 100644 index edcbf6cfcb..0000000000 --- a/tool/mjit_tabs.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true -# This is a script to run a command in ARGV, expanding tabs in some files -# included by vm.c to normalize indentation of MJIT header. You can enable -# this feature by passing `--without-mjit-tabs` in configure. -# -# Note that preprocessor of GCC converts a hard tab to one spaces, where -# we expect it to be shown as 8 spaces. To obviate this script, we need -# to convert all tabs to spaces in these files. - -require 'fileutils' - -EXPAND_TARGETS = %w[ - vm*.* - include/ruby/ruby.h -] - -# These files have no hard tab indentations. Skip normalizing these files from the glob result. -SKIPPED_FILES = %w[ - vm_callinfo.h - vm_debug.h - vm_exec.h - vm_opts.h - vm_sync.h - vm_sync.c -] - -srcdir = File.expand_path('..', __dir__) -targets = EXPAND_TARGETS.flat_map { |t| Dir.glob(File.join(srcdir, t)) } - SKIPPED_FILES.map { |f| File.join(srcdir, f) } -sources = {} -mtimes = {} - -mjit_tabs, *command = ARGV - -targets.each do |target| - next if mjit_tabs != 'false' - unless File.writable?(target) - puts "tool/mjit_tabs.rb: Skipping #{target.dump} as it's not writable." - next - end - source = File.read(target) - begin - expanded = source.gsub(/^\t+/) { |tab| ' ' * 8 * tab.length } - rescue ArgumentError # invalid byte sequence in UTF-8 (Travis, RubyCI) - puts "tool/mjit_tabs.rb: Skipping #{target.dump} as the encoding is #{source.encoding}." - next - end - - sources[target] = source - mtimes[target] = File.mtime(target) - - if sources[target] == expanded - puts "#{target.dump} has no hard tab indentation. This should be ignored in tool/mjit_tabs.rb." - end - File.write(target, expanded) - FileUtils.touch(target, mtime: mtimes[target]) -end - -result = system(*command) - -targets.each do |target| - if sources.key?(target) - File.write(target, sources[target]) - FileUtils.touch(target, mtime: mtimes.fetch(target)) - end -end - -exit result diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index 94af846a7e..5aa07962f9 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -6,6 +6,23 @@ require_relative 'ruby_vm/helpers/c_escape' SUBLIBS = {} REQUIRED = {} +BUILTIN_ATTRS = %w[leaf inline_block use_block c_trace] + +module CompileWarning + @@warnings = 0 + + def warn(message) + @@warnings += 1 + super + end + + def self.reset + w, @@warnings = @@warnings, 0 + w.nonzero? + end +end + +Warning.extend CompileWarning def string_literal(lit, str = []) while lit @@ -25,6 +42,17 @@ def string_literal(lit, str = []) end end +# e.g. [:symbol_literal, [:symbol, [:@ident, "inline", [19, 21]]]] +def symbol_literal(lit) + symbol_literal, symbol_lit = lit + raise "#{lit.inspect} was not :symbol_literal" if symbol_literal != :symbol_literal + symbol, ident_lit = symbol_lit + raise "#{symbol_lit.inspect} was not :symbol" if symbol != :symbol + ident, symbol_name, = ident_lit + raise "#{ident.inspect} was not :@ident" if ident != :@ident + symbol_name +end + def inline_text argc, arg1 raise "argc (#{argc}) of inline! should be 1" unless argc == 1 arg1 = string_literal(arg1) @@ -32,6 +60,16 @@ def inline_text argc, arg1 arg1.join("").rstrip end +def inline_attrs(args) + raise "args was empty" if args.empty? + args.each do |arg| + attr = symbol_literal(arg) + unless BUILTIN_ATTRS.include?(attr) + raise "attr (#{attr}) was not in: #{BUILTIN_ATTRS.join(', ')}" + end + end +end + def make_cfunc_name inlines, name, lineno case name when /\[\]/ @@ -138,15 +176,13 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil if /(.+)[\!\?]\z/ =~ func_name case $1 when 'attr' - text = inline_text(argc, args.first) - if text != 'inline' - raise "Only 'inline' is allowed to be annotated (but got: '#{text}')" - end + # Compile-time validation only. compile.c will parse them. + inline_attrs(args) break when 'cstmt' text = inline_text argc, args.first - func_name = "_bi#{inlines.size}" + func_name = "_bi#{lineno}" cfunc_name = make_cfunc_name(inlines, name, lineno) inlines[cfunc_name] = [lineno, text, locals, func_name] argc -= 1 @@ -154,7 +190,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil text = inline_text argc, args.first code = "return #{text};" - func_name = "_bi#{inlines.size}" + func_name = "_bi#{lineno}" cfunc_name = make_cfunc_name(inlines, name, lineno) locals = [] if $1 == 'cconst' @@ -243,12 +279,25 @@ end def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) f = StringIO.new + + # Avoid generating fetches of lvars we don't need. This is imperfect as it + # will match text inside strings or other false positives. + local_ptrs = [] + local_candidates = text.gsub(/\bLOCAL_PTR\(\K[a-zA-Z_][a-zA-Z0-9_]*(?=\))/) { + local_ptrs << $&; '' + }.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + f.puts '{' lineno += 1 - locals.reverse_each.with_index{|param, i| + # locals is nil outside methods + locals&.reverse_each&.with_index{|param, i| next unless Symbol === param - f.puts "MAYBE_UNUSED(const VALUE) #{param} = rb_vm_lvar(ec, #{-3 - i});" - lineno += 1 + param = param.to_s + lvar = local_candidates.include?(param) + next unless lvar or local_ptrs.include?(param) + f.puts "VALUE *const #{param}__ptr = (VALUE *)&ec->cfp->ep[#{-3 - i}];" + f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" if lvar + lineno += lvar ? 2 : 1 } f.puts "#line #{body_lineno} \"#{line_file}\"" lineno += 1 @@ -272,16 +321,18 @@ def mk_builtin_header file # bs = { func_name => argc } code = File.read(file) - collect_iseq RubyVM::InstructionSequence.compile(code).to_a - collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) - begin - f = File.open(ofile, 'w') - rescue SystemCallError # EACCES, EPERM, EROFS, etc. - # Fall back to the current directory - f = File.open(File.basename(ofile), 'w') + verbose, $VERBOSE = $VERBOSE, true + collect_iseq RubyVM::InstructionSequence.compile(code, base).to_a + ensure + $VERBOSE = verbose end - begin + if warnings = CompileWarning.reset + raise "#{warnings} warnings in #{file}" + end + collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) + + StringIO.open do |f| if File::ALT_SEPARATOR file = file.tr(File::ALT_SEPARATOR, File::SEPARATOR) ofile = ofile.tr(File::ALT_SEPARATOR, File::SEPARATOR) @@ -318,44 +369,6 @@ def mk_builtin_header file end } - bs.each_pair{|func, (argc, cfunc_name)| - decl = ', VALUE' * argc - argv = argc \ - . times \ - . map {|i|", argv[#{i}]"} \ - . join('') - f.puts %'static void' - f.puts %'mjit_compile_invokebuiltin_for_#{func}(VALUE buf, long index, unsigned stack_size, bool inlinable_p)' - f.puts %'{' - f.puts %' rb_str_catf(buf, " VALUE self = GET_SELF();\\n");' - f.puts %' rb_str_catf(buf, " typedef VALUE (*func)(rb_execution_context_t *, VALUE#{decl});\\n");' - if inlines.has_key? cfunc_name - body_lineno, text, locals, func_name = inlines[cfunc_name] - lineno, str = generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) - f.puts %' if (inlinable_p) {' - str.gsub(/^(?!#)/, ' ').each_line {|i| - j = RubyVM::CEscape.rstring2cstr(i).dup - j.sub!(/^ return\b/ , ' val =') - f.printf(%' rb_str_catf(buf, "%%s", %s);\n', j) - } - f.puts(%' return;') - f.puts(%' }') - end - if argc > 0 - f.puts %' if (index == -1) {' - f.puts %' rb_str_catf(buf, " const VALUE *argv = &stack[%d];\\n", stack_size - #{argc});' - f.puts %' }' - f.puts %' else {' - f.puts %' rb_str_catf(buf, " const unsigned int lnum = ISEQ_BODY(GET_ISEQ())->local_table_size;\\n");' - f.puts %' rb_str_catf(buf, " const VALUE *argv = GET_EP() - lnum - VM_ENV_DATA_SIZE + 1 + %ld;\\n", index);' - f.puts %' }' - end - f.puts %' rb_str_catf(buf, " func f = (func)%"PRIuVALUE"; /* == #{cfunc_name} */\\n", (VALUE)#{cfunc_name});' - f.puts %' rb_str_catf(buf, " val = f(ec, self#{argv});\\n");' - f.puts %'}' - f.puts - } - if SUBLIBS[base] f.puts "// sub libraries" SUBLIBS[base].each do |sub| @@ -371,9 +384,9 @@ def mk_builtin_header file f.puts " // table definition" f.puts " static const struct rb_builtin_function #{table}[] = {" bs.each.with_index{|(func, (argc, cfunc_name)), i| - f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc}, mjit_compile_invokebuiltin_for_#{func})," + f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc})," } - f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0, 0)," + f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0)," f.puts " };" f.puts @@ -400,8 +413,13 @@ def mk_builtin_header file f.puts " rb_load_with_builtin_functions(#{base.dump}, #{table});" f.puts "}" - ensure - f.close + + begin + File.write(ofile, f.string) + rescue SystemCallError # EACCES, EPERM, EROFS, etc. + # Fall back to the current directory + File.write(File.basename(ofile), f.string) + end end end diff --git a/tool/mk_rbbin.rb b/tool/mk_rbbin.rb new file mode 100755 index 0000000000..991230f094 --- /dev/null +++ b/tool/mk_rbbin.rb @@ -0,0 +1,48 @@ +#!ruby -s + +OPTIMIZATION = { + inline_const_cache: true, + peephole_optimization: true, + tailcall_optimization: false, + specialized_instruction: true, + operands_unification: true, + instructions_unification: true, + frozen_string_literal: true, + debug_frozen_string_literal: false, + coverage_enabled: false, + debug_level: 0, +} + +file = File.basename(ARGV[0], ".rb") +name = "<internal:#{file}>" +iseq = RubyVM::InstructionSequence.compile(ARGF.read, name, name, **OPTIMIZATION) +puts <<C +/* -*- C -*- */ + +static const char #{file}_builtin[] = { +C +iseq.to_binary.bytes.each_slice(8) do |b| + print " ", b.map {|c| "0x%.2x," % c}.join(" ") + if $comment + print " /* ", b.pack("C*").gsub(/([[ -~]&&[^\\]])|(?m:.)/) { + (c = $1) ? "#{c} " : (c = $&.dump).size == 2 ? c : ". " + }, "*/" + end + puts +end +puts <<C +}; + +#include "ruby/ruby.h" +#include "vm_core.h" + +void +Init_#{file}(void) +{ + const char *builtin = #{file}_builtin; + size_t size = sizeof(#{file}_builtin); + VALUE code = rb_str_new_static(builtin, (long)size); + VALUE iseq = rb_funcallv(rb_cISeq, rb_intern_const("load_from_binary"), 1, &code); + rb_funcallv(iseq, rb_intern_const("eval"), 0, 0); +} +C diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb index fb35522401..db74115730 100755 --- a/tool/mkconfig.rb +++ b/tool/mkconfig.rb @@ -1,4 +1,5 @@ #!./miniruby -s +# frozen-string-literal: true # This script, which is run when ruby is built, generates rbconfig.rb by # parsing information from config.status. rbconfig.rb contains build @@ -63,8 +64,6 @@ File.foreach "config.status" do |line| when /^(?:X|(?:MINI|RUN|(?:HAVE_)?BASE|BOOTSTRAP|BTEST)RUBY(?:_COMMAND)?$)/; next when /^INSTALLDOC|TARGET$/; next when /^DTRACE/; next - when /^MJIT_(CC|SUPPORT)$/; # pass - when /^MJIT_/; next when /^(?:MAJOR|MINOR|TEENY)$/; vars[name] = val; next when /^LIBRUBY_D?LD/; next when /^RUBY_INSTALL_NAME$/; next vars[name] = (install_name = val).dup if $install_name @@ -122,11 +121,11 @@ File.foreach "config.status" do |line| universal, val = val, 'universal' if universal when /^arch$/ if universal - platform = val.sub(/universal/, %q[#{arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/]}]) + platform = val.sub(/universal/, '$(arch)') end when /^target_cpu$/ if universal - val = 'arch' + val = 'cpu' end when /^target$/ val = '"$(target_cpu)-$(target_vendor)-$(target_os)"' @@ -169,8 +168,8 @@ def vars.expand(val, config = self) val.replace(newval) unless newval == val val end -prefix = vars.expand(vars["prefix"] ||= "") -rubyarchdir = vars.expand(vars["rubyarchdir"] ||= "") +prefix = vars.expand(vars["prefix"] ||= +"") +rubyarchdir = vars.expand(vars["rubyarchdir"] ||= +"") relative_archdir = rubyarchdir.rindex(prefix, 0) ? rubyarchdir[prefix.size..-1] : rubyarchdir puts %[\ @@ -193,17 +192,19 @@ print " # Ruby installed directory.\n" print " TOPDIR = File.dirname(__FILE__).chomp!(#{relative_archdir.dump})\n" print " # DESTDIR on make install.\n" print " DESTDIR = ", (drive ? "TOPDIR && TOPDIR[/\\A[a-z]:/i] || " : ""), "'' unless defined? DESTDIR\n" -print <<'ARCH' if universal +print <<"UNIVERSAL", <<'ARCH' if universal + universal = #{universal} +UNIVERSAL arch_flag = ENV['ARCHFLAGS'] || ((e = ENV['RC_ARCHS']) && e.split.uniq.map {|a| "-arch #{a}"}.join(' ')) arch = arch_flag && arch_flag[/\A\s*-arch\s+(\S+)\s*\z/, 1] + cpu = arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/] ARCH -print " universal = #{universal}\n" if universal print " # The hash configurations stored.\n" print " CONFIG = {}\n" print " CONFIG[\"DESTDIR\"] = DESTDIR\n" versions = {} -IO.foreach(File.join(srcdir, "version.h")) do |l| +File.foreach(File.join(srcdir, "version.h")) do |l| m = /^\s*#\s*define\s+RUBY_(PATCHLEVEL)\s+(-?\d+)/.match(l) if m versions[m[1]] = m[2] @@ -224,7 +225,7 @@ IO.foreach(File.join(srcdir, "version.h")) do |l| end end if versions.size != 4 - IO.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| + File.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| m = /^\s*#\s*define\s+RUBY_API_VERSION_(\w+)\s+(-?\d+)/.match(l) if m versions[m[1]] ||= m[2] @@ -255,14 +256,14 @@ end v_others.compact! if $install_name - if install_name and vars.expand("$(RUBY_INSTALL_NAME)") == $install_name + if install_name and vars.expand(+"$(RUBY_INSTALL_NAME)") == $install_name $install_name = install_name end v_fast << " CONFIG[\"ruby_install_name\"] = \"" + $install_name + "\"\n" v_fast << " CONFIG[\"RUBY_INSTALL_NAME\"] = \"" + $install_name + "\"\n" end if $so_name - if so_name and vars.expand("$(RUBY_SO_NAME)") == $so_name + if so_name and vars.expand(+"$(RUBY_SO_NAME)") == $so_name $so_name = so_name end v_fast << " CONFIG[\"RUBY_SO_NAME\"] = \"" + $so_name + "\"\n" @@ -276,7 +277,7 @@ EOS print <<EOS if $unicode_emoji_version CONFIG["UNICODE_EMOJI_VERSION"] = #{$unicode_emoji_version.dump} EOS -print <<EOS if /darwin/ =~ arch +print prefix.start_with?("/System/") ? <<EOS : <<EOS if /darwin/ =~ arch if sdkroot = ENV["SDKROOT"] sdkroot = sdkroot.dup elsif File.exist?(File.join(CONFIG["prefix"], "include")) || @@ -287,6 +288,8 @@ print <<EOS if /darwin/ =~ arch end CONFIG["SDKROOT"] = sdkroot EOS + CONFIG["SDKROOT"] = "" +EOS print <<EOS CONFIG["platform"] = #{platform || '"$(arch)"'} CONFIG["archdir"] = "$(rubyarchdir)" @@ -391,6 +394,7 @@ print <<EOS ) end end +# Non-nil if configured for cross compiling. CROSS_COMPILING = nil unless defined? CROSS_COMPILING EOS diff --git a/tool/mkrunnable.rb b/tool/mkrunnable.rb index 8bfb4fe6a4..ef358e2425 100755 --- a/tool/mkrunnable.rb +++ b/tool/mkrunnable.rb @@ -6,6 +6,7 @@ require './rbconfig' require 'fileutils' +require_relative 'lib/path' case ARGV[0] when "-n" @@ -18,93 +19,7 @@ else include FileUtils end -module Mswin - def ln_safe(src, dest, *opt) - cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] - cmd[1, 0] = opt - return if system("cmd", "/c", *cmd) - # TODO: use RUNAS or something - puts cmd.join(" ") - end - - def ln_dir_safe(src, dest) - ln_safe(src, dest, "/d") - end -end - -def clean_link(src, dest) - begin - link = File.readlink(dest) - rescue - else - return if link == src - File.unlink(dest) - end - yield src, dest -end - -def ln_safe(src, dest) - ln_sf(src, dest) -rescue Errno::ENOENT - # Windows disallows to create broken symboic links, probably because - # it is a kind of reparse points. - raise if File.exist?(src) -end - -alias ln_dir_safe ln_safe - -case RUBY_PLATFORM -when /linux|darwin|solaris/ - def ln_exe(src, dest) - ln(src, dest, force: true) - end -else - alias ln_exe ln_safe -end - -if !File.respond_to?(:symlink) && /mingw|mswin/ =~ (CROSS_COMPILING || RUBY_PLATFORM) - extend Mswin -end - -def clean_path(path) - path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') - nil while path.sub!(/[^\/]+\/\.\.\//, '') - path -end - -def relative_path_from(path, base) - path = clean_path(path) - base = clean_path(base) - path, base = [path, base].map{|s|s.split("/")} - until path.empty? or base.empty? or path[0] != base[0] - path.shift - base.shift - end - path, base = [path, base].map{|s|s.join("/")} - if /(\A|\/)\.\.\// =~ base - File.expand_path(path) - else - base.gsub!(/[^\/]+/, '..') - File.join(base, path) - end -end - -def ln_relative(src, dest, executable = false) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - if executable - return (ln_exe(src, dest) if File.exist?(src)) - end - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_safe(s, d)} -end - -def ln_dir_relative(src, dest) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_dir_safe(s, d)} -end +include Path config = RbConfig::MAKEFILE_CONFIG.merge("prefix" => ".", "exec_prefix" => ".") config.each_value {|s| RbConfig.expand(s, config)} @@ -119,18 +34,25 @@ vendordir = config["vendordir"] rubylibdir = config["rubylibdir"] rubyarchdir = config["rubyarchdir"] archdir = "#{extout}/#{arch}" -[bindir, libdir, archdir].uniq.each do |dir| +exedir = bindir +if libdirname == "archlibdir" + exedir = exedir.sub(%r[/\K(?=[^/]+\z)]) {extout+"/"} +end +[exedir, libdir, archdir].uniq.each do |dir| File.directory?(dir) or mkdir_p(dir) end +unless exedir == bindir + ln_dir_relative(exedir, bindir) +end exeext = config["EXEEXT"] ruby_install_name = config["ruby_install_name"] rubyw_install_name = config["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name -[ruby_install_name, rubyw_install_name, goruby_install_name].map do |ruby| +[ruby_install_name, rubyw_install_name, goruby_install_name].each do |ruby| if ruby and !ruby.empty? ruby += exeext - ln_relative(ruby, "#{bindir}/#{ruby}", true) + ln_relative(ruby, "#{exedir}/#{ruby}", true) end end so = config["LIBRUBY_SO"] diff --git a/tool/notes-github-pr.rb b/tool/notes-github-pr.rb new file mode 100644 index 0000000000..d69d479cdf --- /dev/null +++ b/tool/notes-github-pr.rb @@ -0,0 +1,138 @@ +#!/usr/bin/env ruby +# Add GitHub pull request reference / author info to git notes. + +require 'net/http' +require 'uri' +require 'tmpdir' +require 'json' +require 'yaml' + +# Conversion for people whose GitHub account name and SVN_ACCOUNT_NAME are different. +GITHUB_TO_SVN = { + 'amatsuda' => 'a_matsuda', + 'matzbot' => 'git', + 'jeremyevans' => 'jeremy', + 'znz' => 'kazu', + 'k-tsj' => 'ktsj', + 'nurse' => 'naruse', + 'ioquatix' => 'samuel', + 'suketa' => 'suke', + 'unak' => 'usa', +} + +EMAIL_YML_URL = 'https://raw.githubusercontent.com/ruby/git.ruby-lang.org/refs/heads/master/config/email.yml' +SVN_TO_EMAILS = YAML.safe_load(Net::HTTP.get_response(URI(EMAIL_YML_URL)).tap(&:value).body) + +class GitHub + ENDPOINT = URI.parse('https://api.github.com') + + def initialize(access_token) + @access_token = access_token + end + + # https://developer.github.com/changes/2019-04-11-pulls-branches-for-commit/ + def pulls(owner:, repo:, commit_sha:) + resp = get("/repos/#{owner}/#{repo}/commits/#{commit_sha}/pulls", accept: 'application/vnd.github.groot-preview+json') + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/pulls/#get-a-single-pull-request + def pull_request(owner:, repo:, number:) + resp = get("/repos/#{owner}/#{repo}/pulls/#{number}") + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/users/#get-a-single-user + def user(username:) + resp = get("/users/#{username}") + JSON.parse(resp.body) + end + + private + + def get(path, accept: 'application/vnd.github.v3+json') + Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: ENDPOINT.scheme == 'https') do |http| + headers = { 'Accept': accept, 'Authorization': "bearer #{@access_token}" } + http.get(path, headers).tap(&:value) + end + end +end + +module Git + class << self + def abbrev_ref(refname, repo_path:) + git('rev-parse', '--symbolic', '--abbrev-ref', refname, repo_path: repo_path).strip + end + + def rev_list(arg, first_parent: false, repo_path: nil) + git('rev-list', *[('--first-parent' if first_parent)].compact, arg, repo_path: repo_path).lines.map(&:chomp) + end + + def commit_message(sha) + git('log', '-1', '--pretty=format:%B', sha) + end + + def notes_message(sha) + git('log', '-1', '--pretty=format:%N', sha) + end + + def committer_name(sha) + git('log', '-1', '--pretty=format:%cn', sha) + end + + def committer_email(sha) + git('log', '-1', '--pretty=format:%cE', sha) + end + + private + + def git(*cmd, repo_path: nil) + env = {} + if repo_path + env['GIT_DIR'] = repo_path + end + out = IO.popen(env, ['git', *cmd], &:read) + unless $?.success? + abort "Failed to execute: git #{cmd.join(' ')}\n#{out}" + end + out + end + end +end + +github = GitHub.new(ENV.fetch('GITHUB_TOKEN')) + +repo_path, *rest = ARGV +rest.each_slice(3).map do |oldrev, newrev, _refname| + system('git', 'fetch', 'origin', 'refs/notes/commits:refs/notes/commits', exception: true) + + updated = false + Git.rev_list("#{oldrev}..#{newrev}", first_parent: true).each do |sha| + github.pulls(owner: 'ruby', repo: 'ruby', commit_sha: sha).each do |pull| + number = pull.fetch('number') + url = pull.fetch('html_url') + next unless url.start_with?('https://github.com/ruby/ruby/pull/') + + # "Merged" notes for "Squash and merge" + message = Git.commit_message(sha) + notes = Git.notes_message(sha) + if !message.include?(url) && !message.match(/[ (]##{number}[) ]/) && !notes.include?(url) + system('git', 'notes', 'append', '-m', "Merged: #{url}", sha, exception: true) + updated = true + end + + # "Merged-By" notes for "Rebase and merge" + if Git.committer_name(sha) == 'GitHub' && Git.committer_email(sha) == 'noreply@github.com' + username = github.pull_request(owner: 'ruby', repo: 'ruby', number: number).fetch('merged_by').fetch('login') + email = github.user(username: username).fetch('email') + email ||= SVN_TO_EMAILS[GITHUB_TO_SVN.fetch(username, username)]&.first + system('git', 'notes', 'append', '-m', "Merged-By: #{username}#{(" <#{email}>" if email)}", sha, exception: true) + updated = true + end + end + end + + if updated + system('git', 'push', 'origin', 'refs/notes/commits', exception: true) + end +end diff --git a/tool/notify-slack-commits.rb b/tool/notify-slack-commits.rb new file mode 100644 index 0000000000..73e22b9a03 --- /dev/null +++ b/tool/notify-slack-commits.rb @@ -0,0 +1,87 @@ +#!/usr/bin/env ruby + +require "net/https" +require "open3" +require "json" +require "digest/md5" + +SLACK_WEBHOOK_URLS = [ + ENV.fetch("SLACK_WEBHOOK_URL_ALERTS").chomp, # ruby-lang#alerts + ENV.fetch("SLACK_WEBHOOK_URL_COMMITS").chomp, # ruby-lang#commits + ENV.fetch("SLACK_WEBHOOK_URL_RUBY_JP").chomp, # ruby-jp#ruby-commits +] +GRAVATAR_OVERRIDES = { + "nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars0.githubusercontent.com/u/21976", + "noreply@github.com" => "https://avatars1.githubusercontent.com/u/9919", + "nurse@users.noreply.github.com" => "https://avatars1.githubusercontent.com/u/13423", + "svn-admin@ruby-lang.org" => "https://avatars1.githubusercontent.com/u/29403229", + "svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars1.githubusercontent.com/u/29403229", + "usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars2.githubusercontent.com/u/17790", + "usa@ruby-lang.org" => "https://avatars2.githubusercontent.com/u/17790", + "yui-knk@ruby-lang.org" => "https://avatars0.githubusercontent.com/u/5356517", + "znz@users.noreply.github.com" => "https://avatars3.githubusercontent.com/u/11857", +} + +def escape(s) + s.gsub(/[&<>]/, "&" => "&", "<" => "<", ">" => ">") +end + +ARGV.each_slice(3) do |oldrev, newrev, refname| + out, = Open3.capture2("git", "rev-parse", "--symbolic", "--abbrev-ref", refname) + branch = out.strip + + out, = Open3.capture2("git", "log", "--pretty=format:%H\n%h\n%cn\n%ce\n%ct\n%B", "--abbrev=10", "-z", "#{oldrev}..#{newrev}") + + attachments = [] + out.split("\0").reverse_each do |s| + sha, sha_abbr, committer, committeremail, committertime, body = s.split("\n", 6) + subject, body = body.split("\n", 2) + + # Append notes content to `body` if it's notes + if refname.match(%r[\Arefs/notes/\w+\z]) + # `--diff-filter=AM -M` to exclude rename by git's directory optimization + object = IO.popen(["git", "diff", "--diff-filter=AM", "-M", "--name-only", "#{sha}^..#{sha}"], &:read).chomp + if md = object.match(/\A(?<prefix>\h{2})\/?(?<rest>\h{38})\z/) + body = [body, IO.popen(["git", "notes", "show", md[:prefix] + md[:rest]], &:read)].join + end + end + + gravatar = GRAVATAR_OVERRIDES.fetch(committeremail) do + "https://www.gravatar.com/avatar/#{ Digest::MD5.hexdigest(committeremail.downcase) }" + end + + attachments << { + title: "#{ sha_abbr } (#{ branch }): #{ escape(subject) }", + title_link: "https://github.com/ruby/ruby/commit/#{ sha }", + text: escape((body || "").strip), + footer: committer, + footer_icon: gravatar, + ts: committertime.to_i, + color: '#24282D', + } + end + + # 100 attachments cannot be exceeded. 20 is recommended. https://api.slack.com/docs/message-attachments + attachments.each_slice(20).each do |attachments_group| + payload = { attachments: attachments_group } + + #Net::HTTP.post( + # URI.parse(SLACK_WEBHOOK_URL), + # JSON.generate(payload), + # "Content-Type" => "application/json" + #) + responses = SLACK_WEBHOOK_URLS.map do |url| + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.start do + req = Net::HTTP::Post.new(uri.path) + req.set_form_data(payload: payload.to_json) + http.request(req) + end + end + + results = responses.map { |resp| "#{resp.code} (#{resp.body})" }.join(', ') + puts "#{results} -- #{payload.to_json}" + end +end diff --git a/tool/outdate-bundled-gems.rb b/tool/outdate-bundled-gems.rb index bf6b268700..47ee80bc89 100755 --- a/tool/outdate-bundled-gems.rb +++ b/tool/outdate-bundled-gems.rb @@ -3,26 +3,60 @@ require 'fileutils' require 'rubygems' fu = FileUtils::Verbose + until ARGV.empty? case ARGV.first when '--' ARGV.shift break - when '-n', '--dryrun' + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Don't remove fu = FileUtils::DryRun when /\A--make=/ # just to run when `make -n` when /\A--mflags=(.*)/ fu = FileUtils::DryRun if /\A-\S*n/ =~ $1 + when /\A--gem[-_]platform=(.*)/im + ## --gem-platform=PLATFORM Platform in RubyGems style + gem_platform = $1 + ruby_platform = nil + when /\A--ruby[-_]platform=(.*)/im + ## --ruby-platform=PLATFORM Platform in Ruby style + ruby_platform = $1 + gem_platform = nil + when /\A--ruby[-_]version=(.*)/im + ## --ruby-version=VERSION Ruby version to keep + ruby_version = $1 + when /\A--only=(?:(curdir|srcdir)|all)\z/im + ## --only=(curdir|srcdir|all) Specify directory to remove gems from + only = $1&.downcase + when /\A--all\z/im + ## --all Remove all gems not only bundled gems + all = true + when /\A--help\z/im + ## --help Print this message + puts "Usage: #$0 [options] [srcdir]" + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ raise "#{$0}: unknown option: #{ARGV.first}" else break end + ## ARGV.shift end +gem_platform ||= Gem::Platform.new(ruby_platform).to_s if ruby_platform + class Removal + attr_reader :base + def initialize(base = nil) @base = (File.join(base, "/") if base) @remove = {} @@ -64,24 +98,39 @@ class Removal } end + def sorted + @remove.sort_by {|k, | [-k.count("/"), k]} + end + def each_file - @remove.each {|k, v| yield prefixed(k) if v == :rm_f} + sorted.each {|k, v| yield prefixed(k) if v == :rm_f} end def each_directory - @remove.each {|k, v| yield prefixed(k) if v == :rm_rf} + sorted.each {|k, v| yield prefixed(k) if v == :rm_rf} end end srcdir = Removal.new(ARGV.shift) -curdir = Removal.new +curdir = !srcdir.base || File.identical?(srcdir.base, ".") ? srcdir : Removal.new + +bundled = File.readlines("#{srcdir.base}gems/bundled_gems"). + grep(/^(\w[^\#\s]+)\s+[^\#\s]+(?:\s+[^\#\s]+\s+([^\#\s]+))?/) {$~.captures}.to_h rescue nil srcdir.glob(".bundle/gems/*/") do |dir| - unless srcdir.exist?("gems/#{File.basename(dir)}.gem") + base = File.basename(dir) + next if !all && bundled && !bundled.key?(base[/\A.+(?=-)/]) + unless srcdir.exist?("gems/#{base}.gem") srcdir.rmdir(dir) end end +srcdir.glob(".bundle/.timestamp/*.revision") do |file| + unless bundled&.fetch(File.basename(file, ".revision"), nil) + srcdir.unlink(file) + end +end + srcdir.glob(".bundle/specifications/*.gemspec") do |spec| unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}/") srcdir.unlink(spec) @@ -102,34 +151,40 @@ curdir.glob(".bundle/gems/*/") do |dir| end end -platform = Gem::Platform.local.to_s curdir.glob(".bundle/{extensions,.timestamp}/*/") do |dir| - unless File.basename(dir) == platform + unless gem_platform and File.fnmatch?(gem_platform, File.basename(dir)) curdir.rmdir(dir) end end -baseruby_version = RbConfig::CONFIG['ruby_version'] # This may not have "-static" -curdir.glob(".bundle/{extensions,.timestamp}/#{platform}/*/") do |dir| - version = File.basename(dir).split('-', 2).first # Remove "-static" if exists - unless version == baseruby_version - curdir.rmdir(dir) +if gem_platform + curdir.glob(".bundle/{extensions,.timestamp}/#{gem_platform}/*/") do |dir| + unless ruby_version and File.fnmatch?(ruby_version, File.basename(dir, '-static')) + curdir.rmdir(dir) + end end end -curdir.glob(".bundle/extensions/#{platform}/#{baseruby_version}/*/") do |dir| - unless curdir.exist?(".bundle/specifications/#{File.basename(dir)}.gemspec") - curdir.rmdir(dir) +if ruby_version + curdir.glob(".bundle/extensions/#{gem_platform || '*'}/#{ruby_version}/*/") do |dir| + unless curdir.exist?(".bundle/specifications/#{File.basename(dir)}.gemspec") + curdir.rmdir(dir) + end end -end -curdir.glob(".bundle/.timestamp/#{platform}/#{baseruby_version}/.*.time") do |stamp| - unless curdir.directory?(File.join(".bundle", stamp[%r[/\.([^/]+)\.time\z], 1].gsub('.-.', '/'))) - curdir.unlink(stamp) + curdir.glob(".bundle/.timestamp/#{gem_platform || '*'}/#{ruby_version}/.*.time") do |stamp| + dir = stamp[%r[/\.([^/]+)\.time\z], 1].gsub('.-.', '/')[%r[\A[^/]+/[^/]+]] + unless curdir.directory?(File.join(".bundle", dir)) + curdir.unlink(stamp) + end end end -srcdir.each_file {|f| fu.rm_f(f)} -srcdir.each_directory {|d| fu.rm_rf(d)} -curdir.each_file {|f| fu.rm_f(f)} -curdir.each_directory {|d| fu.rm_rf(d)} +unless only == "curdir" + srcdir.each_file {|f| fu.rm_f(f)} + srcdir.each_directory {|d| fu.rm_rf(d)} +end +unless only == "srcdir" or curdir.equal?(srcdir) + curdir.each_file {|f| fu.rm_f(f)} + curdir.each_directory {|d| fu.rm_rf(d)} +end diff --git a/tool/prereq.status b/tool/prereq.status index 6de00c8a92..6aca615e90 100644 --- a/tool/prereq.status +++ b/tool/prereq.status @@ -41,4 +41,5 @@ s,@rubylibprefix@,,g s,@srcdir@,.,g s/@[A-Za-z][A-Za-z0-9_]*@//g -s/{\$([A-Za-z]*)}//g +s/{\$([^(){}]*)}//g +s/^!/#!/ diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 85d05eff25..874c3ef1d9 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -28,6 +28,7 @@ begin rescue LoadError $" << "zlib.rb" end +require_relative 'lib/path' INDENT = " "*36 STDOUT.sync = true @@ -96,6 +97,20 @@ def parse_args(argv = ARGV) opt.on('--gnumake') {gnumake = true} opt.on('--debug-symbols=SUFFIX', /\w+/) {|name| $debug_symbols = ".#{name}"} + unless $install_procs.empty? + w = (w = ENV["COLUMNS"] and (w = w.to_i) > 80) ? w - 30 : 50 + opt.on("\n""Types for --install and --exclude:") + mesg = +" " + $install_procs.each_key do |t| + if mesg.size + t.size > w + opt.on(mesg) + mesg = +" " + end + mesg << " " << t.to_s + end + opt.on(mesg) + end + opt.order!(argv) do |v| case v when /\AINSTALL[-_]([-\w]+)=(.*)/ @@ -135,6 +150,7 @@ def parse_args(argv = ARGV) end $destdir ||= $mflags.defined?("DESTDIR") + $destdir = File.expand_path($destdir) unless $destdir.empty? if $extout ||= $mflags.defined?("EXTOUT") RbConfig.expand($extout) end @@ -157,6 +173,16 @@ def parse_args(argv = ARGV) end end +Compressors = {".gz"=>"gzip", ".bz2"=>"bzip2"} +def Compressors.for(type) + ext = File.extname(type) + if compress = fetch(ext, nil) + [type.chomp(ext), ext, compress] + else + [type, *find {|_, z| system(z, in: IO::NULL, out: IO::NULL)}] + end +end + $install_procs = Hash.new {[]} def install?(*types, &block) unless types.delete(:nodefault) @@ -205,15 +231,20 @@ def ln_sf(src, dest) end $made_dirs = {} + +def dir_creating(dir) + $made_dirs.fetch(dir) do + $made_dirs[dir] = true + $installed_list.puts(File.join(dir, "")) if $installed_list + yield if defined?(yield) + end +end + def makedirs(dirs) dirs = fu_list(dirs) dirs.collect! do |dir| realdir = with_destdir(dir) - realdir unless $made_dirs.fetch(dir) do - $made_dirs[dir] = true - $installed_list.puts(File.join(dir, "")) if $installed_list - File.directory?(realdir) - end + realdir unless dir_creating(dir) {File.directory?(realdir)} end.compact! super(dirs, :mode => $dir_mode) unless dirs.empty? end @@ -346,6 +377,9 @@ rubyw_install_name = CONFIG["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name bindir = CONFIG["bindir", true] +if CONFIG["libdirname"] == "archlibdir" + archbindir = bindir.sub(%r[/\K(?=[^/]+\z)]) {CONFIG["config_target"] + "/"} +end libdir = CONFIG[CONFIG.fetch("libdirname", "libdir"), true] rubyhdrdir = CONFIG["rubyhdrdir", true] archhdrdir = CONFIG["rubyarchhdrdir"] || (rubyhdrdir + "/" + CONFIG['arch']) @@ -369,113 +403,13 @@ load_relative = CONFIG["LIBRUBY_RELATIVE"] == 'yes' rdoc_noinst = %w[created.rid] -install?(:local, :arch, :bin, :'bin-arch') do - prepare "binary commands", bindir - - install ruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - if rubyw_install_name and !rubyw_install_name.empty? - install rubyw_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support - # to load and execute the second .wasm file. Both are required to execute ruby - if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" - install ruby_install_name+".wasm", bindir, :mode => $prog_mode, :strip => $strip - end - if File.exist? goruby_install_name+exeext - install goruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - if enable_shared and dll != lib - install dll, bindir, :mode => $prog_mode, :strip => $strip - end -end - -install?(:local, :arch, :lib, :'lib-arch') do - prepare "base libraries", libdir - - install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc - install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" - if dll == lib and dll != arc - for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] - ln_sf(dll, File.join(libdir, link)) - end - end - - prepare "arch files", archlibdir - install "rbconfig.rb", archlibdir, :mode => $data_mode - if CONFIG["ARCHFILE"] - for file in CONFIG["ARCHFILE"].split - install file, archlibdir, :mode => $data_mode - end - end -end - -install?(:local, :arch, :data) do - pc = CONFIG["ruby_pc"] - if pc and File.file?(pc) and File.size?(pc) - prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") - install pc, pkgconfigdir, :mode => $data_mode - end -end - -install?(:ext, :arch, :'ext-arch') do - prepare "extension objects", archlibdir - noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split - install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) - prepare "extension objects", sitearchlibdir - prepare "extension objects", vendorarchlibdir - if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and - !(extso = extso.gsub(/\\\n/, '').split).empty? - libpathenv = CONFIG["LIBPATHENV"] - dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] - prepare "external libraries", dest - for file in extso - install file, dest, :mode => $prog_mode - end - end -end -install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do - prepare "extension headers", archhdrdir - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.obj", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.pch", :mode => $data_mode) -end -install?(:ext, :comm, :'ext-comm') do - prepare "extension scripts", rubylibdir - install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) - prepare "extension scripts", sitelibdir - prepare "extension scripts", vendorlibdir -end -install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do - hdrdir = rubyhdrdir + "/ruby" - prepare "extension headers", hdrdir - install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) -end - -install?(:doc, :rdoc) do - if $rdocdir - ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") - prepare "rdoc", ridatadir - install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :html) do - if $htmldir - prepare "html-docs", docdir - install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :capi) do - prepare "capi-docs", docdir - install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode -end - prolog_script = <<EOS bindir="#{load_relative ? '${0%/*}' : bindir.gsub(/\"/, '\\\\"')}" EOS -if CONFIG["LIBRUBY_RELATIVE"] != 'yes' and libpathenv = CONFIG["LIBPATHENV"] +if !load_relative and libpathenv = CONFIG["LIBPATHENV"] pathsep = File::PATH_SEPARATOR prolog_script << <<EOS -libdir="#{load_relative ? '$\{bindir%/bin\}/lib' : libdir.gsub(/\"/, '\\\\"')}" +libdir="#{libdir.gsub(/\"/, '\\\\"')}" export #{libpathenv}="$libdir${#{libpathenv}:+#{pathsep}$#{libpathenv}}" EOS end @@ -584,129 +518,6 @@ $script_installer = Class.new(installer) do break new(ruby_shebang, ruby_bin, ruby_install_name, nil, trans) end -install?(:local, :comm, :bin, :'bin-comm') do - prepare "command scripts", bindir - - install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| - $script_installer.install(src, cmd) - end -end - -install?(:local, :comm, :lib) do - prepare "library scripts", rubylibdir - noinst = %w[*.txt *.rdoc *.gemspec] - install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) -end - -install?(:local, :comm, :hdr, :'comm-hdr') do - prepare "common headers", rubyhdrdir - - noinst = [] - unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ - noinst << "win32.h" - end - noinst = nil if noinst.empty? - install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) -end - -install?(:local, :comm, :man) do - mdocs = Dir["#{srcdir}/man/*.[1-9]"] - prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} - - case $mantype - when /\.(?:(gz)|bz2)\z/ - compress = $1 ? "gzip" : "bzip2" - suffix = $& - end - mandir = File.join(mandir, "man") - has_goruby = File.exist?(goruby_install_name+exeext) - require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype - mdocs.each do |mdoc| - next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' - base = File.basename(mdoc) - if base == "goruby.1" - next unless has_goruby - end - - destdir = mandir + (section = mdoc[/\d+$/]) - destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) - destfile = File.join(destdir, "#{destname}.#{section}") - - if /\Adoc\b/ =~ $mantype - if compress - begin - w = IO.popen(compress, "rb", in: mdoc, &:read) - rescue - else - destfile << suffix - end - end - if w - open_for_install(destfile, $data_mode) {w} - else - install mdoc, destfile, :mode => $data_mode - end - else - class << (w = []) - alias print push - end - if File.basename(mdoc).start_with?('bundle') || - File.basename(mdoc).start_with?('gemfile') - w = File.read(mdoc) - else - File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} - w = w.join("") - end - if compress - begin - w = IO.popen(compress, "r+b") do |f| - Thread.start {f.write w; f.close_write} - f.read - end - rescue - else - destfile << suffix - end - end - open_for_install(destfile, $data_mode) {w} - end - end -end - -install?(:dbg, :nodefault) do - prepare "debugger commands", bindir - prepare "debugger scripts", rubylibdir - conf = RbConfig::MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) - Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| - cmd = $script_installer.transform(File.basename(src, ".in")) - open_for_install(File.join(bindir, cmd), $script_mode) { - RbConfig.expand(File.read(src), conf) - } - end - Dir.glob(File.join(srcdir, "misc/lldb_*")) do |src| - if File.directory?(src) - install_recursive src, File.join(rubylibdir, File.basename(src)) - else - install src, rubylibdir - end - end - install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") - if $debug_symbols - { - ruby_install_name => bindir, - rubyw_install_name => bindir, - goruby_install_name => bindir, - dll => libdir, - }.each do |src, dest| - next if src.empty? - src += $debug_symbols - if File.directory?(src) - install_recursive src, File.join(dest, src) - end - end - end -end - module RbInstall def self.no_write(options = nil) u = File.umask(0022) @@ -716,6 +527,8 @@ module RbInstall const_set(:FileUtils, fu::NoWrite) fu end + # RubyGems 3.0.0 or later supports `dir_mode`, but it uses + # `File` method to apply it, not `FileUtils`. dir_mode = options.delete(:dir_mode) if options end yield @@ -745,47 +558,119 @@ module RbInstall end def collect - ruby_libraries.sort + requirable_features.sort + end + + private + + def features_from_makefile(makefile_path) + makefile = File.read(makefile_path) + + name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + return [] if name.nil? || name.empty? + + feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = feature.sub("$(TARGET)", name) + + target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? + + Array(feature) end class Ext < self - def skip_install?(files) + def requirable_features # install ext only when it's configured - !File.exist?("#{$ext_build_dir}/#{relative_base}/Makefile") + return [] unless File.exist?(makefile_path) + + ruby_features + ext_features + end + + private + + def ruby_features + Dir.glob("**/*.rb", base: "#{makefile_dir}/lib") + end + + def ext_features + features_from_makefile(makefile_path) end - def ruby_libraries - Dir.glob("lib/**/*.rb", base: "#{srcdir}/ext/#{relative_base}") + def makefile_path + if File.exist?("#{makefile_dir}/Makefile") + "#{makefile_dir}/Makefile" + else + # for out-of-place build + "#{$ext_build_dir}/#{relative_base}/Makefile" + end + end + + def makefile_dir + "#{root}/#{relative_base}" + end + + def root + File.expand_path($ext_build_dir, srcdir) end end class Lib < self - def skip_install?(files) - files.empty? + def requirable_features + ruby_features + ext_features end - def ruby_libraries + private + + def ruby_features gemname = File.basename(gemspec, ".gemspec") base = relative_base || gemname # for lib/net/net-smtp.gemspec if m = /.*(?=-(.*)\z)/.match(gemname) base = File.join(base, *m.to_a.select {|n| !base.include?(n)}) end - files = Dir.glob("lib/#{base}{.rb,/**/*.rb}", base: srcdir) + files = Dir.glob("#{base}{.rb,/**/*.rb}", base: root) if !relative_base and files.empty? # no files at the toplevel # pseudo gem like ruby2_keywords - files << "lib/#{gemname}.rb" + files << "#{gemname}.rb" end case gemname when "net-http" - files << "lib/net/https.rb" + files << "net/https.rb" when "optparse" - files << "lib/optionparser.rb" + files << "optionparser.rb" end files end + + def ext_features + loaded_gemspec = load_gemspec("#{root}/#{gemspec}") + extension = loaded_gemspec.extensions.first + return [] unless extension + + extconf = File.expand_path(extension, srcdir) + ext_build_dir = File.dirname(extconf) + makefile_path = "#{ext_build_dir}/Makefile" + return [] unless File.exist?(makefile_path) + + features_from_makefile(makefile_path) + end + + def root + "#{srcdir}/lib" + end + end + + class UnpackedGem < self + def collect + base = @srcdir or return [] + Dir.glob("**/*", File::FNM_DOTMATCH, base: base).select do |n| + next if n == "." + next if File.fnmatch?("*.gemspec", n, File::FNM_DOTMATCH|File::FNM_PATHNAME) + !File.directory?(File.join(base, n)) + end + end end end end @@ -823,10 +708,78 @@ module RbInstall end end - class GemInstaller < Gem::Installer - end + class UnpackedInstaller < Gem::Installer + # This method is mostly copied from old version of Gem::Installer#install + def install_with_default_gem + verify_gem_home + + # The name and require_paths must be verified first, since it could contain + # ruby code that would be eval'ed in #ensure_loadable_spec + verify_spec + + ensure_loadable_spec + + if options[:install_as_default] + Gem.ensure_default_gem_subdirectories gem_home + else + Gem.ensure_gem_subdirectories gem_home + end + + return true if @force + + ensure_dependencies_met unless @ignore_dependencies + + run_pre_install_hooks + + # Set loaded_from to ensure extension_dir is correct + if @options[:install_as_default] + spec.loaded_from = default_spec_file + else + spec.loaded_from = spec_file + end + + # Completely remove any previous gem files + FileUtils.rm_rf gem_dir + FileUtils.rm_rf spec.extension_dir + + dir_mode = options[:dir_mode] + FileUtils.mkdir_p gem_dir, mode: dir_mode && 0o755 + + if @options[:install_as_default] + extract_bin + write_default_spec + else + extract_files + + build_extensions + write_build_info_file + run_post_build_hooks + end + + generate_bin + generate_plugins + + unless @options[:install_as_default] + write_spec + write_cache_file + end + + File.chmod(dir_mode, gem_dir) if dir_mode + + say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? + + Gem::Specification.add_spec(spec) unless @install_dir + + load_plugin + + run_post_install_hooks + + spec + rescue Errno::EACCES => e + # Permission denied - /path/to/foo + raise Gem::FilePermissionError, e.message.split(" - ").last + end - class UnpackedInstaller < GemInstaller def write_cache_file end @@ -850,11 +803,6 @@ module RbInstall super end - def generate_bin_script(filename, bindir) - return if same_bin_script?(filename, bindir) - super - end - def same_bin_script?(filename, bindir) path = File.join(bindir, formatted_program_filename(filename)) begin @@ -873,12 +821,11 @@ module RbInstall super unless $dryrun $installed_list.puts(without_destdir(default_spec_file)) if $installed_list end - end - class GemInstaller def install spec.post_install_message = nil - RbInstall.no_write(options) {super} + dir_creating(without_destdir(gem_dir)) + RbInstall.no_write(options) { install_with_default_gem } end # Now build-ext builds all extensions including bundled gems. @@ -886,10 +833,13 @@ module RbInstall end def generate_bin_script(filename, bindir) + return if same_bin_script?(filename, bindir) name = formatted_program_filename(filename) unless $dryrun super - File.chmod($script_mode, File.join(bindir, name)) + script = File.join(bindir, name) + File.chmod($script_mode, script) + File.unlink("#{script}.lock") rescue nil end $installed_list.puts(File.join(without_destdir(bindir), name)) if $installed_list end @@ -904,38 +854,45 @@ module RbInstall $installed_list.puts(d+"/") if $installed_list end end - end -end -# :startdoc: - -install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do - install_default_gem('lib', srcdir, bindir) -end -install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do - install_default_gem('ext', srcdir, bindir) + def load_plugin + # Suppress warnings for constant re-assignment + verbose, $VERBOSE = $VERBOSE, nil + super + ensure + $VERBOSE = verbose + end + end end -def load_gemspec(file, base = nil) +def load_gemspec(file, base = nil, files: nil) file = File.realpath(file) code = File.read(file, encoding: "utf-8:-") - code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split\([^\)]*\)/m) do - files = [] - if base - Dir.glob("**/*", File::FNM_DOTMATCH, base: base) do |n| - case File.basename(n); when ".", ".."; next; end - next if File.directory?(File.join(base, n)) - files << n.dump - end - end - "[" + files.join(", ") + "]" + + code.gsub!(/^ *#.*/, "") + spec_files = files ? files.map(&:dump).join(", ") : "" + code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split(\([^\)]*\))?/m) do + "[" + spec_files + "]" + end \ + or + code.gsub!(/IO\.popen\(.*git.*?\)/) do + "[" + spec_files + "] || itself" end + spec = eval(code, binding, file) + # for out-of-place build + collected_files = files ? spec.files.concat(files).uniq : spec.files + spec.files = collected_files.map do |f| + if !File.exist?(File.join(base || ".", f)) && f.end_with?(".rb") + "lib/#{f}" + else + f + end + end unless Gem::Specification === spec raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." end spec.loaded_from = base ? File.join(base, File.basename(file)) : file - spec.files.reject! {|n| n.end_with?(".gemspec") or n.start_with?(".git")} spec.date = RUBY_RELEASE_DATE spec @@ -946,6 +903,7 @@ def install_default_gem(dir, srcdir, bindir) install_dir = with_destdir(gem_dir) prepare "default gems from #{dir}", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_default_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -964,14 +922,11 @@ def install_default_gem(dir, srcdir, bindir) base = "#{srcdir}/#{dir}" gems = Dir.glob("**/*.gemspec", base: base).map {|src| - spec = load_gemspec("#{base}/#{src}") - file_collector = RbInstall::Specs::FileCollector.for(srcdir, dir, src) - files = file_collector.collect - if file_collector.skip_install?(files) + files = RbInstall::Specs::FileCollector.for(srcdir, dir, src).collect + if files.empty? next end - spec.files = files - spec + load_gemspec("#{base}/#{src}", files: files) } gems.compact.sort_by(&:name).each do |gemspec| old_gemspecs = Dir[File.join(with_destdir(default_spec_dir), "#{gemspec.name}-*.gemspec")] @@ -990,11 +945,259 @@ def install_default_gem(dir, srcdir, bindir) end end +def mdoc_file?(mdoc) + /^\.Nm / =~ File.read(mdoc, 1024) +end + +# :startdoc: + +install?(:local, :arch, :bin, :'bin-arch') do + prepare "binary commands", (dest = archbindir || bindir) + + def (bins = []).add(name) + push(name) + name + end + + install bins.add(ruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + if rubyw_install_name and !rubyw_install_name.empty? + install bins.add(rubyw_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support + # to load and execute the second .wasm file. Both are required to execute ruby + if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" + install bins.add(ruby_install_name+".wasm"), dest, :mode => $prog_mode, :strip => $strip + end + if File.exist? goruby_install_name+exeext + install bins.add(goruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + if enable_shared and dll != lib + install bins.add(dll), dest, :mode => $prog_mode, :strip => $strip + end + if archbindir + prepare "binary command links", bindir + relpath = Path.relative(archbindir, bindir) + bins.each do |f| + ln_sf(File.join(relpath, f), File.join(bindir, f)) + end + end +end + +install?(:local, :arch, :lib, :'lib-arch') do + prepare "base libraries", libdir + + install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc + install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" + if dll == lib and dll != arc + for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] + ln_sf(dll, File.join(libdir, link)) + end + end + + prepare "arch files", archlibdir + install "rbconfig.rb", archlibdir, :mode => $data_mode + if CONFIG["ARCHFILE"] + for file in CONFIG["ARCHFILE"].split + install file, archlibdir, :mode => $data_mode + end + end +end + +install?(:local, :arch, :data) do + pc = CONFIG["ruby_pc"] + if pc and File.file?(pc) and File.size?(pc) + prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") + install pc, pkgconfigdir, :mode => $data_mode + if (pkgconfig_base = CONFIG["libdir", true]) != libdir + prepare "pkgconfig data link", File.join(pkgconfig_base, "pkgconfig") + ln_sf(File.join("..", Path.relative(pkgconfigdir, pkgconfig_base), pc), + File.join(pkgconfig_base, "pkgconfig", pc)) + end + end +end + +install?(:ext, :arch, :'ext-arch') do + prepare "extension objects", archlibdir + noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split + install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) + prepare "extension objects", sitearchlibdir + prepare "extension objects", vendorarchlibdir + if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and + !(extso = extso.gsub(/\\\n/, '').split).empty? + libpathenv = CONFIG["LIBPATHENV"] + dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] + prepare "external libraries", dest + for file in extso + install file, dest, :mode => $prog_mode + end + end +end + +install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do + prepare "extension headers", archhdrdir + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:ext, :comm, :'ext-comm') do + prepare "extension scripts", rubylibdir + install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) + prepare "extension scripts", sitelibdir + prepare "extension scripts", vendorlibdir +end + +install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do + hdrdir = rubyhdrdir + "/ruby" + prepare "extension headers", hdrdir + install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:doc, :rdoc) do + if $rdocdir + ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") + prepare "rdoc", ridatadir + install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :html) do + if $htmldir + prepare "html-docs", docdir + install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :capi) do + prepare "capi-docs", docdir + install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode +end + +install?(:local, :comm, :bin, :'bin-comm') do + prepare "command scripts", bindir + + install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| + $script_installer.install(src, cmd) + end +end + +install?(:local, :comm, :lib) do + prepare "library scripts", rubylibdir + noinst = %w[*.txt *.rdoc *.gemspec] + install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) +end + +install?(:local, :comm, :hdr, :'comm-hdr') do + prepare "common headers", rubyhdrdir + + noinst = [] + unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ + noinst << "win32.h" + end + noinst = nil if noinst.empty? + install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) +end + +install?(:local, :comm, :man) do + mdocs = Dir["#{srcdir}/man/*.[1-9]"] + prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} + + mantype, suffix, compress = Compressors.for($mantype) + has_goruby = File.exist?(goruby_install_name+exeext) + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ mantype + mdocs.each do |mdoc| + next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' + base = File.basename(mdoc) + if base == "goruby.1" + next unless has_goruby + end + + destdir = File.join(mandir, "man" + (section = mdoc[/\d+$/])) + destname = $script_installer.transform(base.chomp(".#{section}")) + destfile = File.join(destdir, "#{destname}.#{section}") + + if /\Adoc\b/ =~ mantype or !mdoc_file?(mdoc) + if compress + begin + w = IO.popen(compress, "rb", in: mdoc, &:read) + rescue + else + destfile << suffix + end + end + if w + open_for_install(destfile, $data_mode) {w} + else + install mdoc, destfile, :mode => $data_mode + end + else + class << (w = []) + alias print push + end + File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + w = w.join("") + if compress + begin + w = IO.popen(compress, "r+b") do |f| + Thread.start {f.write w; f.close_write} + f.read + end + rescue + else + destfile << suffix + end + end + open_for_install(destfile, $data_mode) {w} + end + end +end + +install?(:dbg, :nodefault) do + prepare "debugger commands", bindir + prepare "debugger scripts", rubylibdir + conf = MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) + Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| + cmd = $script_installer.transform(File.basename(src, ".in")) + open_for_install(File.join(bindir, cmd), $script_mode) { + RbConfig.expand(File.read(src), conf) + } + end + Dir.glob(File.join(srcdir, "misc/lldb_*")) do |src| + if File.directory?(src) + install_recursive src, File.join(rubylibdir, File.basename(src)) + else + install src, rubylibdir + end + end + install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") + if $debug_symbols + { + ruby_install_name => archbindir || bindir, + rubyw_install_name => archbindir || bindir, + goruby_install_name => archbindir || bindir, + dll => libdir, + }.each do |src, dest| + next if src.empty? + src += $debug_symbols + if File.directory?(src) + install_recursive src, File.join(dest, src) + end + end + end +end + +install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do + install_default_gem('lib', srcdir, bindir) +end + +install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do + install_default_gem('ext', srcdir, bindir) +end + install?(:ext, :comm, :gem, :'bundled-gems') do gem_dir = Gem.default_dir install_dir = with_destdir(gem_dir) prepare "bundled gems", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -1023,19 +1226,30 @@ install?(:ext, :comm, :gem, :'bundled-gems') do # the newly installed ruby. ENV.delete('RUBYOPT') + collector = RbInstall::Specs::FileCollector::UnpackedGem File.foreach("#{srcdir}/gems/bundled_gems") do |name| next if /^\s*(?:#|$)/ =~ name next unless /^(\S+)\s+(\S+).*/ =~ name + gem = $1 gem_name = "#$1-#$2" - path = "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" - unless File.exist?(path) - path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec" - unless File.exist?(path) - skipped[gem_name] = "gemspec not found" - next - end + path = [ + # gemspec that removed duplicated dependencies of bundled gems + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec", + # gemspec for C ext gems, It has the original dependencies + # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec", + # original gemspec generated by rubygems + "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + ].find { |gemspec| File.exist?(gemspec) } + if path.nil? + skipped[gem_name] = "gemspec not found" + next end - spec = load_gemspec(path, "#{srcdir}/.bundle/gems/#{gem_name}") + base = "#{srcdir}/.bundle/gems/#{gem_name}" + files = collector.new(path, base, nil).collect + files.delete("#{gem}.gemspec") + files.delete("#{gem_name}.gemspec") + spec = load_gemspec(path, base, files: files) unless spec.platform == Gem::Platform::RUBY skipped[gem_name] = "not ruby platform (#{spec.platform})" next @@ -1044,7 +1258,13 @@ install?(:ext, :comm, :gem, :'bundled-gems') do skipped[gem_name] = "full name unmatch #{spec.full_name}" next end + # Skip install C ext bundled gem if it is build failed or not found + if !spec.extensions.empty? && !File.exist?("#{build_dir}/#{gem_name}/gem.build_complete") + skipped[gem_name] = "extensions not found or build failed #{spec.full_name}" + next + end spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + package = RbInstall::DirPackage.new spec ins = RbInstall::UnpackedInstaller.new(package, options) puts "#{INDENT}#{spec.name} #{spec.version}" @@ -1064,11 +1284,21 @@ install?(:ext, :comm, :gem, :'bundled-gems') do skipped.default = "not found in bundled_gems" puts "skipped bundled gems:" gems.each do |gem| - printf " %-32s%s\n", File.basename(gem), skipped[gem] + gem = File.basename(gem) + printf " %-31s %s\n", gem, skipped[gem.chomp(".gem")] end end end +install?('modular-gc') do + if modular_gc_dir = CONFIG['modular_gc_dir'] and !modular_gc_dir.empty? + dlext = CONFIG['DLEXT', true] + modular_gc_dir = File.expand_path(modular_gc_dir, CONFIG['prefix']) + prepare "modular GC library", modular_gc_dir + install Dir.glob("gc/*/librubygc.*.#{dlext}"), modular_gc_dir + end +end + parse_args() include FileUtils @@ -1085,7 +1315,6 @@ installs = $install.map do |inst| end installs.flatten! installs -= $exclude.map {|exc| $install_procs[exc]}.flatten -puts "Installing to #$destdir" unless installs.empty? installs.each do |block| dir = Dir.pwd begin @@ -1094,5 +1323,9 @@ installs.each do |block| Dir.chdir(dir) end end +unless installs.empty? or $destdir.empty? + require_relative 'lib/colorize' + puts "Installed under #{Colorize.new.info($destdir)}" +end # vi:set sw=2: diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests index 477ba6f9dd..4bcb5707a5 100644 --- a/tool/rbs_skip_tests +++ b/tool/rbs_skip_tests @@ -1,10 +1,57 @@ +# Running tests of RBS gem may fail because of various reasons. +# You can skip tests of RBS gem using this file, instead of pushing a new commit to `ruby/rbs` repository. +# +# The most frequently seen reason is the incompatibilities introduced to the unreleased version, including +# +# * Strict argument type check is introduced +# * A required method parameter is added +# * A method/class is removed +# +# Feel free to skip the tests with this file for that case. +# +# Syntax: +# +# $(test-case-name) ` ` $(optional comment) # Skipping single test case +# $(test-class-name) ` ` $(optional comment) # Skipping a test class +# + +## Failed tests because of testing environment + test_collection_install(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__stdlib_source(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__dependency_no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__rbs_dependency_and__gem_dependency(RBS::CliTest) running tests without Bundler test_collection_install_frozen(RBS::CliTest) running tests without Bundler +test_collection_install_gemspec(RBS::CliTest) running tests without Bundler test_collection_update(RBS::CliTest) running tests without Bundler -test_replicate(EncodingTest) the method was removed in 3.3 NetSingletonTest depending on external resources NetInstanceTest depending on external resources TestHTTPRequest depending on external resources TestSingletonNetHTTPResponse depending on external resources TestInstanceNetHTTPResponse depending on external resources + +test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` + +# Failing because ObjectSpace.count_nodes has been removed +test_count_nodes(ObjectSpaceTest) + +## Unknown failures + +# NoMethodError: undefined method 'inspect' for an instance of RBS::UnitTest::Convertibles::ToInt +test_compile(RegexpSingletonTest) +test_linear_time?(RegexpSingletonTest) +test_new(RegexpSingletonTest) + +## Failed tests caused by unreleased version of Ruby +test_source_location(MethodInstanceTest) +test_source_location(ProcInstanceTest) +test_source_location(UnboundMethodInstanceTest) + +# Errno::ENOENT: No such file or directory - bundle +test_collection_install__pathname_set(RBS::CliTest) +test_collection_install__set_pathname__manifest(RBS::CliTest) diff --git a/tool/rbs_skip_tests_windows b/tool/rbs_skip_tests_windows new file mode 100644 index 0000000000..db12c69419 --- /dev/null +++ b/tool/rbs_skip_tests_windows @@ -0,0 +1,111 @@ +ARGFTest Failing on Windows + +RactorSingletonTest Hangs up on Windows +RactorInstanceTest Hangs up on Windows + +# NotImplementedError: fileno() function is unimplemented on this machine +test_fileno(DirInstanceTest) +test_fchdir(DirSingletonTest) +test_for_fd(DirSingletonTest) + +# ArgumentError: user root doesn't exist +test_home(DirSingletonTest) + +# NameError: uninitialized constant Etc::CS_PATH +test_confstr(EtcSingletonTest) + +# NameError: uninitialized constant Etc::SC_ARG_MAX +test_sysconf(EtcSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-udw6rx/chmod +test_chmod(FileInstanceTest) +test_chmod(FileInstanceTest) +test_truncate(FileInstanceTest) + +# Errno::EISDIR: Is a directory @ rb_sysopen - C:/a/ruby/ruby/src/gems/src/rbs/test/stdlib +test_directory?(FileSingletonTest) + +# NotImplementedError: lutime() function is unimplemented on this machine +test_lutime(FileSingletonTest) + +# NotImplementedError: mkfifo() function is unimplemented on this machine +test_mkfifo(FileSingletonTest) + +# Returns `nil` on Windows +test_getgrgid(EtcSingletonTest) +test_getgrnam(EtcSingletonTest) +test_getpwnam(EtcSingletonTest) +test_getpwuid(EtcSingletonTest) + +# Returns `false` +test_setgid?(FileSingletonTest) +test_setuid?(FileSingletonTest) +test_sticky?(FileSingletonTest) + +test_world_readable?(FileSingletonTest) # Returns `420` +test_world_readable?(FileStatInstanceTest) # Returns `420` +test_world_writable?(FileSingletonTest) # Returns `nil` +test_dev_major(FileStatInstanceTest) # Returns `nil` +test_dev_minor(FileStatInstanceTest) # Returns `nil` +test_rdev_major(FileStatInstanceTest) # Returns `nil` +test_rdev_minor(FileStatInstanceTest) # Returns `nil` + +# ArgumentError: wrong number of arguments (given -403772944, expected 0+) +test_curry(MethodInstanceTest) + +# ArgumentError: no output encoding given +test_tolocale(KconvSingletonTest) + +# Errno::EINVAL: Invalid argument - : +test_system(KernelInstanceTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_load(OpenSSLConfigSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - +test_parse(OpenSSLConfigSingletonTest) +test_parse_config(OpenSSLConfigSingletonTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_each(OpenSSLConfigTest) +test_lookup_and_set(OpenSSLConfigTest) +test_sections(OpenSSLConfigTest) + +# OpenSSL::SSL::SSLError: SSL_connect returned=1 errno=0 peeraddr=185.199.108.153:443 state=error: certificate verify failed (unable to get local issuer certificate) +test_URI_open(OpenURISingletonTest) + +# ArgumentError: both textmode and binmode specified +test_binwrite(PathnameInstanceTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-delete-test20250813-10156-mb3e9i +test_delete(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-binwrite-test20250813-10156-sh8145 +test_open(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_truncate(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_unlink(PathnameInstanceTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /etc/resolv.conf +test_parse_resolv_conf(ResolvDNSConfigSingletonTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvInstanceTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvSingletonTest) + +# ArgumentError: unsupported signal 'SIGUSR2' +test_trap(SignalSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-mgr4tx +test_create(TempfileSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-hp9nzu +test_initialize(TempfileSingletonTest) +test_new(TempfileSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-f8z9pn/test.gz +test_open(ZlibGzipReaderSingletonTest) + +# Errno::EACCES: Permission denied @ rb_file_s_rename +# D:/a/ruby/ruby/src/lib/rubygems/util/atomic_file_writer.rb:42:in 'File.rename' +test_write_binary(GemSingletonTest) diff --git a/tool/rbuninstall.rb b/tool/rbuninstall.rb index f0c286012c..60f5241a4f 100755 --- a/tool/rbuninstall.rb +++ b/tool/rbuninstall.rb @@ -21,15 +21,33 @@ BEGIN { end $dirs = [] $files = [] + COLUMNS = $tty && (ENV["COLUMNS"]&.to_i || begin require 'io/console/size'; rescue; else IO.console_size&.at(1); end)&.then do |n| + n-1 if n > 1 + end + if COLUMNS + $column = 0 + def message(str = nil) + $stdout.print "\b \b" * $column + if str + if str.size > COLUMNS + str = "..." + str[(-COLUMNS+3)..-1] + end + $stdout.print str + end + $stdout.flush + $column = str&.size || 0 + end + else + alias message puts + end } list = ($_.chomp!('/') ? $dirs : $files) list << $_ END { status = true $\ = nil - ors = (!$dryrun and $tty) ? "\e[K\r" : "\n" $files.each do |file| - print "rm #{file}#{ors}" + message "rm #{file}" unless $dryrun file = File.join($destdir, file) if $destdir begin @@ -45,9 +63,10 @@ END { $dirs.each do |dir| unlink[dir] = true end + nonempty = {} while dir = $dirs.pop dir = File.dirname(dir) while File.basename(dir) == '.' - print "rmdir #{dir}#{ors}" + message "rmdir #{dir}" unless $dryrun realdir = $destdir ? File.join($destdir, dir) : dir begin @@ -58,16 +77,23 @@ END { raise unless File.symlink?(realdir) File.unlink(realdir) end - rescue Errno::ENOENT, Errno::ENOTEMPTY + rescue Errno::ENOTEMPTY + nonempty[dir] = true + rescue Errno::ENOENT rescue status = false puts $! else + nonempty.delete(dir) parent = File.dirname(dir) $dirs.push(parent) unless parent == dir or unlink[parent] end end end - print ors.chomp + message + unless nonempty.empty? + puts "Non empty director#{nonempty.size == 1 ? 'y' : 'ies'}:" + nonempty.each_key {|dir| print " #{dir}\n"} + end exit(status) } diff --git a/tool/rdoc-srcdir b/tool/rdoc-srcdir new file mode 100755 index 0000000000..ecc49b4b2c --- /dev/null +++ b/tool/rdoc-srcdir @@ -0,0 +1,30 @@ +#!ruby -W0 + +%w[tsort rdoc].each do |lib| + path = Dir.glob("#{File.dirname(__dir__)}/.bundle/gems/#{lib}-*").first + $LOAD_PATH.unshift("#{path}/lib") +end +require 'rdoc/rdoc' + +# Make only the output directory relative to the invoked directory. +invoked = Dir.pwd + +# Load options and parse files from srcdir. +Dir.chdir(File.dirname(__dir__)) + +options = RDoc::Options.load_options +options.title = options.title.sub(/Ruby \K.*version/) { + File.read("include/ruby/version.h") + .scan(/^ *# *define +RUBY_API_VERSION_(MAJOR|MINOR) +(\d+)/) + .sort # "MAJOR" < "MINOR", fortunately + .to_h.values.join(".") +} +options.parse ARGV + ["#{invoked}/rbconfig.rb"] + +options.singleton_class.define_method(:finish) do + super() + @op_dir = File.expand_path(@op_dir, invoked) +end + +# Do not hide errors when generating documents of Ruby itself. +RDoc::RDoc.new.document options diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index df54c7eb26..95a9688cb2 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -10,13 +10,7 @@ require 'optparse' require 'abbrev' require 'pp' require 'shellwords' -begin - require 'readline' -rescue LoadError - module Readline; end -end - -VERSION = '0.0.1' +require 'reline' opts = OptionParser.new target_version = nil @@ -24,10 +18,9 @@ repo_path = nil api_key = nil ssl_verify = true opts.on('-k REDMINE_API_KEY', '--key=REDMINE_API_KEY', 'specify your REDMINE_API_KEY') {|v| api_key = v} -opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 2.1)') {|v| target_version = v} +opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 3.1)') {|v| target_version = v} opts.on('-r RUBY_REPO_PATH', '--repository=RUBY_REPO_PATH', 'specify repository path') {|v| repo_path = v} opts.on('--[no-]ssl-verify', TrueClass, 'use / not use SSL verify') {|v| ssl_verify = v} -opts.version = VERSION opts.parse!(ARGV) http_options = {use_ssl: true} @@ -35,17 +28,17 @@ http_options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify $openuri_options = {} $openuri_options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify -TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (raise 'need to specify TARGET_VERSION') +TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (puts opts.help; raise 'need to specify TARGET_VERSION') RUBY_REPO_PATH = repo_path || ENV['RUBY_REPO_PATH'] BACKPORT_CF_KEY = 'cf_5' STATUS_CLOSE = 5 -REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (raise 'need to specify REDMINE_API_KEY') +REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (puts opts.help; raise 'need to specify REDMINE_API_KEY') REDMINE_BASE = 'https://bugs.ruby-lang.org' @query = { 'f[]' => BACKPORT_CF_KEY, "op[#{BACKPORT_CF_KEY}]" => '~', - "v[#{BACKPORT_CF_KEY}][]" => "#{TARGET_VERSION}: REQUIRED", + "v[#{BACKPORT_CF_KEY}][]" => "\"#{TARGET_VERSION}: REQUIRED\"", 'limit' => 40, 'status_id' => STATUS_CLOSE, 'sort' => 'updated_on' @@ -70,28 +63,28 @@ COLORS = { } class String - def color(fore=nil, back=nil, bold: false, underscore: false) + def color(fore=nil, back=nil, opts={}, bold: false, underscore: false) seq = "" - if bold - seq << "\e[1m" + if bold || opts[:bold] + seq = seq + "\e[1m" end - if underscore - seq << "\e[2m" + if underscore || opts[:underscore] + seq = seq + "\e[2m" end if fore c = COLORS[fore] raise "unknown foreground color #{fore}" unless c - seq << "\e[#{c}m" + seq = seq + "\e[#{c}m" end if back c = COLORS[back] raise "unknown background color #{back}" unless c - seq << "\e[#{c + 10}m" + seq = seq + "\e[#{c + 10}m" end if seq.empty? self else - seq << self << "\e[0m" + seq = seq + self + "\e[0m" end end end @@ -162,84 +155,6 @@ def more(sio) end end -class << Readline - def readline(prompt = '') - console = IO.console - console.binmode - _, lx = console.winsize - if /mswin|mingw/ =~ RUBY_PLATFORM or /^(?:vt\d\d\d|xterm)/i =~ ENV["TERM"] - cls = "\r\e[2K" - else - cls = "\r" << (" " * lx) - end - cls << "\r" << prompt - console.print prompt - console.flush - line = '' - while true - case c = console.getch - when "\r", "\n" - puts - HISTORY << line - return line - when "\C-?", "\b" # DEL/BS - print "\b \b" if line.chop! - when "\C-u" - print cls - line.clear - when "\C-d" - return nil if line.empty? - line << c - when "\C-p" - HISTORY.pos -= 1 - line = HISTORY.current - print cls - print line - when "\C-n" - HISTORY.pos += 1 - line = HISTORY.current - print cls - print line - else - if c >= " " - print c - line << c - end - end - end - end - - HISTORY = [] - def HISTORY.<<(val) - HISTORY.push(val) - @pos = self.size - self - end - def HISTORY.pos - @pos ||= 0 - end - def HISTORY.pos=(val) - @pos = val - if @pos < 0 - @pos = -1 - elsif @pos >= self.size - @pos = self.size - end - end - def HISTORY.current - @pos ||= 0 - if @pos < 0 || @pos >= self.size - '' - else - self[@pos] - end - end -end unless defined?(Readline.readline) - -def find_svn_log(pattern) - `svn log --xml --stop-on-copy --search="#{pattern}" #{RUBY_REPO_PATH}` -end - def find_git_log(pattern) `git #{RUBY_REPO_PATH ? "-C #{RUBY_REPO_PATH.shellescape}" : ""} log --grep="#{pattern}"` end @@ -275,11 +190,13 @@ def backport_command_string next false if c.match(/\A\d{1,6}\z/) # skip SVN revision # check if the Git revision is included in master - has_commit(c, "master") + has_commit(c, "origin/master") + end.sort_by do |changeset| + Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) end @changesets.define_singleton_method(:validated){true} end - " #{merger_path} --ticket=#{@issue} #{@changesets.sort.join(',')}" + "#{merger_path} --ticket=#{@issue} #{@changesets.join(',')}" end def status_char(obj) @@ -294,7 +211,7 @@ end console = IO.console row, = console.winsize @query['limit'] = row - 2 -puts "Backporter #{VERSION}".color(bold: true) + " for #{TARGET_VERSION}" +puts "Redmine Backporter".color(bold: true) + " for Ruby #{TARGET_VERSION}" class CommandSyntaxError < RuntimeError; end commands = { @@ -306,10 +223,11 @@ commands = { @issues = issues = res["issues"] from = res["offset"] + 1 total = res["total_count"] + closed = issues.count { |x, _| x["status"]["name"] == "Closed" } to = from + issues.size - 1 - puts "#{from}-#{to} / #{total}" + puts "#{from}-#{to} / #{total} (closed: #{closed})" issues.each_with_index do |x, i| - id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]]) + id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]], bold: x["status"]["name"] == "Closed") puts "#{'%2d' % i} #{id} #{x["priority"]["name"][0]} #{status_char(x["status"])} #{x["subject"][0,80]}" end }, @@ -376,9 +294,6 @@ eom "rel" => proc{|args| # this feature requires custom redmine which allows add_related_issue API case args - when /\Ar?(\d+)\z/ # SVN - rev = $1 - uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/trunk/revisions/#{rev}/issues.json") when /\A\h{7,40}\z/ # Git rev = args uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/git/revisions/#{rev}/issues.json") @@ -422,7 +337,7 @@ eom }, "done" => proc{|args| - raise CommandSyntaxError unless /\A(\d+)?(?: by (\h+))?(?:\s*-- +(.*))?\z/ =~ args + raise CommandSyntaxError unless /\A(\d+)?(?: *by (\h+))?(?:\s*-- +(.*))?\z/ =~ args notes = $3 notes.strip! if notes rev = $2 @@ -436,33 +351,22 @@ eom next end - if rev - elsif system("svn info #{RUBY_REPO_PATH&.shellescape}", %i(out err) => IO::NULL) # SVN - if (log = find_svn_log("##@issue]")) && (/revision="(?<rev>\d+)/ =~ log) - rev = "r#{rev}" - end - else # Git - if log = find_git_log("##@issue]") - /^commit (?<rev>\h{40})$/ =~ log - end - end - if log && rev - str = log[/merge revision\(s\) ([^:]+)(?=:)/] - if str - str.insert(5, "d") - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev} #{str}." + if rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") + notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." + elsif rev.nil? && (log = find_git_log("##@issue]")) && !(revs = log.scan(/^commit (\h{40})$/).flatten).empty? + commits = revs.map { |rev| "commit:#{rev}" }.join(", ") + if merged_revs = log[/merge revision\(s\) ([^:]+)(?=:)/] + merged_revs.sub!(/\Amerge/, 'merged') + merged_revs.gsub!(/\h{8,40}/, 'commit:\0') + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits} #{merged_revs}." else - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev}." + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits}." end if notes str << "\n" str << notes end notes = str - elsif rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") - # Backport commit's log doesn't have the issue number. - # Instead of that manually it's provided. - notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." else puts "no commit is found whose log include ##@issue" next @@ -571,7 +475,7 @@ list = Abbrev.abbrev(commands.keys) @changesets = nil while true begin - l = Readline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " + l = Reline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " rescue Interrupt break end diff --git a/tool/release.sh b/tool/release.sh index 0988dc2a67..d467d8f24b 100755 --- a/tool/release.sh +++ b/tool/release.sh @@ -1,7 +1,15 @@ #!/bin/bash # Bash version 3.2+ is required for regexp +# Usage: +# tool/release.sh 3.0.0 +# tool/release.sh 3.0.0-rc1 -EXTS='.tar.gz .tar.bz2 .tar.xz .zip' +EXTS='.tar.gz .tar.xz .zip' +if [[ -n $AWS_ACCESS_KEY_ID ]]; then + AWS_CLI_OPTS="" +else + AWS_CLI_OPTS="--profile ruby" +fi ver=$1 if [[ $ver =~ ^([1-9]\.[0-9])\.([0-9]|[1-9][0-9]|0-(preview[1-9]|rc[1-9]))$ ]]; then @@ -15,5 +23,5 @@ short=${BASH_REMATCH[1]} echo $ver echo $short for ext in $EXTS; do - aws --profile ruby s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext + aws $AWS_CLI_OPTS s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext done diff --git a/tool/releng/gen-mail.rb b/tool/releng/gen-mail.rb index 6dc0e4cec1..17fa499d69 100755 --- a/tool/releng/gen-mail.rb +++ b/tool/releng/gen-mail.rb @@ -10,7 +10,7 @@ end # Confirm current directory is www.ruby-lang.org's working directory def confirm_w_r_l_o_wd File.foreach('.git/config') do |line| - return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + return true if line.include?('ruby/www.ruby-lang.org.git') end abort "Run this script in www.ruby-lang.org's working directory" end diff --git a/tool/releng/update-www-meta.rb b/tool/releng/update-www-meta.rb index 8a5651dcd0..0dd5b25631 100755 --- a/tool/releng/update-www-meta.rb +++ b/tool/releng/update-www-meta.rb @@ -1,6 +1,7 @@ #!/usr/bin/env ruby require "open-uri" require "yaml" +require_relative "../ruby-version" class Tarball attr_reader :version, :size, :sha1, :sha256, :sha512 @@ -41,22 +42,7 @@ eom unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" info = YAML.load(URI(uri).read) @@ -74,9 +60,10 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) rubydir = File.expand_path(File.join(__FILE__, '../../../')) puts %`git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` @@ -155,7 +142,7 @@ eom date = Time.now.utc # use utc to use previous day in midnight entry = <<eom - version: #{ver} - tag: v#{ver.tr('-.', '_')} + tag: #{RubyVersion.tag(ver)} date: #{date.strftime("%Y-%m-%d")} post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ stats: diff --git a/tool/ruby-version.rb b/tool/ruby-version.rb new file mode 100755 index 0000000000..3bbec576e1 --- /dev/null +++ b/tool/ruby-version.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby + +module RubyVersion + def self.tag(version) + major_version = Integer(version.split('.', 2)[0]) + if major_version >= 4 + "v#{version}" + else + "v#{version.tr('.-', '_')}" + end + end + + # Return the previous version to be used for release diff links. + # For a ".0" version, it returns the previous ".0" version. + # For a non-".0" version, it returns the previous teeny version. + def self.previous(version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + major = Integer($1) + minor = Integer($2) + teeny = Integer($3) + + if teeny != 0 + "#{major}.#{minor}.#{teeny-1}" + elsif minor != 0 # && teeny == 0 + "#{major}.#{minor-1}.#{teeny}" + else # minor == 0 && teeny == 0 + case major + when 3 + "2.7.0" + when 4 + "3.4.0" + else + raise "it doesn't know what is the previous version of '#{version}'" + end + end + end +end + +if __FILE__ == $0 + case ARGV[0] + when "tag" + print RubyVersion.tag(ARGV[1]) + when "previous" + print RubyVersion.previous(ARGV[1]) + when "previous-tag" + print RubyVersion.tag(RubyVersion.previous(ARGV[1])) + else + "#{$0}: unexpected command #{ARGV[0].inspect}" + end +end diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb index e03e54e397..f6c0e39600 100644 --- a/tool/ruby_vm/controllers/application_controller.rb +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb index 34fafd1e34..628cb0428b 100644 --- a/tool/ruby_vm/helpers/c_escape.rb +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -17,7 +16,10 @@ module RubyVM::CEscape # generate comment, with escaps. def commentify str - return "/* #{str.b.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" + unless str = str.dump[/\A"\K.*(?="\z)/] + raise Encoding::CompatibilityError, "must be ASCII-compatible (#{str.encoding})" + end + return "/* #{str.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" end # Mimic gensym of CL. diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb index c083dffa7a..f0758dd44f 100644 --- a/tool/ruby_vm/helpers/dumper.rb +++ b/tool/ruby_vm/helpers/dumper.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -33,11 +32,7 @@ class RubyVM::Dumper rescue Errno::ENOENT raise "don't know how to generate #{path}" else - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%-') - else - erb = ERB.new(src, nil, '%-') - end + erb = ERB.new(src, trim_mode: '%-') erb.filename = path.to_path return erb end diff --git a/tool/ruby_vm/helpers/scanner.rb b/tool/ruby_vm/helpers/scanner.rb index ef6de8120e..8998abb2d3 100644 --- a/tool/ruby_vm/helpers/scanner.rb +++ b/tool/ruby_vm/helpers/scanner.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb index 034905f74e..d45d0ba83c 100644 --- a/tool/ruby_vm/loaders/insns_def.rb +++ b/tool/ruby_vm/loaders/insns_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb index aa6fd79e79..0750f1823a 100644 --- a/tool/ruby_vm/loaders/opt_insn_unif_def.rb +++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb index 29aef8a325..e08509a433 100644 --- a/tool/ruby_vm/loaders/opt_operand_def.rb +++ b/tool/ruby_vm/loaders/opt_operand_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb index 3f05c270ee..d626ea0296 100644 --- a/tool/ruby_vm/loaders/vm_opts_h.rb +++ b/tool/ruby_vm/loaders/vm_opts_h.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index de35e7234a..177b701b92 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -21,7 +20,7 @@ class RubyVM::Attribute @key = opts[:name] @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] @type = opts[:type] - @ope_decls = @insn.opes.map do |operand| + @ope_decls = @insn.operands.map do |operand| decl = operand[:decl] if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA' decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci') diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instruction.rb index 6b5f1f6cf8..f87dd74179 100755..100644 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -15,8 +14,8 @@ require_relative 'c_expr' require_relative 'typemap' require_relative 'attribute' -class RubyVM::BareInstructions - attr_reader :template, :name, :opes, :pops, :rets, :decls, :expr +class RubyVM::BareInstruction + attr_reader :template, :name, :operands, :pops, :rets, :decls, :expr def initialize opts = {} @template = opts[:template] @@ -24,7 +23,7 @@ class RubyVM::BareInstructions @loc = opts[:location] @sig = opts[:signature] @expr = RubyVM::CExpr.new opts[:expr] - @opes = typesplit @sig[:ope] + @operands = typesplit @sig[:ope] @pops = typesplit @sig[:pop].reject {|i| i == '...' } @rets = typesplit @sig[:ret].reject {|i| i == '...' } @attrs = opts[:attributes].map {|i| @@ -51,7 +50,7 @@ class RubyVM::BareInstructions def call_attribute name return sprintf 'attr_%s_%s(%s)', name, @name, \ - @opes.map {|i| i[:name] }.compact.join(', ') + @operands.map {|i| i[:name] }.compact.join(', ') end def has_attribute? k @@ -65,7 +64,7 @@ class RubyVM::BareInstructions end def width - return 1 + opes.size + return 1 + operands.size end def declarations @@ -98,7 +97,7 @@ class RubyVM::BareInstructions end def operands_info - opes.map {|o| + operands.map {|o| c, _ = RubyVM::Typemap.fetch o[:type] next c }.join @@ -108,14 +107,6 @@ class RubyVM::BareInstructions /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr end - def always_leaf? - @attrs.fetch('leaf').expr.expr == 'true;' - end - - def leaf_without_check_ints? - @attrs.fetch('leaf').expr.expr == 'leafness_of_check_ints;' - end - def handle_canary stmt # Stack canary is basically a good thing that we want to add, however: # @@ -137,7 +128,7 @@ class RubyVM::BareInstructions end def has_ope? var - return @opes.any? {|i| i[:name] == var[:name] } + return @operands.any? {|i| i[:name] == var[:name] } end def has_pop? var @@ -149,6 +140,10 @@ class RubyVM::BareInstructions @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } end + def zjit_profile? + @attrs.fetch('zjit_profile').expr.expr != 'false;' + end + private def check_attribute_consistency @@ -180,18 +175,19 @@ class RubyVM::BareInstructions # Beware: order matters here because some attribute depends another. generate_attribute 'const char*', 'name', "insn_name(#{bin})" generate_attribute 'enum ruby_vminsn_type', 'bin', bin - generate_attribute 'rb_num_t', 'open', opes.size + generate_attribute 'rb_num_t', 'open', operands.size generate_attribute 'rb_num_t', 'popn', pops.size generate_attribute 'rb_num_t', 'retn', rets.size generate_attribute 'rb_num_t', 'width', width generate_attribute 'rb_snum_t', 'sp_inc', rets.size - pops.size generate_attribute 'bool', 'handles_sp', default_definition_of_handles_sp generate_attribute 'bool', 'leaf', default_definition_of_leaf + generate_attribute 'bool', 'zjit_profile', false end def default_definition_of_handles_sp # Insn with ISEQ should yield it; can handle sp. - return opes.any? {|o| o[:type] == 'ISEQ' } + return operands.any? {|o| o[:type] == 'ISEQ' } end def default_definition_of_leaf @@ -228,13 +224,13 @@ class RubyVM::BareInstructions new h.merge(:template => h) } - def self.fetch name + def self.find(name) @instances.find do |insn| insn.name == name end or raise IndexError, "instruction not found: #{name}" end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb index 073112f545..095ff4f1d9 100644 --- a/tool/ruby_vm/models/c_expr.rb +++ b/tool/ruby_vm/models/c_expr.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -36,6 +35,10 @@ class RubyVM::CExpr end def inspect - sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + if @__LINE__ + sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + else + sprintf "#<%s %s>", @__FILE__, @expr + end end end diff --git a/tool/ruby_vm/models/instructions.rb b/tool/ruby_vm/models/instructions.rb index 1198c7a4a6..7be7064b14 100644 --- a/tool/ruby_vm/models/instructions.rb +++ b/tool/ruby_vm/models/instructions.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -10,13 +9,15 @@ # conditions mentioned in the file COPYING are met. Consult the file for # details. -require_relative 'bare_instructions' -require_relative 'operands_unifications' -require_relative 'instructions_unifications' +require_relative 'bare_instruction' +require_relative 'operands_unification' +require_relative 'instructions_unification' +require_relative 'trace_instruction' +require_relative 'zjit_instruction' -RubyVM::Instructions = RubyVM::BareInstructions.to_a + \ - RubyVM::OperandsUnifications.to_a + \ - RubyVM::InstructionsUnifications.to_a - -require_relative 'trace_instructions' +RubyVM::Instructions = RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all + + RubyVM::TraceInstruction.all + + RubyVM::ZJITInstruction.all RubyVM::Instructions.freeze diff --git a/tool/ruby_vm/models/instructions_unifications.rb b/tool/ruby_vm/models/instructions_unification.rb index 214ba5fcc2..5c798e6d54 100644 --- a/tool/ruby_vm/models/instructions_unifications.rb +++ b/tool/ruby_vm/models/instructions_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,9 +11,9 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_insn_unif_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::InstructionsUnifications +class RubyVM::InstructionsUnification include RubyVM::CEscape attr_reader :name @@ -23,7 +22,7 @@ class RubyVM::InstructionsUnifications @location = opts[:location] @name = namegen opts[:signature] @series = opts[:signature].map do |i| - RubyVM::BareInstructions.fetch i # Misshit is fatal + RubyVM::BareInstruction.find(i) # Misshit is fatal end end @@ -37,7 +36,7 @@ class RubyVM::InstructionsUnifications new h end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/operands_unifications.rb b/tool/ruby_vm/models/operands_unification.rb index ee4e3a695d..ce118648ca 100644 --- a/tool/ruby_vm/models/operands_unifications.rb +++ b/tool/ruby_vm/models/operands_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,16 +11,16 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_operand_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::OperandsUnifications < RubyVM::BareInstructions +class RubyVM::OperandsUnification < RubyVM::BareInstruction include RubyVM::CEscape attr_reader :preamble, :original, :spec def initialize opts = {} name = opts[:signature][0] - @original = RubyVM::BareInstructions.fetch name + @original = RubyVM::BareInstruction.find(name) template = @original.template parts = compose opts[:location], opts[:signature], template[:signature] json = template.dup @@ -38,8 +37,8 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions end def operand_shift_of var - before = @original.opes.find_index var - after = @opes.find_index var + before = @original.operands.find_index var + after = @operands.find_index var raise "no #{var} for #{@name}" unless before and after return before - after end @@ -50,7 +49,7 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions case val when '*' then next nil else - type = @original.opes[i][:type] + type = @original.operands[i][:type] expr = RubyVM::Typemap.typecast_to_VALUE type, val next "#{ptr}[#{i}] == #{expr}" end @@ -85,7 +84,7 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions def compose location, spec, template name = namegen spec *, argv = *spec - opes = @original.opes + opes = @original.operands if opes.size != argv.size raise sprintf("operand size mismatch for %s (%s's: %d, given: %d)", name, template[:name], opes.size, argv.size) @@ -130,12 +129,12 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions new h end - def self.to_a + def self.all @instances end def self.each_group - to_a.group_by(&:original).each_pair do |k, v| + all.group_by(&:original).each_pair do |k, v| yield k, v end end diff --git a/tool/ruby_vm/models/trace_instructions.rb b/tool/ruby_vm/models/trace_instruction.rb index 4ed4c8cb42..6a3ad53c44 100644 --- a/tool/ruby_vm/models/trace_instructions.rb +++ b/tool/ruby_vm/models/trace_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -11,9 +10,9 @@ # details. require_relative '../helpers/c_escape' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::TraceInstructions +class RubyVM::TraceInstruction include RubyVM::CEscape attr_reader :name @@ -61,11 +60,11 @@ class RubyVM::TraceInstructions private - @instances = RubyVM::Instructions.map {|i| new i } + @instances = (RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all).map {|i| new(i) } - def self.to_a + def self.all @instances end - - RubyVM::Instructions.push(*to_a) end diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb index d762dd3321..68ef5a41a5 100644 --- a/tool/ruby_vm/models/typemap.rb +++ b/tool/ruby_vm/models/typemap.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/zjit_instruction.rb b/tool/ruby_vm/models/zjit_instruction.rb new file mode 100644 index 0000000000..04764e4c61 --- /dev/null +++ b/tool/ruby_vm/models/zjit_instruction.rb @@ -0,0 +1,56 @@ +require_relative '../helpers/c_escape' +require_relative 'bare_instruction' + +# Profile YARV instructions to optimize code generated by ZJIT +class RubyVM::ZJITInstruction + include RubyVM::CEscape + + attr_reader :name + + def initialize(orig) + @orig = orig + @name = as_tr_cpp "zjit @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute?(*) + return false + end + + @instances = RubyVM::BareInstruction.all.filter(&:zjit_profile?).map {|i| new(i) } + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb index 47d8da5513..ad8603b1a8 100644 --- a/tool/ruby_vm/scripts/insns2vm.rb +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/tests/.gitkeep b/tool/ruby_vm/tests/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 --- a/tool/ruby_vm/tests/.gitkeep +++ /dev/null diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb index b633ab4d32..8bb28db1c1 100644 --- a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -6,6 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. %# +% +% stack_increase = proc do |i| +% if i.has_attribute?('sp_inc') +% '-127' +% else +% sprintf("%4d", i.rets.size - i.pops.size) +% end +% end +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); @@ -13,15 +23,14 @@ rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) { static const signed char t[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map { |i| - if i.has_attribute?('sp_inc') - '-127' - else - sprintf("%4d", i.rets.size - i.pops.size) - end - }.join(', ') -%>, +% insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, % end +#endif }; signed char c = t[insn]; @@ -42,7 +51,7 @@ comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *o % end case <%= i.bin %>: return <%= attr_function %>(<%= - i.opes.map.with_index do |v, j| + i.operands.map.with_index do |v, j| if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc') v = v.dup v[:type] = 'CALL_INFO' diff --git a/tool/ruby_vm/views/_insn_entry.erb b/tool/ruby_vm/views/_insn_entry.erb index f34afddb1f..6ec33461c4 100644 --- a/tool/ruby_vm/views/_insn_entry.erb +++ b/tool/ruby_vm/views/_insn_entry.erb @@ -20,11 +20,11 @@ INSN_ENTRY(<%= insn.name %>) <%= render_c_expr konst -%> % end % -% insn.opes.each_with_index do |ope, i| +% insn.operands.each_with_index do |ope, i| <%= ope[:decl] %> = (<%= ope[:type] %>)GET_OPERAND(<%= i + 1 %>); % end # define INSN_ATTR(x) <%= insn.call_attribute(' ## x ## ') %> - const bool leaf = INSN_ATTR(leaf); + const bool MAYBE_UNUSED(leaf) = INSN_ATTR(leaf); % insn.pops.reverse_each.with_index.reverse_each do |pop, i| <%= pop[:decl] %> = <%= insn.cast_from_VALUE pop, "TOPN(#{i})"%>; % end @@ -35,13 +35,13 @@ INSN_ENTRY(<%= insn.name %>) % end /* ### Instruction preambles. ### */ - if (! leaf) ADD_PC(INSN_ATTR(width)); + ADD_PC(INSN_ATTR(width)); % if insn.handles_sp? POPN(INSN_ATTR(popn)); % end <%= insn.handle_canary "SETUP_CANARY(leaf)" -%> COLLECT_USAGE_INSN(INSN_ATTR(bin)); -% insn.opes.each_with_index do |ope, i| +% insn.operands.each_with_index do |ope, i| COLLECT_USAGE_OPERAND(INSN_ATTR(bin), <%= i %>, <%= ope[:name] %>); % end % unless body.empty? @@ -68,7 +68,6 @@ INSN_ENTRY(<%= insn.name %>) VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_MOVED)); % end % end - if (leaf) ADD_PC(INSN_ATTR(width)); # undef INSN_ATTR /* ### Leave the instruction. ### */ diff --git a/tool/ruby_vm/views/_insn_leaf_info.erb b/tool/ruby_vm/views/_insn_leaf_info.erb new file mode 100644 index 0000000000..f30366ffda --- /dev/null +++ b/tool/ruby_vm/views/_insn_leaf_info.erb @@ -0,0 +1,18 @@ +MAYBE_UNUSED(static bool insn_leaf(int insn, const VALUE *opes)); +static bool +insn_leaf(int insn, const VALUE *opes) +{ + switch (insn) { +% RubyVM::Instructions.each do |insn| +% next if insn.is_a?(RubyVM::TraceInstruction) || insn.is_a?(RubyVM::ZJITInstruction) + case <%= insn.bin %>: + return attr_leaf_<%= insn.name %>(<%= + insn.operands.map.with_index do |ope, i| + "(#{ope[:type]})opes[#{i}]" + end.join(', ') + %>); +% end + default: + return false; + } +} diff --git a/tool/ruby_vm/views/_insn_len_info.erb b/tool/ruby_vm/views/_insn_len_info.erb index 569dca5845..b29a405918 100644 --- a/tool/ruby_vm/views/_insn_len_info.erb +++ b/tool/ruby_vm/views/_insn_len_info.erb @@ -5,6 +5,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% CONSTFUNC(MAYBE_UNUSED(static int insn_len(VALUE insn))); RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ @@ -13,9 +16,14 @@ RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO const uint8_t rb_vm_insn_len_info[] = { -% RubyVM::Instructions.each_slice 23 do |a| - <%= a.map(&:width).join(', ') -%>, +% insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, % end +#if USE_ZJIT +% zjit_insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_len_info); diff --git a/tool/ruby_vm/views/_insn_name_info.erb b/tool/ruby_vm/views/_insn_name_info.erb index e7ded75e65..2862908631 100644 --- a/tool/ruby_vm/views/_insn_name_info.erb +++ b/tool/ruby_vm/views/_insn_name_info.erb @@ -6,10 +6,14 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.name } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% next_offset = 0 +% name_offset = proc do |i| +% offset = sprintf("%4d", next_offset) +% next_offset += i.name.length + 1 # insn.name + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_name(VALUE insn))); @@ -20,18 +24,29 @@ extern const unsigned short rb_vm_insn_name_offset[VM_INSTRUCTION_SIZE]; RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO -const int rb_vm_max_insn_name_size = <%= a.map(&:size).max %>; +%# "trace_" is longer than "zjit_", so USE_ZJIT doesn't impact the max name size. +const int rb_vm_max_insn_name_size = <%= RubyVM::Instructions.map { |i| i.name.size }.max %>; const char rb_vm_insn_name_base[] = -% a.each do |i| - <%=cstr i%> "\0" +% insns.each do |i| + <%= cstr i.name %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each do |i| + <%= cstr i.name %> "\0" % end +#endif ; const unsigned short rb_vm_insn_name_offset[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%4d", i) }.join(', ') %>, +% insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, % end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_name_offset); diff --git a/tool/ruby_vm/views/_insn_operand_info.erb b/tool/ruby_vm/views/_insn_operand_info.erb index 996c33e960..410869fcd3 100644 --- a/tool/ruby_vm/views/_insn_operand_info.erb +++ b/tool/ruby_vm/views/_insn_operand_info.erb @@ -6,10 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.operands_info } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% operands_info = proc { |i| sprintf("%-6s", cstr(i.operands_info)) } +% +% next_offset = 0 +% op_offset = proc do |i| +% offset = sprintf("%3d", next_offset) +% next_offset += i.operands_info.length + 1 # insn.operands_info + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_op_types(VALUE insn))); CONSTFUNC(MAYBE_UNUSED(static int insn_op_type(VALUE insn, long pos))); @@ -21,15 +27,25 @@ RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO const char rb_vm_insn_op_base[] = -% a.each_slice 5 do |d| - <%= d.map {|i| sprintf("%-6s", cstr(i)) }.join(' "\0" ') %> "\0" +% insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" % end +#endif ; const unsigned short rb_vm_insn_op_offset[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%3d", i) }.join(', ') %>, +% insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, % end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_op_offset); diff --git a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb b/tool/ruby_vm/views/_insn_sp_pc_dependency.erb deleted file mode 100644 index 95528fbbf4..0000000000 --- a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb +++ /dev/null @@ -1,27 +0,0 @@ -%# -*- C -*- -%# Copyright (c) 2019 Takashi Kokubun. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -%# -PUREFUNC(MAYBE_UNUSED(static bool insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes))); - -static bool -insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes) -{ - switch (insn) { -% RubyVM::Instructions.each do |insn| -% # handles_sp?: If true, it requires to move sp in JIT -% # always_leaf?: If false, it may call an arbitrary method. pc should be moved -% # before the call, and the method may refer to caller's pc (lineno). -% unless !insn.is_a?(RubyVM::TraceInstructions) && !insn.handles_sp? && insn.always_leaf? - case <%= insn.bin %>: -% end -% end - return true; - default: - return false; - } -} diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb index 1735db2196..2756fa2dec 100644 --- a/tool/ruby_vm/views/_leaf_helpers.erb +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -10,10 +10,6 @@ #include "iseq.h" -// This is used to tell MJIT that this insn would be leaf if CHECK_INTS didn't exist. -// It should be used only when RUBY_VM_CHECK_INTS is directly written in insns.def. -static bool leafness_of_check_ints = false; - static bool leafness_of_defined(rb_num_t op_type) { @@ -25,7 +21,7 @@ leafness_of_defined(rb_num_t op_type) case DEFINED_YIELD: case DEFINED_REF: case DEFINED_ZSUPER: - return false; + return true; case DEFINED_CONST: case DEFINED_CONST_FROM: /* has rb_autoload_load(); */ diff --git a/tool/ruby_vm/views/_sp_inc_helpers.erb b/tool/ruby_vm/views/_sp_inc_helpers.erb index d0b0bd79ef..740fe10142 100644 --- a/tool/ruby_vm/views/_sp_inc_helpers.erb +++ b/tool/ruby_vm/views/_sp_inc_helpers.erb @@ -18,7 +18,7 @@ sp_inc_of_sendish(const struct rb_callinfo *ci) * 3. Pop receiver. * 4. Push return value. */ - const int argb = (vm_ci_flag(ci) & VM_CALL_ARGS_BLOCKARG) ? 1 : 0; + const int argb = (vm_ci_flag(ci) & (VM_CALL_ARGS_BLOCKARG | VM_CALL_FORWARDING)) ? 1 : 0; const int argc = vm_ci_argc(ci); const int recv = 1; const int retn = 1; diff --git a/tool/ruby_vm/views/_zjit_helpers.erb b/tool/ruby_vm/views/_zjit_helpers.erb new file mode 100644 index 0000000000..1185dbd9d8 --- /dev/null +++ b/tool/ruby_vm/views/_zjit_helpers.erb @@ -0,0 +1,31 @@ +#if USE_ZJIT + +MAYBE_UNUSED(static int vm_bare_insn_to_zjit_insn(int insn)); +static int +vm_bare_insn_to_zjit_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case BIN(<%= insn.jump_destination %>): + return <%= insn.bin %>; +% end + default: + return insn; + } +} + +MAYBE_UNUSED(static int vm_zjit_insn_to_bare_insn(int insn)); +static int +vm_zjit_insn_to_bare_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case <%= insn.bin %>: + return BIN(<%= insn.jump_destination %>); +% end + default: + return insn; + } +} + +#endif diff --git a/tool/ruby_vm/views/_zjit_instruction.erb b/tool/ruby_vm/views/_zjit_instruction.erb new file mode 100644 index 0000000000..7fd657697c --- /dev/null +++ b/tool/ruby_vm/views/_zjit_instruction.erb @@ -0,0 +1,12 @@ +#if USE_ZJIT + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + rb_zjit_profile_insn(BIN(<%= insn.jump_destination %>), ec); + DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} + +#endif diff --git a/tool/ruby_vm/views/insns.inc.erb b/tool/ruby_vm/views/insns.inc.erb index 29981a8a2d..6521a89b8a 100644 --- a/tool/ruby_vm/views/insns.inc.erb +++ b/tool/ruby_vm/views/insns.inc.erb @@ -6,21 +6,36 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'contains YARV instruction list', edit: __FILE__, } -%> +#ifndef INSNS_INC +#define INSNS_INC 1 + /* BIN : Basic Instruction Name */ #define BIN(n) YARVINSN_##n enum ruby_vminsn_type { -% RubyVM::Instructions.each do |i| +% insns.each do |i| + <%= i.bin %>, +% end +#if USE_ZJIT +% zjit_insns.each do |i| <%= i.bin %>, % end +#endif VM_INSTRUCTION_SIZE }; +#define VM_BARE_INSTRUCTION_SIZE <%= RubyVM::Instructions.count { |i| i.name !~ /\A(trace|zjit)_/ } %> + #define ASSERT_VM_INSTRUCTION_SIZE(array) \ STATIC_ASSERT(numberof_##array, numberof(array) == VM_INSTRUCTION_SIZE) + +#endif diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb index 2ca5aca7cf..48dd0e8832 100644 --- a/tool/ruby_vm/views/insns_info.inc.erb +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -11,12 +11,16 @@ this_file: 'contains instruction information for yarv instruction sequence.', edit: __FILE__, } %> +#ifndef INSNS_INFO_INC +#define INSNS_INFO_INC 1 <%= render 'insn_type_chars' %> <%= render 'insn_name_info' %> <%= render 'insn_len_info' %> <%= render 'insn_operand_info' %> <%= render 'leaf_helpers' %> <%= render 'sp_inc_helpers' %> +<%= render 'zjit_helpers' %> <%= render 'attributes' %> +<%= render 'insn_leaf_info' %> <%= render 'comptime_insn_stack_increase' %> -<%= render 'insn_sp_pc_dependency' %> +#endif diff --git a/tool/ruby_vm/views/lib/ruby_vm/mjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/mjit/instruction.rb.erb deleted file mode 100644 index 4b20e896a2..0000000000 --- a/tool/ruby_vm/views/lib/ruby_vm/mjit/instruction.rb.erb +++ /dev/null @@ -1,40 +0,0 @@ -module RubyVM::MJIT # :nodoc: all - Instruction = Struct.new( - :name, - :bin, - :len, - :expr, - :declarations, - :preamble, - :opes, - :pops, - :rets, - :always_leaf?, - :leaf_without_check_ints?, - :handles_sp?, - ) - - INSNS = { -% RubyVM::Instructions.each_with_index do |insn, i| -% next if insn.name.start_with?('trace_') - <%= i %> => Instruction.new( - name: :<%= insn.name %>, - bin: <%= i %>, # BIN(<%= insn.name %>) - len: <%= insn.width %>, # insn_len - expr: <<-EXPR, -<%= insn.expr.expr.dump.sub(/\A"/, '').sub(/"\z/, '').gsub(/\\n/, "\n").gsub(/\\t/, ' ' * 8) %> - EXPR - declarations: <%= insn.declarations.inspect %>, - preamble: <%= insn.preamble.map(&:expr).inspect %>, - opes: <%= insn.opes.inspect %>, - pops: <%= insn.pops.inspect %>, - rets: <%= insn.rets.inspect %>, - always_leaf?: <%= insn.always_leaf? %>, - leaf_without_check_ints?: <%= insn.leaf_without_check_ints? %>, - handles_sp?: <%= insn.handles_sp? %>, - ), -% end - } - - private_constant(*constants) -end diff --git a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb new file mode 100644 index 0000000000..793528af5d --- /dev/null +++ b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb @@ -0,0 +1,14 @@ +module RubyVM::RJIT # :nodoc: all + Instruction = Data.define(:name, :bin, :len, :operands) + + INSNS = { +% RubyVM::Instructions.each_with_index do |insn, i| + <%= i %> => Instruction.new( + name: :<%= insn.name %>, + bin: <%= i %>, # BIN(<%= insn.name %>) + len: <%= insn.width %>, # insn_len + operands: <%= (insn.operands unless insn.name.start_with?(/trace_|zjit_/)).inspect %>, + ), +% end + } +end diff --git a/tool/ruby_vm/views/mjit_sp_inc.inc.erb b/tool/ruby_vm/views/mjit_sp_inc.inc.erb deleted file mode 100644 index 7b925420dd..0000000000 --- a/tool/ruby_vm/views/mjit_sp_inc.inc.erb +++ /dev/null @@ -1,17 +0,0 @@ -static rb_snum_t -mjit_call_attribute_sp_inc(const int insn, const VALUE *operands) -{ - switch (insn) { -% (RubyVM::BareInstructions.to_a + RubyVM::OperandsUnifications.to_a).each do |insn| - case BIN(<%= insn.name %>): { -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end - return <%= insn.call_attribute('sp_inc') %>; - } -% end - default: - rb_bug("unexpected insn in mjit_call_attribute_sp_inc"); - } -} diff --git a/tool/ruby_vm/views/opt_sc.inc.erb b/tool/ruby_vm/views/opt_sc.inc.erb deleted file mode 100644 index e58c81989f..0000000000 --- a/tool/ruby_vm/views/opt_sc.inc.erb +++ /dev/null @@ -1,40 +0,0 @@ -/* -*- C -*- */ - -%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -% raise ':FIXME:TBW' if RubyVM::VmOptsH['STACK_CACHING'] -<%= render 'copyright' %> -<%= render 'notice', locals: { - this_file: 'is for threaded code', - edit: __FILE__, -} -%> - -#define SC_STATE_SIZE 6 - -#define SCS_XX 1 -#define SCS_AX 2 -#define SCS_BX 3 -#define SCS_AB 4 -#define SCS_BA 5 - -#define SC_ERROR 0xffffffff - -static const VALUE sc_insn_info[][SC_STATE_SIZE] = { -#define NO_SC { SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR } -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map{|i| 'NO_SC' }.join(', ') %>, -% end -#undef NO_SC -}; - -static const VALUE sc_insn_next[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map{|i| 'SCS_XX' }.join(', ') %>, -% end -}; - -ASSERT_VM_INSTRUCTION_SIZE(sc_insn_next); diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb index 0190f9e07a..9d9cf0a43a 100644 --- a/tool/ruby_vm/views/optinsn.inc.erb +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -23,20 +23,20 @@ insn_operands_unification(INSN *iobj) /* do nothing */; break; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| case <%= orig.bin %>: % unifs.each do |insn| /* <%= insn.pretty_name %> */ if ( <%= insn.condition('op') %> ) { -% insn.opes.each_with_index do |o, x| +% insn.operands.each_with_index do |o, x| % n = insn.operand_shift_of(o) % if n != 0 then op[<%= x %>] = op[<%= x + n %>]; % end % end iobj->insn_id = <%= insn.bin %>; - iobj->operand_size = <%= insn.opes.size %>; + iobj->operand_size = <%= insn.operands.size %>; break; } % end @@ -56,7 +56,7 @@ rb_insn_unified_local_var_level(VALUE insn) switch (insn) { default: return -1; /* do nothing */; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| % unifs.each do|insn| case <%= insn.bin %>: % insn.spec.map{|(var,val)|val}.reject{|i| i == '*' }.each do |val| diff --git a/tool/ruby_vm/views/optunifs.inc.erb b/tool/ruby_vm/views/optunifs.inc.erb index e92a95beff..c096712936 100644 --- a/tool/ruby_vm/views/optunifs.inc.erb +++ b/tool/ruby_vm/views/optunifs.inc.erb @@ -7,7 +7,6 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % raise ':FIXME:TBW' if RubyVM::VmOptsH['INSTRUCTIONS_UNIFICATION'] -% n = RubyVM::Instructions.size <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -16,6 +15,4 @@ /* Let .bss section automatically initialize this variable */ /* cf. Section 6.7.8 of ISO/IEC 9899:1999 */ -static const int *const *const unified_insns_data[<%= n %>]; - -ASSERT_VM_INSTRUCTION_SIZE(unified_insns_data); +static const int *const *const unified_insns_data[VM_INSTRUCTION_SIZE]; diff --git a/tool/ruby_vm/views/vm.inc.erb b/tool/ruby_vm/views/vm.inc.erb index c1a3faf60a..38bf5f05ae 100644 --- a/tool/ruby_vm/views/vm.inc.erb +++ b/tool/ruby_vm/views/vm.inc.erb @@ -13,18 +13,22 @@ } -%> #include "vm_insnhelper.h" -% RubyVM::BareInstructions.to_a.each do |insn| +% RubyVM::BareInstruction.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::OperandsUnifications.to_a.each do |insn| +% RubyVM::OperandsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::InstructionsUnifications.to_a.each do |insn| +% RubyVM::InstructionsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::TraceInstructions.to_a.each do |insn| +% RubyVM::ZJITInstruction.all.each do |insn| +<%= render 'zjit_instruction', locals: { insn: insn } -%> +% end +% +% RubyVM::TraceInstruction.all.each do |insn| <%= render 'trace_instruction', locals: { insn: insn } -%> % end diff --git a/tool/ruby_vm/views/vmtc.inc.erb b/tool/ruby_vm/views/vmtc.inc.erb index 99cbd92614..39dc8bfa6b 100644 --- a/tool/ruby_vm/views/vmtc.inc.erb +++ b/tool/ruby_vm/views/vmtc.inc.erb @@ -6,6 +6,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' -%> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -13,9 +16,14 @@ } -%> static const void *const insns_address_table[] = { -% RubyVM::Instructions.each do |i| +% insns.each do |i| LABEL_PTR(<%= i.name %>), % end +#if USE_ZJIT +% zjit_insns.each do |i| + LABEL_PTR(<%= i.name %>), +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(insns_address_table); diff --git a/tool/run-gcov.rb b/tool/run-gcov.rb index 5df7622aa3..46626e4703 100644 --- a/tool/run-gcov.rb +++ b/tool/run-gcov.rb @@ -47,7 +47,8 @@ Pathname.glob("**/*.gcda").sort.each do |gcda| )? Creating\ .*\n \n - )+\z + )+ + (Lines\ executed:.*\n)?\z )x raise "Unexpected gcov output" end diff --git a/tool/run-lcov.rb b/tool/run-lcov.rb index f27578200a..bdccc29a11 100644 --- a/tool/run-lcov.rb +++ b/tool/run-lcov.rb @@ -20,7 +20,7 @@ def backup_gcda_files(gcda_files) end def run_lcov(*args) - system("lcov", "--rc", "lcov_branch_coverage=1", *args) + system("lcov", "--rc", "geninfo_unexecuted_blocks=1", "--rc", "lcov_branch_coverage=1", *args, exception: true) end $info_files = [] @@ -41,11 +41,19 @@ def run_lcov_remove(info_src, info_out) ext/-test-/* ext/nkf/nkf-utf8/nkf.c ).each {|f| dirs << File.join(File.dirname(__dir__), f) } - run_lcov("--remove", info_src, *dirs, "-o", info_out) + run_lcov("--ignore-errors", "unused", "--remove", info_src, *dirs, "-o", info_out) end def run_genhtml(info, out) - system("genhtml", "--branch-coverage", "--ignore-errors", "source", info, "-o", out) + base_dir = File.dirname(File.dirname(__dir__)) + ignore_errors = %w(source unmapped category).reject do |a| + Open3.capture3("genhtml", "--ignore-errors", a)[1].include?("unknown argument for --ignore-errors") + end + system("genhtml", + "--branch-coverage", + "--prefix", base_dir, + *ignore_errors.flat_map {|a| ["--ignore-errors", a] }, + info, "-o", out, exception: true) end def gen_rb_lcov(file) diff --git a/tool/runruby.rb b/tool/runruby.rb index 1efe38fd13..ec63d1008a 100755 --- a/tool/runruby.rb +++ b/tool/runruby.rb @@ -11,6 +11,8 @@ when ENV['RUNRUBY_USE_GDB'] == 'true' debugger = :gdb when ENV['RUNRUBY_USE_LLDB'] == 'true' debugger = :lldb +when ENV['RUNRUBY_USE_RR'] == 'true' + debugger = :rr when ENV['RUNRUBY_YJIT_STATS'] use_yjit_stat = true end @@ -134,12 +136,15 @@ if File.file?(libruby_so) env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) end end +# Work around a bug in FreeBSD 13.2 which can cause fork(2) to hang +# See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271490 +env['LD_BIND_NOW'] = 'yes' if /freebsd/ =~ RUBY_PLATFORM ENV.update env if debugger case debugger - when :gdb, nil + when :gdb debugger = %W'gdb -x #{srcdir}/.gdbinit' if File.exist?(gdb = 'run.gdb') or File.exist?(gdb = File.join(abs_archdir, 'run.gdb')) @@ -153,6 +158,8 @@ if debugger debugger.push('-s', lldb) end debugger << '--' + when :rr + debugger = ['rr', 'record'] end if idx = precommand.index(:debugger) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 50c35761f3..14d7a3893d 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -3,6 +3,9 @@ # See `tool/sync_default_gems.rb --help` for how to use this. require 'fileutils' +require "rbconfig" +require "find" +require "tempfile" module SyncDefaultGems include FileUtils @@ -10,101 +13,337 @@ module SyncDefaultGems module_function - REPOSITORIES = { - rubygems: 'rubygems/rubygems', - rdoc: 'ruby/rdoc', - reline: 'ruby/reline', - json: 'flori/json', - psych: 'ruby/psych', - fileutils: 'ruby/fileutils', - fiddle: 'ruby/fiddle', - stringio: 'ruby/stringio', - "io-console": 'ruby/io-console', - "io-nonblock": 'ruby/io-nonblock', - "io-wait": 'ruby/io-wait', - csv: 'ruby/csv', - etc: 'ruby/etc', - date: 'ruby/date', - zlib: 'ruby/zlib', - fcntl: 'ruby/fcntl', - strscan: 'ruby/strscan', - ipaddr: 'ruby/ipaddr', - logger: 'ruby/logger', - ostruct: 'ruby/ostruct', - irb: 'ruby/irb', - forwardable: "ruby/forwardable", - mutex_m: "ruby/mutex_m", - racc: "ruby/racc", - singleton: "ruby/singleton", - open3: "ruby/open3", - getoptlong: "ruby/getoptlong", - pstore: "ruby/pstore", - delegate: "ruby/delegate", - benchmark: "ruby/benchmark", - cgi: "ruby/cgi", - readline: "ruby/readline", - "readline-ext": "ruby/readline-ext", - observer: "ruby/observer", - timeout: "ruby/timeout", - yaml: "ruby/yaml", - uri: "ruby/uri", - openssl: "ruby/openssl", - did_you_mean: "ruby/did_you_mean", - weakref: "ruby/weakref", - tempfile: "ruby/tempfile", - tmpdir: "ruby/tmpdir", - English: "ruby/English", - "net-protocol": "ruby/net-protocol", - "net-http": "ruby/net-http", - bigdecimal: "ruby/bigdecimal", - optparse: "ruby/optparse", - set: "ruby/set", - find: "ruby/find", - rinda: "ruby/rinda", - erb: "ruby/erb", - nkf: "ruby/nkf", - tsort: "ruby/tsort", - abbrev: "ruby/abbrev", - shellwords: "ruby/shellwords", - base64: "ruby/base64", - syslog: "ruby/syslog", - "open-uri": "ruby/open-uri", - securerandom: "ruby/securerandom", - resolv: "ruby/resolv", - "resolv-replace": "ruby/resolv-replace", - time: "ruby/time", - pp: "ruby/pp", - prettyprint: "ruby/prettyprint", - drb: "ruby/drb", - pathname: "ruby/pathname", - digest: "ruby/digest", - error_highlight: "ruby/error_highlight", - syntax_suggest: ["ruby/syntax_suggest", "main"], - un: "ruby/un", - win32ole: "ruby/win32ole", - } + # upstream: "owner/repo" + # branch: "branch_name" + # mappings: [ ["path_in_upstream", "path_in_ruby"], ... ] + # NOTE: path_in_ruby is assumed to be "owned" by this gem, and the contents + # will be removed before sync + # exclude: [ "fnmatch_pattern_after_mapping", ... ] + Repository = Data.define(:upstream, :branch, :mappings, :exclude) do + def excluded?(newpath) + p = newpath + until p == "." + return true if exclude.any? {|pat| File.fnmatch?(pat, p, File::FNM_PATHNAME|File::FNM_EXTGLOB)} + p = File.dirname(p) + end + false + end + + def rewrite_for_ruby(path) + newpath = mappings.find do |src, dst| + if path == src || path.start_with?(src + "/") + break path.sub(src, dst) + end + end + return nil unless newpath + return nil if excluded?(newpath) + newpath + end + end CLASSICAL_DEFAULT_BRANCH = "master" - class << REPOSITORIES - def [](gem) - repo, branch = super - return repo, branch || CLASSICAL_DEFAULT_BRANCH - end + def repo((upstream, branch), mappings, exclude: []) + branch ||= CLASSICAL_DEFAULT_BRANCH + exclude += ["ext/**/depend"] + Repository.new(upstream:, branch:, mappings:, exclude:) + end - def each_pair - super do |gem, (repo, branch)| - yield gem, [repo, branch || CLASSICAL_DEFAULT_BRANCH] + def lib((upstream, branch), gemspec_in_subdir: false) + _org, name = upstream.split("/") + gemspec_dst = gemspec_in_subdir ? "lib/#{name}/#{name}.gemspec" : "lib/#{name}.gemspec" + repo([upstream, branch], [ + ["lib/#{name}.rb", "lib/#{name}.rb"], + ["lib/#{name}", "lib/#{name}"], + ["test/test_#{name}.rb", "test/test_#{name}.rb"], + ["test/#{name}", "test/#{name}"], + ["#{name}.gemspec", gemspec_dst], + ]) + end + + # Note: tool/auto_review_pr.rb also depends on these constants. + NO_UPSTREAM = [ + "lib/unicode_normalize", # not to match with "lib/un" + ] + REPOSITORIES = { + Onigmo: repo("k-takata/Onigmo", [ + ["regcomp.c", "regcomp.c"], + ["regenc.c", "regenc.c"], + ["regenc.h", "regenc.h"], + ["regerror.c", "regerror.c"], + ["regexec.c", "regexec.c"], + ["regint.h", "regint.h"], + ["regparse.c", "regparse.c"], + ["regparse.h", "regparse.h"], + ["regsyntax.c", "regsyntax.c"], + ["onigmo.h", "include/ruby/onigmo.h"], + ["enc", "enc"], + ]), + "io-console": repo("ruby/io-console", [ + ["ext/io/console", "ext/io/console"], + ["test/io/console", "test/io/console"], + ["lib/io/console", "ext/io/console/lib/console"], + ["io-console.gemspec", "ext/io/console/io-console.gemspec"], + ]), + "io-nonblock": repo("ruby/io-nonblock", [ + ["ext/io/nonblock", "ext/io/nonblock"], + ["test/io/nonblock", "test/io/nonblock"], + ["io-nonblock.gemspec", "ext/io/nonblock/io-nonblock.gemspec"], + ]), + "io-wait": repo("ruby/io-wait", [ + ["ext/io/wait", "ext/io/wait"], + ["test/io/wait", "test/io/wait"], + ["io-wait.gemspec", "ext/io/wait/io-wait.gemspec"], + ]), + "net-http": repo("ruby/net-http", [ + ["lib/net/http.rb", "lib/net/http.rb"], + ["lib/net/http", "lib/net/http"], + ["test/net/http", "test/net/http"], + ["net-http.gemspec", "lib/net/http/net-http.gemspec"], + ]), + "net-protocol": repo("ruby/net-protocol", [ + ["lib/net/protocol.rb", "lib/net/protocol.rb"], + ["test/net/protocol", "test/net/protocol"], + ["net-protocol.gemspec", "lib/net/net-protocol.gemspec"], + ]), + "open-uri": lib("ruby/open-uri"), + "win32-registry": repo("ruby/win32-registry", [ + ["lib/win32/registry.rb", "ext/win32/lib/win32/registry.rb"], + ["test/win32/test_registry.rb", "test/win32/test_registry.rb"], + ["win32-registry.gemspec", "ext/win32/win32-registry.gemspec"], + ]), + English: lib("ruby/English"), + cgi: repo("ruby/cgi", [ + ["ext/cgi", "ext/cgi"], + ["lib/cgi/escape.rb", "lib/cgi/escape.rb"], + ["test/cgi/test_cgi_escape.rb", "test/cgi/test_cgi_escape.rb"], + ["test/cgi/update_env.rb", "test/cgi/update_env.rb"], + ]), + date: repo("ruby/date", [ + ["doc/date", "doc/date"], + ["ext/date", "ext/date"], + ["lib", "ext/date/lib"], + ["test/date", "test/date"], + ["date.gemspec", "ext/date/date.gemspec"], + ], exclude: [ + "ext/date/lib/date_core.bundle", + ]), + delegate: lib("ruby/delegate"), + did_you_mean: repo("ruby/did_you_mean", [ + ["lib/did_you_mean.rb", "lib/did_you_mean.rb"], + ["lib/did_you_mean", "lib/did_you_mean"], + ["test", "test/did_you_mean"], + ["did_you_mean.gemspec", "lib/did_you_mean/did_you_mean.gemspec"], + ], exclude: [ + "test/did_you_mean/lib", + "test/did_you_mean/tree_spell/test_explore.rb", + ]), + digest: repo("ruby/digest", [ + ["ext/digest/lib/digest/sha2", "ext/digest/sha2/lib/sha2"], + ["ext/digest", "ext/digest"], + ["lib/digest.rb", "ext/digest/lib/digest.rb"], + ["lib/digest/version.rb", "ext/digest/lib/digest/version.rb"], + ["lib/digest/sha2.rb", "ext/digest/sha2/lib/sha2.rb"], + ["test/digest", "test/digest"], + ["digest.gemspec", "ext/digest/digest.gemspec"], + ]), + erb: repo("ruby/erb", [ + ["ext/erb", "ext/erb"], + ["lib/erb", "lib/erb"], + ["lib/erb.rb", "lib/erb.rb"], + ["test/erb", "test/erb"], + ["erb.gemspec", "lib/erb/erb.gemspec"], + ["libexec/erb", "libexec/erb"], + ]), + error_highlight: repo("ruby/error_highlight", [ + ["lib/error_highlight.rb", "lib/error_highlight.rb"], + ["lib/error_highlight", "lib/error_highlight"], + ["test", "test/error_highlight"], + ["error_highlight.gemspec", "lib/error_highlight/error_highlight.gemspec"], + ]), + etc: repo("ruby/etc", [ + ["ext/etc", "ext/etc"], + ["test/etc", "test/etc"], + ["etc.gemspec", "ext/etc/etc.gemspec"], + ]), + fcntl: repo("ruby/fcntl", [ + ["ext/fcntl", "ext/fcntl"], + ["fcntl.gemspec", "ext/fcntl/fcntl.gemspec"], + ]), + fileutils: lib("ruby/fileutils"), + find: lib("ruby/find"), + forwardable: lib("ruby/forwardable", gemspec_in_subdir: true), + ipaddr: lib("ruby/ipaddr"), + json: repo("ruby/json", [ + ["ext/json/ext", "ext/json"], + ["test/json", "test/json"], + ["lib", "ext/json/lib"], + ["json.gemspec", "ext/json/json.gemspec"], + ], exclude: [ + "ext/json/lib/json/ext/.keep", + "ext/json/lib/json/pure.rb", + "ext/json/lib/json/pure", + "ext/json/lib/json/truffle_ruby", + "test/json/lib", + "ext/json/extconf.rb", + ]), + mmtk: repo(["ruby/mmtk", "main"], [ + ["gc/mmtk", "gc/mmtk"], + ]), + open3: lib("ruby/open3", gemspec_in_subdir: true).tap { + it.exclude << "lib/open3/jruby_windows.rb" + }, + openssl: repo("ruby/openssl", [ + ["ext/openssl", "ext/openssl"], + ["lib", "ext/openssl/lib"], + ["test/openssl", "test/openssl"], + ["sample", "sample/openssl"], + ["openssl.gemspec", "ext/openssl/openssl.gemspec"], + ["History.md", "ext/openssl/History.md"], + ], exclude: [ + "test/openssl/envutil.rb", + "ext/openssl/depend", + ]), + optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap { + it.mappings << ["doc/optparse", "doc/optparse"] + }, + pathname: repo("ruby/pathname", [ + ["ext/pathname/pathname.c", "pathname.c"], + ["lib/pathname_builtin.rb", "pathname_builtin.rb"], + ["lib/pathname.rb", "lib/pathname.rb"], + ["test/pathname", "test/pathname"], + ]), + pp: lib("ruby/pp"), + prettyprint: lib("ruby/prettyprint"), + prism: repo(["ruby/prism", "main"], [ + ["ext/prism", "prism"], + ["lib/prism.rb", "lib/prism.rb"], + ["lib/prism", "lib/prism"], + ["test/prism", "test/prism"], + ["src", "prism"], + ["prism.gemspec", "lib/prism/prism.gemspec"], + ["include/prism", "prism"], + ["include/prism.h", "prism/prism.h"], + ["config.yml", "prism/config.yml"], + ["templates", "prism/templates"], + ], exclude: [ + "prism/templates/{javascript,java,rbi,sig}", + "test/prism/snapshots_test.rb", + "test/prism/snapshots", + "prism/extconf.rb", + "prism/srcs.mk*", + ]), + psych: repo("ruby/psych", [ + ["ext/psych", "ext/psych"], + ["lib", "ext/psych/lib"], + ["test/psych", "test/psych"], + ["psych.gemspec", "ext/psych/psych.gemspec"], + ], exclude: [ + "ext/psych/lib/org", + "ext/psych/lib/psych.jar", + "ext/psych/lib/psych_jars.rb", + "ext/psych/lib/psych.{bundle,so}", + "ext/psych/lib/2.*", + "ext/psych/yaml/LICENSE", + "ext/psych/.gitignore", + ]), + resolv: repo("ruby/resolv", [ + ["lib/resolv.rb", "lib/resolv.rb"], + ["test/resolv", "test/resolv"], + ["resolv.gemspec", "lib/resolv.gemspec"], + ["ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32/resolv.rb"], + ["ext/win32/resolv", "ext/win32/resolv"], + ]), + rubygems: repo("ruby/rubygems", [ + ["lib/rubygems.rb", "lib/rubygems.rb"], + ["lib/rubygems", "lib/rubygems"], + ["test/rubygems", "test/rubygems"], + ["bundler/lib/bundler.rb", "lib/bundler.rb"], + ["bundler/lib/bundler", "lib/bundler"], + ["bundler/exe/bundle", "libexec/bundle"], + ["bundler/exe/bundler", "libexec/bundler"], + ["bundler/bundler.gemspec", "lib/bundler/bundler.gemspec"], + ["bundler/spec", "spec/bundler"], + *["bundle", "parallel_rspec", "rspec"].map {|binstub| + ["bundler/bin/#{binstub}", "spec/bin/#{binstub}"] + }, + *%w[dev_gems test_gems rubocop_gems standard_gems].flat_map {|gemfile| + ["rb.lock", "rb"].map do |ext| + ["tool/bundler/#{gemfile}.#{ext}", "tool/bundler/#{gemfile}.#{ext}"] + end + }, + ], exclude: [ + "spec/bundler/bin", + "spec/bundler/support/artifice/vcr_cassettes", + "spec/bundler/support/artifice/used_cassettes.txt", + "lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}", + ]), + securerandom: lib("ruby/securerandom"), + shellwords: lib("ruby/shellwords"), + singleton: lib("ruby/singleton"), + stringio: repo("ruby/stringio", [ + ["ext/stringio", "ext/stringio"], + ["test/stringio", "test/stringio"], + ["stringio.gemspec", "ext/stringio/stringio.gemspec"], + ["doc/stringio", "doc/stringio"], + ], exclude: [ + "ext/stringio/README.md", + ]), + strscan: repo("ruby/strscan", [ + ["ext/strscan", "ext/strscan"], + ["lib", "ext/strscan/lib"], + ["test/strscan", "test/strscan"], + ["strscan.gemspec", "ext/strscan/strscan.gemspec"], + ["doc/strscan", "doc/strscan"], + ], exclude: [ + "ext/strscan/regenc.h", + "ext/strscan/regint.h", + ]), + syntax_suggest: lib(["ruby/syntax_suggest", "main"], gemspec_in_subdir: true), + tempfile: lib("ruby/tempfile"), + time: lib("ruby/time"), + timeout: lib("ruby/timeout"), + tmpdir: lib("ruby/tmpdir"), + un: lib("ruby/un"), + uri: lib("ruby/uri", gemspec_in_subdir: true), + weakref: lib("ruby/weakref"), + yaml: lib("ruby/yaml", gemspec_in_subdir: true), + zlib: repo("ruby/zlib", [ + ["ext/zlib", "ext/zlib"], + ["test/zlib", "test/zlib"], + ["zlib.gemspec", "ext/zlib/zlib.gemspec"], + ]), + }.transform_keys(&:to_s) + + class << Repository + def find_upstream(file) + return if NO_UPSTREAM.any? {|dst| file.start_with?(dst) } + REPOSITORIES.find do |repo_name, repository| + if repository.mappings.any? {|_src, dst| file.start_with?(dst) } + break repo_name + end end end + + def group(files) + files.group_by {|file| find_upstream(file)} + end end + # Allow synchronizing commits up to this FETCH_DEPTH. We've historically merged PRs + # with about 250 commits to ruby/ruby, so we use this depth for ruby/ruby in general. + FETCH_DEPTH = 500 + def pipe_readlines(args, rs: "\0", chomp: true) IO.popen(args) do |f| f.readlines(rs, chomp: chomp) end end + def porcelain_status(*pattern) + pipe_readlines(%W"git status --porcelain --no-renames -z --" + pattern) + end + def replace_rdoc_ref(file) src = File.binread(file) changed = false @@ -123,326 +362,123 @@ module SyncDefaultGems end def replace_rdoc_ref_all - result = pipe_readlines(%W"git status porcelain -z -- *.c *.rb *.rdoc") + result = porcelain_status("*.c", "*.rb", "*.rdoc") result.map! {|line| line[/\A.M (.*)/, 1]} result.compact! + return if result.empty? result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + result) result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} end + def replace_rdoc_ref_all_full + Dir.glob("**/*.{c,rb,rdoc}").inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def rubygems_do_fixup + gemspec_content = File.readlines("lib/bundler/bundler.gemspec").map do |line| + next if line =~ /LICENSE\.md/ + + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") + end.compact.join + File.write("lib/bundler/bundler.gemspec", gemspec_content) + + ["bundle", "parallel_rspec", "rspec"].each do |binstub| + path = "spec/bin/#{binstub}" + next unless File.exist?(path) + content = File.read(path).gsub("../spec", "../bundler") + File.write(path, content) + chmod("+x", path) + end + end + # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. def sync_default_gems(gem) - repo, = REPOSITORIES[gem.to_sym] - puts "Sync #{repo}" - - upstream = File.join("..", "..", repo) - - case gem - when "rubygems" - rm_rf(%w[lib/rubygems lib/rubygems.rb test/rubygems]) - cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") - cp_r("#{upstream}/test/rubygems", "test") - rm_rf(%w[lib/bundler lib/bundler.rb libexec/bundler libexec/bundle spec/bundler tool/bundler/*]) - cp_r(Dir.glob("#{upstream}/bundler/lib/bundler*"), "lib") - cp_r(Dir.glob("#{upstream}/bundler/exe/bundle*"), "libexec") - - gemspec_content = File.readlines("#{upstream}/bundler/bundler.gemspec").map do |line| - next if line =~ /LICENSE\.md/ - - line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec").gsub('"exe"', '"libexec"') - end.compact.join - File.write("lib/bundler/bundler.gemspec", gemspec_content) - - cp_r("#{upstream}/bundler/spec", "spec/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/dev_gems*"), "tool/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/test_gems*"), "tool/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/rubocop_gems*"), "tool/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/standard_gems*"), "tool/bundler") - rm_rf(%w[spec/bundler/support/artifice/vcr_cassettes]) - rm_rf Dir.glob("lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}") - when "rdoc" - rm_rf(%w[lib/rdoc lib/rdoc.rb test/rdoc libexec/rdoc libexec/ri]) - cp_r(Dir.glob("#{upstream}/lib/rdoc*"), "lib") - cp_r("#{upstream}/doc/rdoc", "doc") - cp_r("#{upstream}/test/rdoc", "test") - cp_r("#{upstream}/rdoc.gemspec", "lib/rdoc") - cp_r("#{upstream}/Gemfile", "lib/rdoc") - cp_r("#{upstream}/Rakefile", "lib/rdoc") - cp_r("#{upstream}/exe/rdoc", "libexec") - cp_r("#{upstream}/exe/ri", "libexec") - parser_files = { - 'lib/rdoc/markdown.kpeg' => 'lib/rdoc/markdown.rb', - 'lib/rdoc/markdown/literals.kpeg' => 'lib/rdoc/markdown/literals.rb', - 'lib/rdoc/rd/block_parser.ry' => 'lib/rdoc/rd/block_parser.rb', - 'lib/rdoc/rd/inline_parser.ry' => 'lib/rdoc/rd/inline_parser.rb' - } - Dir.chdir(upstream) do - `bundle install` - parser_files.each_value do |dst| - `bundle exec rake #{dst}` + config = REPOSITORIES[gem] + puts "Sync #{config.upstream}" + + upstream = File.join("..", "..", config.upstream) + + config.mappings.each do |src, dst| + rm_rf(dst) + end + + copied = Set.new + config.mappings.each do |src, dst| + prefix = File.join(upstream, src) + # Maybe mapping needs to be updated? + next unless File.exist?(prefix) + Find.find(prefix) do |path| + next if File.directory?(path) + if copied.add?(path) + newpath = config.rewrite_for_ruby(path.sub(%r{\A#{Regexp.escape(upstream)}/}, "")) + next unless newpath + mkdir_p(File.dirname(newpath)) + cp(path, newpath) end end - parser_files.each_pair do |src, dst| - rm_rf(src) - cp_r("#{upstream}/#{dst}", dst) - end - `git checkout lib/rdoc/.document` - rm_rf(%w[lib/rdoc/Gemfile lib/rdoc/Rakefile]) - when "reline" - rm_rf(%w[lib/reline lib/reline.rb test/reline]) - cp_r(Dir.glob("#{upstream}/lib/reline*"), "lib") - cp_r("#{upstream}/test/reline", "test") - cp_r("#{upstream}/reline.gemspec", "lib/reline") - when "irb" - rm_rf(%w[lib/irb lib/irb.rb test/irb]) - cp_r(Dir.glob("#{upstream}/lib/irb*"), "lib") - cp_r("#{upstream}/test/irb", "test") - cp_r("#{upstream}/irb.gemspec", "lib/irb") - cp_r("#{upstream}/man/irb.1", "man/irb.1") - cp_r("#{upstream}/doc/irb", "doc") - when "json" - rm_rf(%w[ext/json test/json]) - cp_r("#{upstream}/ext/json/ext", "ext/json") - cp_r("#{upstream}/tests", "test/json") - rm_rf("test/json/lib") - cp_r("#{upstream}/lib", "ext/json") - cp_r("#{upstream}/json.gemspec", "ext/json") - cp_r("#{upstream}/VERSION", "ext/json") - rm_rf(%w[ext/json/lib/json/ext ext/json/lib/json/pure.rb ext/json/lib/json/pure]) - `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend ext/json/depend` - when "psych" - rm_rf(%w[ext/psych test/psych]) - cp_r("#{upstream}/ext/psych", "ext") - cp_r("#{upstream}/lib", "ext/psych") - cp_r("#{upstream}/test/psych", "test") - rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) - rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) - rm_rf(["ext/psych/yaml/LICENSE"]) - cp_r("#{upstream}/psych.gemspec", "ext/psych") - `git checkout ext/psych/depend ext/psych/.gitignore` - when "fiddle" - rm_rf(%w[ext/fiddle test/fiddle]) - cp_r("#{upstream}/ext/fiddle", "ext") - cp_r("#{upstream}/lib", "ext/fiddle") - cp_r("#{upstream}/test/fiddle", "test") - cp_r("#{upstream}/fiddle.gemspec", "ext/fiddle") - `git checkout ext/fiddle/depend` - rm_rf(%w[ext/fiddle/lib/fiddle.{bundle,so}]) - when "stringio" - rm_rf(%w[ext/stringio test/stringio]) - cp_r("#{upstream}/ext/stringio", "ext") - cp_r("#{upstream}/test/stringio", "test") - cp_r("#{upstream}/stringio.gemspec", "ext/stringio") - `git checkout ext/stringio/depend ext/stringio/README.md` - when "io-console" - rm_rf(%w[ext/io/console test/io/console]) - cp_r("#{upstream}/ext/io/console", "ext/io") - cp_r("#{upstream}/test/io/console", "test/io") - mkdir_p("ext/io/console/lib") - cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") - rm_rf("ext/io/console/lib/console/ffi") - cp_r("#{upstream}/io-console.gemspec", "ext/io/console") - `git checkout ext/io/console/depend` - when "io-nonblock" - rm_rf(%w[ext/io/nonblock test/io/nonblock]) - cp_r("#{upstream}/ext/io/nonblock", "ext/io") - cp_r("#{upstream}/test/io/nonblock", "test/io") - cp_r("#{upstream}/io-nonblock.gemspec", "ext/io/nonblock") - `git checkout ext/io/nonblock/depend` - when "io-wait" - rm_rf(%w[ext/io/wait test/io/wait]) - cp_r("#{upstream}/ext/io/wait", "ext/io") - cp_r("#{upstream}/test/io/wait", "test/io") - cp_r("#{upstream}/io-wait.gemspec", "ext/io/wait") - `git checkout ext/io/wait/depend` - when "etc" - rm_rf(%w[ext/etc test/etc]) - cp_r("#{upstream}/ext/etc", "ext") - cp_r("#{upstream}/test/etc", "test") - cp_r("#{upstream}/etc.gemspec", "ext/etc") - `git checkout ext/etc/depend` - when "date" - rm_rf(%w[ext/date test/date]) - cp_r("#{upstream}/doc/date", "doc") - cp_r("#{upstream}/ext/date", "ext") - cp_r("#{upstream}/lib", "ext/date") - cp_r("#{upstream}/test/date", "test") - cp_r("#{upstream}/date.gemspec", "ext/date") - `git checkout ext/date/depend` - rm_rf(["ext/date/lib/date_core.bundle"]) - when "zlib" - rm_rf(%w[ext/zlib test/zlib]) - cp_r("#{upstream}/ext/zlib", "ext") - cp_r("#{upstream}/test/zlib", "test") - cp_r("#{upstream}/zlib.gemspec", "ext/zlib") - `git checkout ext/zlib/depend` - when "fcntl" - rm_rf(%w[ext/fcntl]) - cp_r("#{upstream}/ext/fcntl", "ext") - cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") - `git checkout ext/fcntl/depend` - when "strscan" - rm_rf(%w[ext/strscan test/strscan]) - cp_r("#{upstream}/ext/strscan", "ext") - cp_r("#{upstream}/test/strscan", "test") - cp_r("#{upstream}/strscan.gemspec", "ext/strscan") - rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) - `git checkout ext/strscan/depend` - when "racc" - rm_rf(%w[lib/racc lib/racc.rb ext/racc test/racc]) - parser_files = %w[ - lib/racc/parser-text.rb - ] - Dir.chdir(upstream) do - `bundle install` - parser_files.each do |file| - `bundle exec rake #{file}` - end + end + + porcelain_status().each do |line| + /\A(?:.)(?:.) (?<path>.*)\z/ =~ line or raise + if config.excluded?(path) + puts "Restoring excluded file: #{path}" + IO.popen(%W"git checkout --" + [path], "rb", &:read) end - cp_r(Dir.glob("#{upstream}/lib/racc*"), "lib") - mkdir_p("ext/racc/cparse") - cp_r(Dir.glob("#{upstream}/ext/racc/cparse/*"), "ext/racc/cparse") - cp_r("#{upstream}/test", "test/racc") - cp_r("#{upstream}/racc.gemspec", "lib/racc") - rm_rf("test/racc/lib") - rm_rf("lib/racc/cparse-jruby.jar") - `git checkout ext/racc/cparse/README ext/racc/cparse/depend` - when "cgi" - rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) - cp_r("#{upstream}/ext/cgi", "ext") - cp_r("#{upstream}/lib", ".") - rm_rf("lib/cgi/escape.jar") - cp_r("#{upstream}/test/cgi", "test") - cp_r("#{upstream}/cgi.gemspec", "lib/cgi") - `git checkout ext/cgi/escape/depend` - when "openssl" - rm_rf(%w[ext/openssl test/openssl]) - cp_r("#{upstream}/ext/openssl", "ext") - cp_r("#{upstream}/lib", "ext/openssl") - cp_r("#{upstream}/test/openssl", "test") - rm_rf("test/openssl/envutil.rb") - cp_r("#{upstream}/openssl.gemspec", "ext/openssl") - cp_r("#{upstream}/History.md", "ext/openssl") - `git checkout ext/openssl/depend` - when "net-protocol" - rm_rf(%w[lib/net/protocol.rb lib/net/net-protocol.gemspec test/net/protocol]) - cp_r("#{upstream}/lib/net/protocol.rb", "lib/net") - cp_r("#{upstream}/test/net/protocol", "test/net") - cp_r("#{upstream}/net-protocol.gemspec", "lib/net") - when "net-http" - rm_rf(%w[lib/net/http.rb lib/net/http test/net/http]) - cp_r("#{upstream}/lib/net/http.rb", "lib/net") - cp_r("#{upstream}/lib/net/http", "lib/net") - cp_r("#{upstream}/test/net/http", "test/net") - cp_r("#{upstream}/net-http.gemspec", "lib/net/http") - when "readline-ext" - rm_rf(%w[ext/readline test/readline]) - cp_r("#{upstream}/ext/readline", "ext") - cp_r("#{upstream}/test/readline", "test") - cp_r("#{upstream}/readline-ext.gemspec", "ext/readline") - `git checkout ext/readline/depend` - when "did_you_mean" - rm_rf(%w[lib/did_you_mean lib/did_you_mean.rb test/did_you_mean]) - cp_r(Dir.glob("#{upstream}/lib/did_you_mean*"), "lib") - cp_r("#{upstream}/did_you_mean.gemspec", "lib/did_you_mean") - cp_r("#{upstream}/test", "test/did_you_mean") - rm_rf("test/did_you_mean/lib") - rm_rf(%w[test/did_you_mean/tree_spell/test_explore.rb]) - when "erb" - rm_rf(%w[lib/erb* test/erb libexec/erb]) - cp_r("#{upstream}/lib/erb.rb", "lib") - cp_r("#{upstream}/test/erb", "test") - cp_r("#{upstream}/erb.gemspec", "lib") - cp_r("#{upstream}/libexec/erb", "libexec") - when "nkf" - rm_rf(%w[ext/nkf test/nkf]) - cp_r("#{upstream}/ext/nkf", "ext") - cp_r("#{upstream}/lib", "ext/nkf") - cp_r("#{upstream}/test/nkf", "test") - cp_r("#{upstream}/nkf.gemspec", "ext/nkf") - `git checkout ext/nkf/depend` - when "syslog" - rm_rf(%w[ext/syslog test/syslog test/test_syslog.rb]) - cp_r("#{upstream}/ext/syslog", "ext") - cp_r("#{upstream}/lib", "ext/syslog") - cp_r("#{upstream}/test/syslog", "test") - cp_r("#{upstream}/test/test_syslog.rb", "test") - cp_r("#{upstream}/syslog.gemspec", "ext/syslog") - `git checkout ext/syslog/depend` - when "bigdecimal" - rm_rf(%w[ext/bigdecimal test/bigdecimal]) - cp_r("#{upstream}/ext/bigdecimal", "ext") - cp_r("#{upstream}/sample", "ext/bigdecimal") - cp_r("#{upstream}/lib", "ext/bigdecimal") - cp_r("#{upstream}/test/bigdecimal", "test") - cp_r("#{upstream}/bigdecimal.gemspec", "ext/bigdecimal") - `git checkout ext/bigdecimal/depend` - when "pathname" - rm_rf(%w[ext/pathname test/pathname]) - cp_r("#{upstream}/ext/pathname", "ext") - cp_r("#{upstream}/test/pathname", "test") - cp_r("#{upstream}/lib", "ext/pathname") - cp_r("#{upstream}/pathname.gemspec", "ext/pathname") - `git checkout ext/pathname/depend` - when "digest" - rm_rf(%w[ext/digest test/digest]) - cp_r("#{upstream}/ext/digest", "ext") - mkdir_p("ext/digest/lib/digest") - cp_r("#{upstream}/lib/digest.rb", "ext/digest/lib/") - cp_r("#{upstream}/lib/digest/version.rb", "ext/digest/lib/digest/") - mkdir_p("ext/digest/sha2/lib") - cp_r("#{upstream}/lib/digest/sha2.rb", "ext/digest/sha2/lib") - move("ext/digest/lib/digest/sha2", "ext/digest/sha2/lib") - cp_r("#{upstream}/test/digest", "test") - cp_r("#{upstream}/digest.gemspec", "ext/digest") - `git checkout ext/digest/depend ext/digest/*/depend` - when "set" - sync_lib gem, upstream - cp_r("#{upstream}/test", ".") - when "optparse" - sync_lib gem, upstream - rm_rf(%w[doc/optparse]) - mkdir_p("doc/optparse") - cp_r("#{upstream}/doc/optparse", "doc") - when "error_highlight" - rm_rf(%w[lib/error_highlight lib/error_highlight.rb test/error_highlight]) - cp_r(Dir.glob("#{upstream}/lib/error_highlight*"), "lib") - cp_r("#{upstream}/error_highlight.gemspec", "lib/error_highlight") - cp_r("#{upstream}/test", "test/error_highlight") - when "win32ole" - sync_lib gem, upstream - rm_rf(%w[ext/win32ole/lib]) - Dir.mkdir(*%w[ext/win32ole/lib]) - move("lib/win32ole/win32ole.gemspec", "ext/win32ole") - move(Dir.glob("lib/win32ole*"), "ext/win32ole/lib") - when "open3" - sync_lib gem, upstream - rm_rf("lib/open3/jruby_windows.rb") - when "syntax_suggest" - sync_lib gem, upstream - rm_rf(%w[spec/syntax_suggest libexec/syntax_suggest]) - cp_r("#{upstream}/spec", "spec/syntax_suggest") - cp_r("#{upstream}/exe/syntax_suggest", "libexec/syntax_suggest") - else - sync_lib gem, upstream end + + # RubyGems/Bundler needs special care + if gem == "rubygems" + rubygems_do_fixup + end + + check_prerelease_version(gem) + + # Architecture-dependent files must not pollute libdir. + rm_rf(Dir["lib/**/*.#{RbConfig::CONFIG['DLEXT']}"]) replace_rdoc_ref_all end - IGNORE_FILE_PATTERN = - /\A(?:[A-Z]\w*\.(?:md|txt) - |[^\/]+\.yml - |\.git.* - |[A-Z]\w+file - |COPYING - |rakelib\/.* - )\z/mx - - def message_filter(repo, sha, input: ARGF) - log = input.read - log.delete!("\r") + def check_prerelease_version(gem) + return if ["rubygems", "mmtk", "cgi", "pathname", "Onigmo"].include?(gem) + + require "net/https" + require "json" + require "uri" + + uri = URI("https://rubygems.org/api/v1/versions/#{gem.downcase}/latest.json") + response = Net::HTTP.get(uri) + latest_version = JSON.parse(response)["version"] + + gemspec = [ + "lib/#{gem}/#{gem}.gemspec", + "lib/#{gem}.gemspec", + "ext/#{gem}/#{gem}.gemspec", + "ext/#{gem.split("-").join("/")}/#{gem}.gemspec", + "lib/#{gem.split("-").first}/#{gem}.gemspec", + "ext/#{gem.split("-").first}/#{gem}.gemspec", + "lib/#{gem.split("-").join("/")}/#{gem}.gemspec", + ].find{|gemspec| File.exist?(gemspec)} + spec = Gem::Specification.load(gemspec) + puts "#{gem}-#{spec.version} is not latest version of rubygems.org" if spec.version.to_s != latest_version + end + + def message_filter(repo, sha, log, context: nil) + unless repo.count("/") == 1 and /\A\S+\z/ =~ repo + raise ArgumentError, "invalid repository: #{repo}" + end + unless /\A\h{10,40}\z/ =~ sha + raise ArgumentError, "invalid commit-hash: #{sha}" + end repo_url = "https://github.com/#{repo}" - subject, log = log.split(/\n(?:[ \t]*(?:\n|\z))/, 2) + + # Log messages generated by GitHub web UI have inconsistent line endings + log = log.delete("\r") + log << "\n" if !log.end_with?("\n") + + # Split the subject from the log message according to git conventions. + # SPECIAL TREAT: when the first line ends with a dot `.` (which is not + # obeying the conventions too), takes only that line. + subject, log = log.split(/\A.+\.\K\n(?=\S)|\n(?:[ \t]*(?:\n|\z))/, 2) conv = proc do |s| mod = true if s.gsub!(/\b(?:(?i:fix(?:e[sd])?|close[sd]?|resolve[sd]?) +)\K#(?=\d+\b)|\bGH-#?(?=\d+\b)|\(\K#(?=\d+\))/) { "#{repo_url}/pull/" @@ -460,151 +496,303 @@ module SyncDefaultGems end end commit_url = "#{repo_url}/commit/#{sha[0,10]}\n" + sync_note = context ? "#{commit_url}\n#{context}" : commit_url if log and !log.empty? log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last conv[log] log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) { - ($~.begin(1) ? "" : "\n\n") + commit_url + ($~.begin(2) ? "\n" : "") + ($~.begin(1) ? "" : "\n\n") + sync_note + ($~.begin(2) ? "\n" : "") } else - log = commit_url + log = sync_note end - puts subject, "\n", log + "#{subject}\n\n#{log}" end - # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh - # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. - # @param ranges [Array<String>] "before..after". Note that it will NOT sync "before" (but commits after that). - # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. - def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo, default_branch = REPOSITORIES[gem.to_sym] - puts "Sync #{repo} with commit history." + def log_format(format, args, &block) + IO.popen(%W[git -c core.autocrlf=false -c core.eol=lf + log --no-show-signature --format=#{format}] + args, "rb", &block) + end - IO.popen(%W"git remote") do |f| - unless f.read.split.include?(gem) - `git remote add #{gem} git@github.com:#{repo}.git` + def commits_in_range(upto, exclude, toplevel:) + args = [upto, *exclude.map {|s|"^#{s}"}] + log_format('%H,%P,%s', %W"--first-parent" + args) do |f| + f.read.split("\n").reverse.flat_map {|commit| + hash, parents, subject = commit.split(',', 3) + parents = parents.split + + # Non-merge commit + if parents.size <= 1 + puts "#{hash} #{subject}" + next [[hash, subject]] + end + + # Clean 2-parent merge commit: follow the other parent as long as it + # contains no potentially-non-clean merges + if parents.size == 2 && + IO.popen(%W"git diff-tree --remerge-diff #{hash}", "rb", &:read).empty? + puts "\e[2mChecking the other parent of #{hash} #{subject}\e[0m" + ret = catch(:quit) { + commits_in_range(parents[1], exclude + [parents[0]], toplevel: false) + } + next ret if ret + end + + unless toplevel + puts "\e[1mMerge commit with possible conflict resolution #{hash} #{subject}\e[0m" + throw :quit + end + + puts "#{hash} #{subject} " \ + "\e[1m[merge commit with possible conflicts, will do a squash merge]\e[0m" + [[hash, subject]] + } + end + end + + # Returns commit list as array of [commit_hash, subject, sync_note]. + def commits_in_ranges(ranges) + ranges.flat_map do |range| + exclude, upto = range.include?("..") ? range.split("..", 2) : ["#{range}~1", range] + puts "Looking for commits in range #{exclude}..#{upto}" + commits_in_range(upto, exclude.empty? ? [] : [exclude], toplevel: true) + end.uniq + end + + #-- + # Following methods used by sync_default_gems_with_commits return + # true: success + # false: skipped + # nil: failed + #++ + + def resolve_conflicts(gem, sha, edit) + # Discover unmerged files: any unstaged changes + changes = porcelain_status() + conflict = changes.grep(/\A(?:.[^ ?]) /) {$'} + # If -e option is given, open each conflicted file with an editor + unless conflict.empty? + if edit + case + when (editor = ENV["GIT_EDITOR"] and !editor.empty?) + when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) + end + if editor + system([editor, conflict].join(' ')) + conflict.delete_if {|f| !File.exist?(f)} + return true if conflict.empty? + return system(*%w"git add --", *conflict) + end end + return false end - system(*%W"git fetch --no-tags #{gem}") - if ranges == true - pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" - log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", &:read) - ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] + return true + end + + def collect_cacheinfo(tree) + pipe_readlines(%W"git ls-tree -r -t -z #{tree}").filter_map do |line| + fields, path = line.split("\t", 2) + mode, type, object = fields.split(" ", 3) + next unless type == "blob" + [mode, type, object, path] end + end - commits = ranges.flat_map do |range| - unless range.include?("..") - range = "#{range}~1..#{range}" - end + def rewrite_cacheinfo(gem, blobs) + config = REPOSITORIES[gem] + rewritten = [] + ignored = blobs.dup + ignored.delete_if do |mode, type, object, path| + newpath = config.rewrite_for_ruby(path) + next unless newpath + rewritten << [mode, type, object, newpath] + end + [rewritten, ignored] + end + + def make_commit_info(gem, sha) + config = REPOSITORIES[gem] + headers, orig = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + /^author (?<author_name>.+?) <(?<author_email>.*?)> (?<author_date>.+?)$/ =~ headers or + raise "unable to parse author info for commit #{sha}" + author = { + "GIT_AUTHOR_NAME" => author_name, + "GIT_AUTHOR_EMAIL" => author_email, + "GIT_AUTHOR_DATE" => author_date, + } + context = nil + if /^parent (?<first_parent>.{40})\nparent .{40}$/ =~ headers + # Squashing a merge commit: keep authorship information + context = IO.popen(%W"git shortlog #{first_parent}..#{sha} --", "rb", &:read) + end + message = message_filter(config.upstream, sha, orig, context: context) + [author, message] + end - IO.popen(%W"git log --format=%H,%s #{range} --") do |f| - f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} + def fixup_commit(gem, commit) + wt = File.join("tmp", "sync_default_gems-fixup-worktree") + if File.directory?(wt) + IO.popen(%W"git -C #{wt} clean -xdf", "rb", &:read) + IO.popen(%W"git -C #{wt} reset --hard #{commit}", "rb", &:read) + else + IO.popen(%W"git worktree remove --force #{wt}", "rb", err: File::NULL, &:read) + IO.popen(%W"git worktree add --detach #{wt} #{commit}", "rb", &:read) + end + raise "git worktree prepare failed for commit #{commit}" unless $?.success? + + Dir.chdir(wt) do + if gem == "rubygems" + rubygems_do_fixup end + replace_rdoc_ref_all_full end - # Ignore Merge commit and insufficiency commit for ruby core repository. - commits.delete_if do |sha, subject| - files = pipe_readlines(%W"git diff-tree -z --no-commit-id --name-only -r #{sha}") - subject.start_with?("Merge", "Auto Merge") or files.all?(IGNORE_FILE_PATTERN) + IO.popen(%W"git -C #{wt} add -u", "rb", &:read) + IO.popen(%W"git -C #{wt} commit --amend --no-edit", "rb", &:read) + IO.popen(%W"git -C #{wt} rev-parse HEAD", "rb", &:read).chomp + end + + def make_and_fixup_commit(gem, original_commit, cacheinfo, parent: nil, message: nil, author: nil) + tree = Tempfile.create("sync_default_gems-#{gem}-index") do |f| + File.unlink(f.path) + IO.popen({"GIT_INDEX_FILE" => f.path}, + %W"git update-index --index-info", "wb", out: IO::NULL) do |io| + cacheinfo.each do |mode, type, object, path| + io.puts("#{mode} #{type} #{object}\t#{path}") + end + end + raise "git update-index failed" unless $?.success? + + IO.popen({"GIT_INDEX_FILE" => f.path}, %W"git write-tree --missing-ok", "rb", &:read).chomp end - if commits.empty? - puts "No commits to pick" - return true + args = ["-m", message || "Rewriten commit for #{original_commit}"] + args += ["-p", parent] if parent + commit = IO.popen({**author}, %W"git commit-tree #{tree}" + args, "rb", &:read).chomp + + # Apply changes that require a working tree + commit = fixup_commit(gem, commit) + + commit + end + + def rewrite_commit(gem, sha) + author, message = make_commit_info(gem, sha) + new_blobs = collect_cacheinfo("#{sha}") + new_rewritten, new_ignored = rewrite_cacheinfo(gem, new_blobs) + + headers, _ = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + first_parent = headers[/^parent (.{40})$/, 1] + unless first_parent + # Root commit, first time to sync this repo + return make_and_fixup_commit(gem, sha, new_rewritten, message: message, author: author) end - puts "Try to pick these commits:" - puts commits.map{|commit| commit.join(": ")} - puts "----" + old_blobs = collect_cacheinfo(first_parent) + old_rewritten, old_ignored = rewrite_cacheinfo(gem, old_blobs) + if old_ignored != new_ignored + paths = (old_ignored + new_ignored - (old_ignored & new_ignored)) + .map {|*_, path| path}.uniq + puts "\e\[1mIgnoring file changes not in mappings: #{paths.join(" ")}\e\[0m" + end + changed_paths = (old_rewritten + new_rewritten - (old_rewritten & new_rewritten)) + .map {|*_, path| path}.uniq + if changed_paths.empty? + puts "Skip commit only for tools or toplevel" + return false + end - failed_commits = [] + # Build commit objects from "cacheinfo" + new_parent = make_and_fixup_commit(gem, first_parent, old_rewritten) + new_commit = make_and_fixup_commit(gem, sha, new_rewritten, parent: new_parent, message: message, author: author) + puts "Created a temporary commit for cherry-pick: #{new_commit}" + new_commit + end - ENV["FILTER_BRANCH_SQUELCH_WARNING"] = "1" + def pickup_commit(gem, sha, edit) + rewritten = rewrite_commit(gem, sha) - require 'shellwords' - filter = [ - ENV.fetch('RUBY', 'ruby').shellescape, - File.realpath(__FILE__).shellescape, - "--message-filter", - ] - commits.each do |sha, subject| - puts "Pick #{sha} from #{repo}." + # No changes remaining after rewriting + return false unless rewritten - skipped = false - result = IO.popen(%W"git cherry-pick #{sha}", &:read) - if result =~ /nothing\ to\ commit/ - `git reset` - skipped = true + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{rewritten}", "rb", err: [:child, :out], &:read) + unless $?.success? + if result =~ /The previous cherry-pick is now empty/ + system(*%w"git cherry-pick --skip") puts "Skip empty commit #{sha}" - end - next if skipped - - case gem - when "rubygems" - %w[bundler/spec/support/artifice/vcr_cassettes].each do |rem| - if File.exist?(rem) - system("git", "reset", rem) - rm_rf(rem) - end - end - system(*%w[git add spec/bundler]) + return false end - if result.empty? - skipped = true - elsif /^CONFLICT/ =~ result - result = pipe_readlines(%W"git status --porcelain -z") - result.map! {|line| line[/\A(?:.U|[UA]A) (.*)/, 1]} - result.compact! - ignore, conflict = result.partition {|name| IGNORE_FILE_PATTERN =~ name} - unless ignore.empty? - system(*%W"git reset HEAD --", *ignore) - File.unlink(*ignore) - ignore = pipe_readlines(%W"git status --porcelain -z" + ignore).map! {|line| line[/\A.. (.*)/, 1]} - system(*%W"git checkout HEAD --", *ignore) unless ignore.empty? - end - unless conflict.empty? - if edit - case - when (editor = ENV["GIT_EDITOR"] and !editor.empty?) - when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) - end - if editor - system([editor, conflict].join(' ')) - end - end - end - skipped = !system({"GIT_EDITOR"=>"true"}, *%W"git cherry-pick --no-edit --continue") + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless resolve_conflicts(gem, sha, edit) + system(*%w"git --no-pager diff") if !edit # If failed, show `git diff` unless editing + `git reset` && `git checkout .` && `git clean -fd` # Clean up un-committed diffs + return nil # Fail unless cherry-picked end - if skipped - failed_commits << sha - `git reset` && `git checkout .` && `git clean -fd` - puts "Failed to pick #{sha}" - next + # Commit cherry-picked commit + if porcelain_status().empty? + system(*%w"git cherry-pick --skip") + return false + else + system(*%w"git cherry-pick --continue --no-edit") + return nil unless $?.success? end + end + + new_head = IO.popen(%W"git rev-parse HEAD", "rb", &:read).chomp + puts "Committed cherry-pick as #{new_head}" + return true + end + + # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. + # @param ranges [Array<String>, true] "commit", "before..after", or true. Note that it will NOT sync "before" (but commits after that). + # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. + def sync_default_gems_with_commits(gem, ranges, edit: nil) + config = REPOSITORIES[gem] + repo, default_branch = config.upstream, config.branch + puts "Sync #{repo} with commit history." - head = `git log --format=%H -1 HEAD`.chomp - system(*%w"git reset --quiet HEAD~ --") - amend = replace_rdoc_ref_all - system(*%W"git reset --quiet #{head} --") - if amend - `git commit --amend --no-edit --all` + # Fetch the repository to be synchronized + IO.popen(%W"git remote") do |f| + unless f.read.split.include?(gem) + `git remote add #{gem} https://github.com/#{repo}.git` end + end + system(*%W"git fetch --no-tags --depth=#{FETCH_DEPTH} #{gem} #{default_branch}") - puts "Update commit message: #{sha}" + # If -a is given, discover all commits since the last picked commit + if ranges == true + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" + log = log_format('%B', %W"-E --grep=#{pattern} -n1 --", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] + end + commits = commits_in_ranges(ranges) + if commits.empty? + puts "No commits to pick" + return true + end - IO.popen(%W[git filter-branch -f --msg-filter #{[filter, repo, sha].join(' ')} -- HEAD~1..HEAD], &:read) - unless $?.success? - puts "Failed to modify commit message of #{sha}" - break + failed_commits = [] + commits.each do |sha, subject| + puts "----" + puts "Pick #{sha} #{subject}" + case pickup_commit(gem, sha, edit) + when false + # skipped + when nil + failed_commits << [sha, subject] end end unless failed_commits.empty? puts "---- failed commits ----" - puts failed_commits + failed_commits.each do |sha, subject| + puts "#{sha} #{subject}" + end return false end return true @@ -631,9 +819,9 @@ module SyncDefaultGems end def update_default_gems(gem, release: false) - - repository, default_branch = REPOSITORIES[gem.to_sym] - author, repository = repository.split('/') + config = REPOSITORIES[gem] + author, repository = config.upstream.split('/') + default_branch = config.branch puts "Update #{author}/#{repository}" @@ -670,31 +858,21 @@ module SyncDefaultGems if ARGV[1] update_default_gems(ARGV[1]) else - REPOSITORIES.each_key {|gem| update_default_gems(gem.to_s)} + REPOSITORIES.each_key {|gem| update_default_gems(gem)} end when "all" - if ARGV[1] == "release" - REPOSITORIES.each_key do |gem| - update_default_gems(gem.to_s, release: true) - sync_default_gems(gem.to_s) - end - else - REPOSITORIES.each_key {|gem| sync_default_gems(gem.to_s)} + REPOSITORIES.each_key do |gem| + next if ["Onigmo"].include?(gem) + update_default_gems(gem, release: true) if ARGV[1] == "release" + sync_default_gems(gem) end when "list" ARGV.shift pattern = Regexp.new(ARGV.join('|')) - REPOSITORIES.each_pair do |name, (gem)| - next unless pattern =~ name or pattern =~ gem - printf "%-15s https://github.com/%s\n", name, gem + REPOSITORIES.each do |gem, config| + next unless pattern =~ gem or pattern =~ config.upstream + printf "%-15s https://github.com/%s\n", gem, config.upstream end - when "--message-filter" - ARGV.shift - if ARGV.size < 2 - abort "usage: #{$0} --message-filter repository commit-hash [input...]" - end - message_filter(*ARGV.shift(2)) - exit when "rdoc-ref" ARGV.shift pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV @@ -710,7 +888,13 @@ module SyncDefaultGems puts <<-HELP \e[1mSync with upstream code of default libraries\e[0m -\e[1mImport a default library through `git clone` and `cp -rf` (git commits are lost)\e[0m +\e[1mImport all default gems through `git clone` and `cp -rf` (git commits are lost)\e[0m + ruby #$0 all + +\e[1mImport all released version of default gems\e[0m + ruby #$0 all release + +\e[1mImport a default gem with specific gem same as all command\e[0m ruby #$0 rubygems \e[1mPick a single commit from the upstream repository\e[0m @@ -719,6 +903,12 @@ module SyncDefaultGems \e[1mPick a commit range from the upstream repository\e[0m ruby #$0 rubygems 97e9768612..9e53702832 +\e[1mPick all commits since the last picked commit\e[0m + ruby #$0 -a rubygems + +\e[1mUpdate repositories of default gems\e[0m + ruby #$0 up + \e[1mList known libraries\e[0m ruby #$0 list diff --git a/tool/test-annocheck.sh b/tool/test-annocheck.sh index 0224152d00..6896869e07 100755 --- a/tool/test-annocheck.sh +++ b/tool/test-annocheck.sh @@ -14,7 +14,8 @@ set -x DOCKER="$(command -v docker || command -v podman)" TAG=ruby-fedora-annocheck -TOOL_DIR=$(dirname "${0}") +TOOL_DIR="$(dirname "${0}")" +TMP_DIR="tmp/annocheck" DOCKER_RUN_VOLUME_OPTS= if [ -z "${CI-}" ]; then @@ -27,7 +28,13 @@ else # volume in container in container on GitHub Actions # <.github/workflows/compilers.yml>. TAG="${TAG}-copy" - "${DOCKER}" build --rm -t "${TAG}" --build-arg=FILES="${*}" -f ${TOOL_DIR}/annocheck/Dockerfile-copy . + rm -rf "${TMP_DIR}" + mkdir -p "${TMP_DIR}" + for file in "${@}"; do + cp -p "${file}" "${TMP_DIR}" + done + "${DOCKER}" build --rm -t "${TAG}" --build-arg=IN_DIR="${TMP_DIR}" -f ${TOOL_DIR}/annocheck/Dockerfile-copy . + rm -rf "${TMP_DIR}" fi "${DOCKER}" run --rm -t ${DOCKER_RUN_VOLUME_OPTS} "${TAG}" annocheck --verbose ${TEST_ANNOCHECK_OPTS-} "${@}" diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb index 26ad265608..778fe3311a 100644 --- a/tool/test-bundled-gems.rb +++ b/tool/test-bundled-gems.rb @@ -1,69 +1,112 @@ require 'rbconfig' require 'timeout' require 'fileutils' +require 'shellwords' require_relative 'lib/colorize' +require_relative 'lib/gem_env' ENV.delete("GNUMAKEFLAGS") github_actions = ENV["GITHUB_ACTIONS"] == "true" +DEFAULT_ALLOWED_FAILURES = RUBY_PLATFORM =~ /mswin|mingw/ ? [ + 'debug', + 'irb', + 'csv', +] : [] allowed_failures = ENV['TEST_BUNDLED_GEMS_ALLOW_FAILURES'] || '' -allowed_failures = allowed_failures.split(',').reject(&:empty?) +allowed_failures = allowed_failures.split(',').concat(DEFAULT_ALLOWED_FAILURES).uniq.reject(&:empty?) -ENV["GEM_PATH"] = [File.realpath('.bundle'), File.realpath('../.bundle', __dir__)].join(File::PATH_SEPARATOR) +# make test-bundled-gems BUNDLED_GEMS=gem1,gem2,gem3 +bundled_gems = nil if (bundled_gems = ARGV.first&.split(","))&.empty? colorize = Colorize.new rake = File.realpath("../../.bundle/bin/rake", __FILE__) gem_dir = File.realpath('../../gems', __FILE__) -dummy_rake_compiler_dir = File.realpath('../dummy-rake-compiler', __FILE__) -rubylib = [File.expand_path(dummy_rake_compiler_dir), ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +run_opts = ENV["RUN_OPTS"]&.shellsplit exit_code = 0 ruby = ENV['RUBY'] || RbConfig.ruby failed = [] File.foreach("#{gem_dir}/bundled_gems") do |line| - next if /^\s*(?:#|$)/ =~ line - gem = line.split.first - next if ARGV.any? {|pat| !File.fnmatch?(pat, gem)} - puts "#{github_actions ? "##[group]" : "\n"}Testing the #{gem} gem" + next unless gem = line[/^[^\s\#]+/] + next if bundled_gems&.none? {|pat| File.fnmatch?(pat, gem)} + next unless File.directory?("#{gem_dir}/src/#{gem}/test") - test_command = "#{ruby} -C #{gem_dir}/src/#{gem} #{rake} test" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", rake, "test"] first_timeout = 600 # 10min toplib = gem - case gem - when "typeprof" + unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + toplib = gem.tr("-", "/") + next unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + end + case gem when "rbs" - test_command << " stdlib_test validate RBS_SKIP_TESTS=#{__dir__}/rbs_skip_tests" + # TODO: We should skip test file instead of test class/methods + skip_test_files = %w[ + ] + + skip_test_files.each do |file| + path = "#{gem_dir}/src/#{gem}/#{file}" + File.unlink(path) if File.exist?(path) + end + + rbs_skip_tests = [ + File.join(__dir__, "/rbs_skip_tests") + ] + + if /mswin|mingw/ =~ RUBY_PLATFORM + rbs_skip_tests << File.join(__dir__, "/rbs_skip_tests_windows") + end + + test_command.concat %W[stdlib_test validate RBS_SKIP_TESTS=#{rbs_skip_tests.join(File::PATH_SEPARATOR)} SKIP_RBS_VALIDATION=true] first_timeout *= 3 when "debug" # Since debug gem requires debug.so in child processes without - # acitvating the gem, we preset necessary paths in RUBYLIB + # activating the gem, we preset necessary paths in RUBYLIB # environment variable. load_path = true - when /\Anet-/ - toplib = gem.tr("-", "/") + when "test-unit" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", "test/run.rb"] + + when "csv" + first_timeout = 30 + + when "win32ole" + next unless /mswin|mingw/ =~ RUBY_PLATFORM end if load_path libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) next unless $?.success? - puts libs ENV["RUBYLIB"] = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) else ENV["RUBYLIB"] = rubylib end + # 93(bright yellow) is copied from .github/workflows/mingw.yml + puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" print "[command]" if github_actions - puts test_command - pid = Process.spawn(test_command, "#{/mingw|mswin/ =~ RUBY_PLATFORM ? 'new_' : ''}pgroup": true) - {nil => first_timeout, INT: 30, TERM: 10, KILL: nil}.each do |sig, sec| + p test_command + timeouts = {nil => first_timeout, INT: 30, TERM: 10, KILL: nil} + if /mingw|mswin/ =~ RUBY_PLATFORM + timeouts.delete(:TERM) # Inner process signal on Windows + group = :new_pgroup + pg = "" + else + group = :pgroup + pg = "-" + end + pid = Process.spawn(*test_command, group => true) + timeouts.each do |sig, sec| if sig puts "Sending #{sig} signal" - Process.kill("-#{sig}", pid) + Process.kill("#{pg}#{sig}", pid) end begin break Timeout.timeout(sec) {Process.wait(pid)} @@ -71,12 +114,12 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| end rescue Interrupt exit_code = Signal.list["INT"] - Process.kill("-KILL", pid) + Process.kill("#{pg}KILL", pid) Process.wait(pid) break end - print "##[endgroup]\n" if github_actions + print "::endgroup::\n" if github_actions unless $?.success? mesg = "Tests failed " + @@ -84,11 +127,11 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| "with exit code #{$?.exitstatus}") puts colorize.decorate(mesg, "fail") if allowed_failures.include?(gem) - mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES" + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" puts colorize.decorate(mesg, "skip") else failed << gem - exit_code = $?.exitstatus if $?.exitstatus + exit_code = 1 end end end diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb index c4ffa66a4f..28ef0bf7f8 100644 --- a/tool/test-coverage.rb +++ b/tool/test-coverage.rb @@ -4,6 +4,16 @@ Coverage.start(lines: true, branches: true, methods: true) TEST_COVERAGE_DATA_FILE = "test-coverage.dat" +FILTER_PATHS = %w[ + lib/bundler/vendor + lib/rubygems/resolver/molinillo + lib/rubygems/tsort + lib/rubygems/optparse + tool + test + spec +] + def merge_coverage_data(res1, res2) res1.each do |path, cov1| cov2 = res2[path] @@ -77,8 +87,8 @@ def invoke_simplecov_formatter res.each do |path, cov| next unless path.start_with?(base_dir) || path.start_with?(cur_dir) - next if path.start_with?(File.join(base_dir, "test")) - simplecov_result[path] = cov[:lines] + next if FILTER_PATHS.any? {|dir| path.start_with?(File.join(base_dir, dir))} + simplecov_result[path] = cov end a, b = base_dir, cur_dir @@ -104,6 +114,10 @@ pid = $$ pwd = Dir.pwd at_exit do + # Some tests leave GC.stress enabled, causing slow coverage processing. + # Reset it here to avoid performance issues. + GC.stress = false + exit_exc = $! Dir.chdir(pwd) do diff --git a/tool/test/init.rb b/tool/test/init.rb new file mode 100644 index 0000000000..3fd1419a9c --- /dev/null +++ b/tool/test/init.rb @@ -0,0 +1,26 @@ +# This file includes the settings for "make test-all" and "make test-tool". +# Note that this file is loaded not only by test/runner.rb but also by tool/lib/test/unit/parallel.rb. + +# Prevent test-all from using bundled gems +["GEM_HOME", "GEM_PATH"].each do |gem_env| + # Preserve the gem environment prepared by tool/runruby.rb for test-tool, which uses bundled gems. + ENV["BUNDLED_#{gem_env}"] = ENV[gem_env] + + ENV[gem_env] = "".freeze +end +ENV["GEM_SKIP"] = "".freeze + +ENV.delete("RUBY_CODESIGN") + +Warning[:experimental] = false + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) + +require 'test/unit' + +require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') +require "tracepointchecker" +require "zombie_hunter" +require "iseq_loader_checker" +require "gc_checker" +require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') diff --git a/tool/test/runner.rb b/tool/test/runner.rb index c629943090..9001fc2d06 100644 --- a/tool/test/runner.rb +++ b/tool/test/runner.rb @@ -1,16 +1,7 @@ # frozen_string_literal: true require 'rbconfig' -$LOAD_PATH.unshift File.expand_path("../lib", __dir__) - -require 'test/unit' - -require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') -require "tracepointchecker" -require "zombie_hunter" -require "iseq_loader_checker" -require "gc_checker" -require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') +require_relative "init" case $0 when __FILE__ @@ -18,6 +9,6 @@ when __FILE__ when "-e" # No default directory else - dir = File.expand_path("..", $0) + dir = File.realdirpath("..", $0) end exit Test::Unit::AutoRunner.run(true, dir) diff --git a/tool/test/test_commit_email.rb b/tool/test/test_commit_email.rb new file mode 100644 index 0000000000..db441584fd --- /dev/null +++ b/tool/test/test_commit_email.rb @@ -0,0 +1,102 @@ +require 'test/unit' +require 'shellwords' +require 'tmpdir' +require 'fileutils' +require 'open3' + +class TestCommitEmail < Test::Unit::TestCase + STDIN_DELIMITER = "---\n" + + def setup + omit 'git command is not available' unless system('git', '--version', out: File::NULL, err: File::NULL) + + @ruby = Dir.mktmpdir + Dir.chdir(@ruby) do + git('init', '--initial-branch=master') + git('config', 'user.name', 'Jóhän Grübél') + git('config', 'user.email', 'johan@example.com') + env = { + 'GIT_AUTHOR_DATE' => '2025-10-08T12:00:00Z', + 'GIT_CONFIG_GLOBAL' => @ruby + "/gitconfig", + 'TZ' => 'UTC', + } + git('commit', '--allow-empty', '-m', 'New repository initialized by cvs2svn.', env:) + git('commit', '--allow-empty', '-m', 'Initial revision', env:) + git('commit', '--allow-empty', '-m', 'version 1.0.0', env:) + end + + @sendmail = File.join(Dir.mktmpdir, 'sendmail') + File.write(@sendmail, <<~SENDMAIL, mode: "wx", perm: 0755) + #!/bin/sh + echo #{STDIN_DELIMITER.chomp.dump} + exec cat + SENDMAIL + + @commit_email = File.expand_path('../../tool/commit-email.rb', __dir__) + end + + def teardown + # Clean up temporary files if #setup was not omitted + if @sendmail + File.unlink(@sendmail) + Dir.rmdir(File.dirname(@sendmail)) + end + if @ruby + FileUtils.rm_rf(@ruby) + end + end + + def test_sendmail_encoding + omit 'the sendmail script does not work on windows' if windows? + + Dir.chdir(@ruby) do + before_rev = git('rev-parse', 'HEAD^').chomp + after_rev = git('rev-parse', 'HEAD').chomp + short_rev = after_rev[0...10] + + out, _, status = EnvUtil.invoke_ruby([ + { 'SENDMAIL' => @sendmail, 'TZ' => 'UTC' }.merge!(gem_env), + @commit_email, './', 'cvs-admin@ruby-lang.org', + before_rev, after_rev, 'refs/heads/master', + '--viewer-uri', 'https://github.com/ruby/ruby/commit/', + '--error-to', 'cvs-admin@ruby-lang.org', + ], '', true) + stdin = out.b.split(STDIN_DELIMITER.b, 2).last.force_encoding('UTF-8') + + assert_true(status.success?) + assert_equal(stdin, <<~EOS) + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: =?UTF-8?B?SsOzaMOkbiBHcsO8YsOpbA==?= <noreply@ruby-lang.org> + To: cvs-admin@ruby-lang.org + Subject: #{short_rev} (master): =?UTF-8?B?dmVyc2lvbuOAgDEuMC4w?= + J=C3=B3h=C3=A4n Gr=C3=BCb=C3=A9l\t2025-10-08 12:00:00 +0000 (Wed, 08 Oct 2= + 025) + + New Revision: #{short_rev} + + https://github.com/ruby/ruby/commit/#{short_rev} + + Log: + version=E3=80=801.0.0= + EOS + end + end + + private + + # Resurrect the gem environment preserved by tool/test/init.rb. + # This should work as long as you have run `make up` or `make install`. + def gem_env + { 'GEM_PATH' => ENV['BUNDLED_GEM_PATH'], 'GEM_HOME' => ENV['BUNDLED_GEM_HOME'] } + end + + def git(*cmd, env: {}) + out, status = Open3.capture2(env, 'git', *cmd) + unless status.success? + raise "git #{cmd.shelljoin}\n#{out}" + end + out + end +end diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb index 3987b1bec5..252687f3f3 100755 --- a/tool/test/test_sync_default_gems.rb +++ b/tool/test/test_sync_default_gems.rb @@ -1,6 +1,8 @@ #!/usr/bin/ruby require 'test/unit' require 'stringio' +require 'tmpdir' +require 'rubygems/version' require_relative '../sync_default_gems' module Test_SyncDefaultGems @@ -18,14 +20,8 @@ module Test_SyncDefaultGems expected.concat(trailers.map {_1+"\n"}) end - out, err = capture_output do - SyncDefaultGems.message_filter(repo, sha, input: StringIO.new(input, "r")) - end - - all_assertions do |a| - a.for("error") {assert_empty err} - a.for("result") {assert_pattern_list(expected, out)} - end + out = SyncDefaultGems.message_filter(repo, sha, input) + assert_pattern_list(expected, out) end def test_subject_only @@ -72,5 +68,306 @@ module Test_SyncDefaultGems ] assert_message_filter(expected, trailers, [expected, "", trailers].join("\n")) end + + def test_dot_ending_subject + expected = [ + "subject with a dot.", + "", + "- next body line", + ] + assert_message_filter(expected, nil, [expected[0], expected[2], ""].join("\n")) + end end + + class TestSyncWithCommits < Test::Unit::TestCase + def setup + super + @target = nil + pend "No git" unless system("git --version", out: IO::NULL) + @testdir = Dir.mktmpdir("sync") + user, email = "Ruby", "test@ruby-lang.org" + @git_config = %W"HOME USER GIT_CONFIG_GLOBAL GNUPGHOME".each_with_object({}) {|k, c| c[k] = ENV[k]} + ENV["HOME"] = @testdir + ENV["USER"] = user + ENV["GNUPGHOME"] = @testdir + '/.gnupg' + expire = EnvUtil.apply_timeout_scale(30).to_i + # Generate a new unprotected key with default parameters that + # expires after 30 seconds. + if @gpgsign = system(*%w"gpg --quiet --batch --passphrase", "", + "--quick-generate-key", email, *%W"default default seconds=#{expire}", + err: IO::NULL) + # Fetch the generated public key. + signingkey = IO.popen(%W"gpg --quiet --list-public-key #{email}", &:read)[/^pub .*\n +\K\h+/] + end + ENV["GIT_CONFIG_GLOBAL"] = @testdir + "/gitconfig" + git(*%W"config --global user.email", email) + git(*%W"config --global user.name", user) + git(*%W"config --global init.defaultBranch default") + if signingkey + git(*%W"config --global user.signingkey", signingkey) + git(*%W"config --global commit.gpgsign true") + git(*%W"config --global gpg.program gpg") + git(*%W"config --global log.showSignature true") + end + @target = "sync-test" + SyncDefaultGems::REPOSITORIES[@target] = SyncDefaultGems.repo( + ["ruby/#{@target}", "default"], + [ + ["lib", "lib"], + ["test", "test"], + ], + exclude: [ + "test/fixtures/*", + ], + ) + @sha = {} + @origdir = Dir.pwd + Dir.chdir(@testdir) + ["src", @target].each do |dir| + git(*%W"init -q #{dir}") + File.write("#{dir}/.gitignore", "*~\n") + Dir.mkdir("#{dir}/lib") + File.write("#{dir}/lib/common.rb", ":ok\n") + Dir.mkdir("#{dir}/.github") + Dir.mkdir("#{dir}/.github/workflows") + File.write("#{dir}/.github/workflows/default.yml", "default:\n") + git(*%W"add .gitignore lib/common.rb .github", chdir: dir) + git(*%W"commit -q -m", "Initialize", chdir: dir) + if dir == "src" + File.write("#{dir}/lib/fine.rb", "return\n") + Dir.mkdir("#{dir}/test") + File.write("#{dir}/test/test_fine.rb", "return\n") + git(*%W"add lib/fine.rb test/test_fine.rb", chdir: dir) + git(*%W"commit -q -m", "Looks fine", chdir: dir) + end + Dir.mkdir("#{dir}/tool") + File.write("#{dir}/tool/ok", "#!/bin/sh\n""echo ok\n") + git(*%W"add tool/ok", chdir: dir) + git(*%W"commit -q -m", "Add tool #{dir}", chdir: dir) + @sha[dir] = top_commit(dir) + end + git(*%W"remote add #{@target} ../#{@target}", chdir: "src") + end + + def teardown + if @target + if @gpgsign + system(*%W"gpgconf --kill all") + end + Dir.chdir(@origdir) + SyncDefaultGems::REPOSITORIES.delete(@target) + ENV.update(@git_config) + FileUtils.rm_rf(@testdir) + end + super + end + + def capture_process_output_to(outputs) + return yield unless outputs&.empty? == false + IO.pipe do |r, w| + orig = outputs.map {|out| out.dup} + outputs.each {|out| out.reopen(w)} + w.close + reader = Thread.start {r.read} + yield + ensure + outputs.each {|out| o = orig.shift; out.reopen(o); o.close} + return reader.value + end + end + + def capture_process_outputs + out = err = nil + synchronize do + out = capture_process_output_to(STDOUT) do + err = capture_process_output_to(STDERR) do + yield + end + end + end + return out, err + end + + def git(*commands, **opts) + system("git", *commands, exception: true, **opts) + end + + def top_commit(dir, format: "%H") + IO.popen(%W[git log --no-show-signature --format=#{format} -1], chdir: dir, &:read)&.chomp + end + + def assert_sync(commits = true, success: true, editor: nil) + result = nil + out = capture_process_output_to([STDOUT, STDERR]) do + Dir.chdir("src") do + orig_editor = ENV["GIT_EDITOR"] + ENV["GIT_EDITOR"] = editor || 'false' + edit = true if editor + + result = SyncDefaultGems.sync_default_gems_with_commits(@target, commits, edit: edit) + ensure + ENV["GIT_EDITOR"] = orig_editor + end + end + assert_equal(success, result, out) + out + end + + def test_sync + File.write("#@target/lib/common.rb", "# OK!\n") + git(*%W"commit -q -m", "OK", "lib/common.rb", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# OK!\n", File.read("src/lib/common.rb")) + log = top_commit("src", format: "%B").lines + assert_equal("[ruby/#@target] OK\n", log.first, out) + assert_match(%r[/ruby/#{@target}/commit/\h+$], log.last, out) + assert_operator(top_commit(@target), :start_with?, log.last[/\h+$/], out) + end + + def test_skip_tool + git(*%W"rm -q tool/ok", chdir: @target) + git(*%W"commit -q -m", "Remove tool", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_test_fixtures + Dir.mkdir("#@target/test") + Dir.mkdir("#@target/test/fixtures") + File.write("#@target/test/fixtures/fixme.rb", "") + git(*%W"add test/fixtures/fixme.rb", chdir: @target) + git(*%W"commit -q -m", "Add fixtures", chdir: @target) + out = assert_sync(["#{@sha[@target]}..#{@target}/default"]) + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= NEWS!!!\n") + git(*%W"add --", "docs/NEWS.md", chdir: @target) + File.write("#@target/docs/hello.md", "Hello\n") + git(*%W"add --", "docs/hello.md", chdir: @target) + git(*%W"commit -q -m", "It's a news", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_adding_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= New library\n") + File.write("#@target/lib/news.rb", "return\n") + git(*%W"add --", "docs/NEWS.md", "lib/news.rb", chdir: @target) + git(*%W"commit -q -m", "New lib", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal "return\n", File.read("src/lib/news.rb") + assert_include top_commit("src", format: "oneline"), "[ruby/#{@target}] New lib" + assert_not_operator File, :exist?, "src/docs" + end + + def test_gitignore + File.write("#@target/.gitignore", "*.bak\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add .github", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", + *%W".gitignore lib/common.rb .github", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("*~\n", File.read("src/.gitignore"), out) + assert_equal("#!/bin/sh\n""echo ok\n", File.read("src/tool/ok"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_gitignore_after_conflict + File.write("src/Gemfile", "# main\n") + git(*%W"add Gemfile", chdir: "src") + git(*%W"commit -q -m", "Add Gemfile", chdir: "src") + File.write("#@target/Gemfile", "# conflict\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add Gemfile .github lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# main\n", File.read("src/Gemfile"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_delete_after_conflict + File.write("#@target/lib/bad.rb", "raise\n") + git(*%W"add lib/bad.rb", chdir: @target) + git(*%W"commit -q -m", "Add bad.rb", chdir: @target) + out = assert_sync + assert_equal("raise\n", File.read("src/lib/bad.rb")) + + git(*%W"rm lib/bad.rb", chdir: "src", out: IO::NULL) + git(*%W"commit -q -m", "Remove bad.rb", chdir: "src") + + File.write("#@target/lib/bad.rb", "raise 'bar'\n") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + git(*%W"add lib/bad.rb lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict", chdir: @target) + + head = top_commit("src") + out = assert_sync(editor: "git rm -f lib/bad.rb") + assert_not_equal(head, top_commit("src")) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/lib/bad.rb", out) + end + + def test_squash_merge + # This test is known to fail with git 2.43.0, which is used by Ubuntu 24.04. + # We don't know which exact version fixed it, but we know git 2.52.0 works. + stdout, status = Open3.capture2('git', '--version', err: File::NULL) + omit 'git version check failed' unless status.success? + git_version = stdout[/\Agit version \K\S+/] + omit "git #{git_version} is too old" if Gem::Version.new(git_version) < Gem::Version.new('2.44.0') + + # 2---. <- branch + # / \ + # 1---3---3'<- merge commit with conflict resolution + File.write("#@target/lib/conflict.rb", "# 1\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict.rb", chdir: @target) + + git(*%W"checkout -q -b branch", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 2\n") + File.write("#@target/lib/new.rb", "# new\n") + git(*%W"add lib/conflict.rb lib/new.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in branch", chdir: @target) + + git(*%W"checkout -q default", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 3\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in default", chdir: @target) + + # How can I suppress "Auto-merging ..." message from git merge? + git(*%W"merge -X ours -m", "Merge commit", "branch", chdir: @target, out: IO::NULL) + + out = assert_sync() + assert_equal("# 3\n", File.read("src/lib/conflict.rb"), out) + subject, body = top_commit("src", format: "%B").split("\n\n", 2) + assert_equal("[ruby/#@target] Merge commit", subject, out) + assert_includes(body, "Commit in branch", out) + end + + def test_no_upstream_file + group = SyncDefaultGems::Repository.group(%w[ + lib/un.rb + lib/unicode_normalize/normalize.rb + lib/unicode_normalize/tables.rb + lib/net/https.rb + ]) + expected = { + "un" => %w[lib/un.rb], + "net-http" => %w[lib/net/https.rb], + nil => %w[lib/unicode_normalize/normalize.rb lib/unicode_normalize/tables.rb], + } + assert_equal(expected, group) + end + end if /darwin|linux/ =~ RUBY_PLATFORM end diff --git a/tool/test/testunit/test4test_load_failure.rb b/tool/test/testunit/test4test_load_failure.rb new file mode 100644 index 0000000000..e1570c2542 --- /dev/null +++ b/tool/test/testunit/test4test_load_failure.rb @@ -0,0 +1 @@ +raise LoadError, "no-such-library" diff --git a/tool/test/testunit/test4test_timeout.rb b/tool/test/testunit/test4test_timeout.rb new file mode 100644 index 0000000000..3225f66398 --- /dev/null +++ b/tool/test/testunit/test4test_timeout.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' +require 'timeout' + +class TestForTestTimeout < Test::Unit::TestCase + 10.times do |i| + define_method("test_timeout_#{i}") do + Timeout.timeout(0.001) do + sleep + end + end + end +end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index 709b495572..d9bdc8f3c5 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -8,6 +8,8 @@ class TestAssertion < Test::Unit::TestCase end def test_timeout_separately + pend "hang-up" if /mswin|mingw/ =~ RUBY_PLATFORM + assert_raise(Timeout::Error) do assert_separately([], <<~"end;", timeout: 0.1) sleep @@ -15,6 +17,29 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assertion_count_separately + beginning = self._assertions + + assert_separately([], "") + assertions_at_nothing = self._assertions - beginning + + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "assert true") + assert_equal(1, self._assertions - prev_assertions) + + omit unless Process.respond_to?(:fork) + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "Process.fork {assert true}; assert true") + assert_equal(2, self._assertions - prev_assertions) + + prev_assertions = self._assertions + assertions_at_nothing + # TODO: assertions before `fork` are counted twice; it is possible + # to reset `_assertions` at `Process._fork`, but the hook can + # interfere in other tests. + assert_separately([], "assert true; Process.fork {assert true}") + assert_equal(3, self._assertions - prev_assertions) + end + def return_in_assert_raise assert_raise(RuntimeError) do return @@ -27,6 +52,38 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assert_raise_with_message + my_error = Class.new(StandardError) + + assert_raise_with_message(my_error, "with message") do + raise my_error, "with message" + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(RuntimeError, "with message") do + raise my_error, "with message" + end + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(my_error, "without message") do + raise my_error, "with message" + end + end + end + + def test_assert_raise_kind_of + my_error = Class.new(StandardError) + + assert_raise_kind_of(my_error) do + raise my_error + end + + assert_raise_kind_of(StandardError) do + raise my_error + end + end + def test_assert_pattern_list assert_pattern_list([/foo?/], "foo") assert_not_pattern_list([/foo?/], "afoo") @@ -50,4 +107,122 @@ class TestAssertion < Test::Unit::TestCase assert_pattern_list(pattern_list, actual, message) end end + + def test_caller_bactrace_location + begin + line = __LINE__; assert_fail_for_backtrace_location + rescue Test::Unit::AssertionFailedError => e + end + location = Test::Unit::Runner.new.location(e) + assert_equal "#{__FILE__}:#{line}", location + end + + def assert_fail_for_backtrace_location + assert false + end + + VersionClass = Struct.new(:version) do + def version?(*ver) + Test::Unit::CoreAssertions.version_match?(ver, self.version) + end + end + + V14_6_0 = VersionClass.new([14, 6, 0]) + V15_0_0 = VersionClass.new([15, 0, 0]) + + def test_version_match_integer + assert_not_operator(V14_6_0, :version?, 13) + assert_operator(V14_6_0, :version?, 14) + assert_not_operator(V14_6_0, :version?, 15) + assert_not_operator(V15_0_0, :version?, 14) + assert_operator(V15_0_0, :version?, 15) + end + + def test_version_match_integer_range + assert_operator(V14_6_0, :version?, 13..14) + assert_not_operator(V15_0_0, :version?, 13..14) + assert_not_operator(V14_6_0, :version?, 13...14) + assert_not_operator(V15_0_0, :version?, 13...14) + end + + def test_version_match_array_range + assert_operator(V14_6_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V14_6_0, :version?, [14, 0]...[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[14, 6]) + assert_operator(V14_6_0, :version?, [14, 0]..[15]) + assert_operator(V15_0_0, :version?, [14, 0]..[15]) + assert_operator(V14_6_0, :version?, [14, 0]...[15]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[15]) + end + + def test_version_match_integer_endless_range + assert_operator(V14_6_0, :version?, 14..) + assert_operator(V15_0_0, :version?, 14..) + assert_not_operator(V14_6_0, :version?, 15..) + assert_operator(V15_0_0, :version?, 15..) + end + + def test_version_match_integer_endless_range_exclusive + assert_operator(V14_6_0, :version?, 14...) + assert_operator(V15_0_0, :version?, 14...) + assert_not_operator(V14_6_0, :version?, 15...) + assert_operator(V15_0_0, :version?, 15...) + end + + def test_version_match_array_endless_range + assert_operator(V14_6_0, :version?, [14, 5]..) + assert_operator(V15_0_0, :version?, [14, 5]..) + assert_not_operator(V14_6_0, :version?, [14, 7]..) + assert_operator(V15_0_0, :version?, [14, 7]..) + assert_not_operator(V14_6_0, :version?, [15]..) + assert_operator(V15_0_0, :version?, [15]..) + assert_not_operator(V14_6_0, :version?, [15, 0]..) + assert_operator(V15_0_0, :version?, [15, 0]..) + end + + def test_version_match_array_endless_range_exclude_end + assert_operator(V14_6_0, :version?, [14, 5]...) + assert_operator(V15_0_0, :version?, [14, 5]...) + assert_not_operator(V14_6_0, :version?, [14, 7]...) + assert_operator(V15_0_0, :version?, [14, 7]...) + assert_not_operator(V14_6_0, :version?, [15]...) + assert_operator(V15_0_0, :version?, [15]...) + assert_not_operator(V14_6_0, :version?, [15, 0]...) + assert_operator(V15_0_0, :version?, [15, 0]...) + end + + def test_version_match_integer_beginless_range + assert_operator(V14_6_0, :version?, ..14) + assert_not_operator(V15_0_0, :version?, ..14) + assert_operator(V14_6_0, :version?, ..15) + assert_operator(V15_0_0, :version?, ..15) + + assert_not_operator(V14_6_0, :version?, ...14) + assert_not_operator(V15_0_0, :version?, ...14) + assert_operator(V14_6_0, :version?, ...15) + assert_not_operator(V15_0_0, :version?, ...15) + end + + def test_version_match_array_beginless_range + assert_not_operator(V14_6_0, :version?, ..[14, 5]) + assert_not_operator(V15_0_0, :version?, ..[14, 5]) + assert_operator(V14_6_0, :version?, ..[14, 6]) + assert_not_operator(V15_0_0, :version?, ..[14, 6]) + assert_operator(V14_6_0, :version?, ..[15]) + assert_operator(V15_0_0, :version?, ..[15]) + assert_operator(V14_6_0, :version?, ..[15, 0]) + assert_operator(V15_0_0, :version?, ..[15, 0]) + end + + def test_version_match_array_beginless_range_exclude_end + assert_not_operator(V14_6_0, :version?, ...[14, 5]) + assert_not_operator(V15_0_0, :version?, ...[14, 5]) + assert_not_operator(V14_6_0, :version?, ...[14, 6]) + assert_not_operator(V15_0_0, :version?, ...[14, 6]) + assert_operator(V14_6_0, :version?, ...[15]) + assert_not_operator(V15_0_0, :version?, ...[15]) + assert_operator(V14_6_0, :version?, ...[15, 0]) + assert_not_operator(V15_0_0, :version?, ...[15, 0]) + end end diff --git a/tool/test/testunit/test_hideskip.rb b/tool/test/testunit/test_hideskip.rb index e15947fe53..a470368bca 100644 --- a/tool/test/testunit/test_hideskip.rb +++ b/tool/test/testunit/test_hideskip.rb @@ -6,14 +6,13 @@ class TestHideSkip < Test::Unit::TestCase assert_not_match(/^ *1\) Skipped/, hideskip) assert_match(/^ *1\) Skipped.*^ *2\) Skipped/m, hideskip("--show-skip")) output = hideskip("--hide-skip") - output.gsub!(/Successful MJIT finish\n/, '') if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? assert_match(/assertions\/s.\n+2 tests, 0 assertions, 0 failures, 0 errors, 2 skips/, output) end private def hideskip(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", "--verbose", *args], err: [:child, :out]) {|f| f.read } diff --git a/tool/test/testunit/test_launchable.rb b/tool/test/testunit/test_launchable.rb new file mode 100644 index 0000000000..76be876456 --- /dev/null +++ b/tool/test/testunit/test_launchable.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: false +require 'test/unit' +require 'tempfile' +require 'json' +require_relative '../../lib/launchable' + +class TestLaunchable < Test::Unit::TestCase + def test_json_stream_writer + Tempfile.create(['launchable-test-', '.json']) do |f| + json_stream_writer = Launchable::JsonStreamWriter.new(f.path) + json_stream_writer.write_array('testCases') + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 42, + status: "TEST_FAILED", + stdout: nil, + stderr: nil, + createdAt: "2021-10-05T12:34:00", + data: { + lineNumber: 1 + } + } + ) + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 45, + status: "TEST_PASSED", + stdout: "This is stdout", + stderr: "This is stderr", + createdAt: "2021-10-05T12:36:00", + data: { + lineNumber: 10 + } + } + ) + json_stream_writer.close() + expected = <<JSON +{ + "testCases": [ + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 42, + "status": "TEST_FAILED", + "stdout": null, + "stderr": null, + "createdAt": "2021-10-05T12:34:00", + "data": { + "lineNumber": 1 + } + }, + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 45, + "status": "TEST_PASSED", + "stdout": "This is stdout", + "stderr": "This is stderr", + "createdAt": "2021-10-05T12:36:00", + "data": { + "lineNumber": 10 + } + } + ] +} +JSON + assert_equal(expected, f.read) + end + end +end diff --git a/tool/test/testunit/test_load_failure.rb b/tool/test/testunit/test_load_failure.rb new file mode 100644 index 0000000000..8defa9e39a --- /dev/null +++ b/tool/test/testunit/test_load_failure.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +require 'test/unit' + +class TestLoadFailure < Test::Unit::TestCase + def test_load_failure + assert_not_predicate(load_failure, :success?) + end + + def test_load_failure_parallel + assert_not_predicate(load_failure("-j2"), :success?) + end + + private + + def load_failure(*args) + IO.popen([*@__runner_options__[:ruby], "#{__dir__}/../runner.rb", + "#{__dir__}/test4test_load_failure.rb", + "--verbose", *args], err: [:child, :out]) {|f| + assert_include(f.read, "test4test_load_failure.rb") + } + $? + end +end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb index 006354aee2..adf7d62ecd 100644 --- a/tool/test/testunit/test_parallel.rb +++ b/tool/test/testunit/test_parallel.rb @@ -6,14 +6,22 @@ module TestParallel PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" TESTS = "#{__dir__}/tests_for_parallel" # use large timeout for --jit-wait - TIMEOUT = EnvUtil.apply_timeout_scale(defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? ? 100 : 30) + TIMEOUT = EnvUtil.apply_timeout_scale(100) + + def self.timeout(n, &blk) + start_time = Time.now + Timeout.timeout(n, &blk) + rescue Timeout::Error + end_time = Time.now + raise Timeout::Error, "execution expired (start: #{ start_time }, end: #{ end_time })" + end class TestParallelWorker < Test::Unit::TestCase def setup i, @worker_in = IO.pipe @worker_out, o = IO.pipe - @worker_pid = spawn(*@options[:ruby], PARALLEL_RB, - "--ruby", @options[:ruby].join(" "), + @worker_pid = spawn(*@__runner_options__[:ruby], PARALLEL_RB, + "--ruby", @__runner_options__[:ruby].join(" "), "-j", "t1", "-v", out: o, in: i) [i,o].each(&:close) end @@ -22,10 +30,10 @@ module TestParallel if @worker_pid && @worker_in begin begin - @worker_in.puts "quit" + @worker_in.puts "quit normal" rescue IOError, Errno::EPIPE end - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@worker_pid) end rescue Timeout::Error @@ -45,7 +53,7 @@ module TestParallel end def test_run - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) @@ -58,7 +66,7 @@ module TestParallel end def test_run_multiple_testcase_in_one_file - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_second.rb test" assert_match(/^okay/,@worker_out.gets) @@ -75,7 +83,7 @@ module TestParallel end def test_accept_run_command_multiple_times - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) @@ -99,7 +107,7 @@ module TestParallel end def test_p - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_first.rb test" while buf = @worker_out.gets break if /^p (.+?)$/ =~ buf @@ -110,7 +118,7 @@ module TestParallel end def test_done - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_forth.rb test" while buf = @worker_out.gets break if /^done (.+?)$/ =~ buf @@ -118,43 +126,43 @@ module TestParallel assert_not_nil($1, "'done' was not found") result = Marshal.load($1.chomp.unpack1("m")) - assert_equal(5, result[0]) - pend "TODO: result[1] returns 17. We should investigate it" do - assert_equal(12, result[1]) - end - assert_kind_of(Array,result[2]) - assert_kind_of(Array,result[3]) - assert_kind_of(Array,result[4]) - assert_kind_of(Array,result[2][1]) - assert_kind_of(Test::Unit::AssertionFailedError,result[2][0][2]) - assert_kind_of(Test::Unit::PendedError,result[2][1][2]) - assert_kind_of(Test::Unit::PendedError,result[2][2][2]) - assert_kind_of(Exception, result[2][3][2]) - assert_equal(result[5], "TestE") + tests, asserts, reports, failures, loadpaths, suite = result + assert_equal(5, tests) + assert_equal(12, asserts) + assert_kind_of(Array, reports) + assert_kind_of(Array, failures) + assert_kind_of(Array, loadpaths) + reports.sort_by! {|_, t| t} + assert_kind_of(Array, reports[1]) + assert_kind_of(Test::Unit::AssertionFailedError, reports[0][2]) + assert_kind_of(Test::Unit::PendedError, reports[1][2]) + assert_kind_of(Test::Unit::PendedError, reports[2][2]) + assert_kind_of(Exception, reports[3][2]) + assert_equal("TestE", suite) end end def test_quit - Timeout.timeout(TIMEOUT) do - @worker_in.puts "quit" + ::TestParallel.timeout(TIMEOUT) do + @worker_in.puts "quit normal" assert_match(/^bye$/m,@worker_out.read) end end end class TestParallel < Test::Unit::TestCase - def spawn_runner(*opt_args) + def spawn_runner(*opt_args, jobs: "t1", env: {}) @test_out, o = IO.pipe - @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", - "--ruby", @options[:ruby].join(" "), - "-j","t1",*opt_args, out: o, err: o) + @test_pid = spawn(env, *@__runner_options__[:ruby], TESTS+"/runner.rb", + "--ruby", @__runner_options__[:ruby].join(" "), + "-j", jobs, *opt_args, out: o, err: o) o.close end def teardown begin if @test_pid - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@test_pid) end end @@ -166,52 +174,48 @@ module TestParallel end def test_ignore_jzero - @test_out, o = IO.pipe - @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", - "--ruby", @options[:ruby].join(" "), - "-j","0", out: File::NULL, err: o) - o.close - Timeout.timeout(TIMEOUT) { + spawn_runner(jobs: "0") + ::TestParallel.timeout(TIMEOUT) { assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) } end def test_should_run_all_without_any_leaks spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^9 tests/,buf) end def test_should_retry_failed_on_workers - spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner "--retry" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying\.+$/,buf) end def test_no_retry_option spawn_runner "--no-retry" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} refute_match(/^Retrying\.+$/,buf) assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf) end def test_jobs_status spawn_runner "--jobs-status" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/\d+=ptest_(first|second|third|forth) */,buf) end def test_separate # this test depends to --jobs-status spawn_runner "--jobs-status", "--separate" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1, message("retried tests should run in different processes") {buf}) end def test_hungup - spawn_runner "--worker-timeout=1", "test4test_hungup.rb" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner("--worker-timeout=1", "--retry", "test4test_hungup.rb", env: {"RUBY_CRASH_REPORT"=>nil}) + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying hung up testcases\.+$/, buf) assert_match(/^2 tests,.* 0 failures,/, buf) end diff --git a/tool/test/testunit/test_sorting.rb b/tool/test/testunit/test_sorting.rb index 7678249ec2..3e5d7bfdcc 100644 --- a/tool/test/testunit/test_sorting.rb +++ b/tool/test/testunit/test_sorting.rb @@ -10,7 +10,7 @@ class TestTestUnitSorting < Test::Unit::TestCase end def sorting(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", "--verbose", *args], err: [:child, :out]) {|f| f.read } diff --git a/tool/test/testunit/test_timeout.rb b/tool/test/testunit/test_timeout.rb new file mode 100644 index 0000000000..452f5e1a7e --- /dev/null +++ b/tool/test/testunit/test_timeout.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestTiemout < Test::Unit::TestCase + def test_timeout + cmd = [*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_timeout.rb"] + result = IO.popen(cmd, err: [:child, :out], &:read) + assert_not_match(/^T{10}$/, result) + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb index 8831676e19..54474c828d 100644 --- a/tool/test/testunit/tests_for_parallel/ptest_forth.rb +++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb @@ -8,19 +8,19 @@ class TestE < Test::Unit::TestCase assert_equal(1,1) end - def test_always_skip - skip "always" + def test_always_omit + omit "always" end def test_always_fail assert_equal(0,1) end - def test_skip_after_unknown_error + def test_pend_after_unknown_error begin raise UnknownError, "unknown error" rescue - skip "after raise" + pend "after raise" end end diff --git a/tool/test/testunit/tests_for_parallel/slow_helper.rb b/tool/test/testunit/tests_for_parallel/slow_helper.rb new file mode 100644 index 0000000000..38067c1f47 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/slow_helper.rb @@ -0,0 +1,8 @@ +require 'test/unit' + +module TestSlowTimeout + def test_slow + sleep_for = EnvUtil.apply_timeout_scale((ENV['sec'] || 3).to_i) + sleep sleep_for if on_parallel_worker? + end +end diff --git a/tool/test/testunit/tests_for_parallel/test4test_hungup.rb b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb index 65a75f7c4d..49f503ba9e 100644 --- a/tool/test/testunit/tests_for_parallel/test4test_hungup.rb +++ b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb @@ -8,7 +8,7 @@ class TestHung < Test::Unit::TestCase def test_hungup_at_worker if on_parallel_worker? - sleep 10 + sleep EnvUtil.apply_timeout_scale(10) end assert true end diff --git a/tool/test/webrick/.htaccess b/tool/test/webrick/.htaccess deleted file mode 100644 index 69d4659b9f..0000000000 --- a/tool/test/webrick/.htaccess +++ /dev/null @@ -1 +0,0 @@ -this file should not be published. diff --git a/tool/test/webrick/test_cgi.rb b/tool/test/webrick/test_cgi.rb deleted file mode 100644 index 7a75cf565e..0000000000 --- a/tool/test/webrick/test_cgi.rb +++ /dev/null @@ -1,170 +0,0 @@ -# coding: US-ASCII -# frozen_string_literal: false -require_relative "utils" -require "webrick" -require "test/unit" - -class TestWEBrickCGI < Test::Unit::TestCase - CRLF = "\r\n" - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def start_cgi_server(log_tester=TestWEBrick::DefaultLogTester, &block) - config = { - :CGIInterpreter => TestWEBrick::RubyBin, - :DocumentRoot => File.dirname(__FILE__), - :DirectoryIndex => ["webrick.cgi"], - :RequestCallback => Proc.new{|req, res| - def req.meta_vars - meta = super - meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) - meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] - return meta - end - }, - } - if RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32/ - config[:CGIPathEnv] = ENV['PATH'] # runtime dll may not be in system dir. - end - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - block.call(server, addr, port, log) - } - end - - def test_cgi - start_cgi_server{|server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.cgi") - http.request(req){|res| assert_equal("/webrick.cgi", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi/path/info") - http.request(req){|res| assert_equal("/path/info", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi/%3F%3F%3F?foo=bar") - http.request(req){|res| assert_equal("/???", res.body, log.call)} - unless RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32|java/ - # Path info of res.body is passed via ENV. - # ENV[] returns different value on Windows depending on locale. - req = Net::HTTP::Get.new("/webrick.cgi/%A4%DB%A4%B2/%A4%DB%A4%B2") - http.request(req){|res| - assert_equal("/\xA4\xDB\xA4\xB2/\xA4\xDB\xA4\xB2", res.body, log.call)} - end - req = Net::HTTP::Get.new("/webrick.cgi?a=1;a=2;b=x") - http.request(req){|res| assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi?a=1&a=2&b=x") - http.request(req){|res| assert_equal("a=1, a=2, b=x", res.body, log.call)} - - req = Net::HTTP::Post.new("/webrick.cgi?a=x;a=y;b=1") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req, "a=1;a=2;b=x"){|res| - assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Post.new("/webrick.cgi?a=x&a=y&b=1") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req, "a=1&a=2&b=x"){|res| - assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Get.new("/") - http.request(req){|res| - ary = res.body.lines.to_a - assert_match(%r{/$}, ary[0], log.call) - assert_match(%r{/webrick.cgi$}, ary[1], log.call) - } - - req = Net::HTTP::Get.new("/webrick.cgi") - req["Cookie"] = "CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001" - http.request(req){|res| - assert_equal( - "CUSTOMER=WILE_E_COYOTE\nPART_NUMBER=ROCKET_LAUNCHER_0001\n", - res.body, log.call) - } - - req = Net::HTTP::Get.new("/webrick.cgi") - cookie = %{$Version="1"; } - cookie << %{Customer="WILE_E_COYOTE"; $Path="/acme"; } - cookie << %{Part_Number="Rocket_Launcher_0001"; $Path="/acme"; } - cookie << %{Shipping="FedEx"; $Path="/acme"} - req["Cookie"] = cookie - http.request(req){|res| - assert_equal("Customer=WILE_E_COYOTE, Shipping=FedEx", - res["Set-Cookie"], log.call) - assert_equal("Customer=WILE_E_COYOTE\n" + - "Part_Number=Rocket_Launcher_0001\n" + - "Shipping=FedEx\n", res.body, log.call) - } - } - end - - def test_bad_request - log_tester = lambda {|log, access_log| - assert_match(/BadRequest/, log.join) - } - start_cgi_server(log_tester) {|server, addr, port, log| - sock = TCPSocket.new(addr, port) - begin - sock << "POST /webrick.cgi HTTP/1.0" << CRLF - sock << "Content-Type: application/x-www-form-urlencoded" << CRLF - sock << "Content-Length: 1024" << CRLF - sock << CRLF - sock << "a=1&a=2&b=x" - sock.close_write - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, sock.read, log.call) - ensure - sock.close - end - } - end - - def test_cgi_env - start_cgi_server do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.cgi/dumpenv") - req['proxy'] = 'http://example.com/' - req['hello'] = 'world' - http.request(req) do |res| - env = Marshal.load(res.body) - assert_equal 'world', env['HTTP_HELLO'] - assert_not_operator env, :include?, 'HTTP_PROXY' - end - end - end - - CtrlSeq = [0x7f, *(1..31)].pack("C*").gsub(/\s+/, '') - CtrlPat = /#{Regexp.quote(CtrlSeq)}/o - DumpPat = /#{Regexp.quote(CtrlSeq.dump[1...-1])}/o - - def test_bad_uri - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR bad URI/, log[0]) - } - start_cgi_server(log_tester) {|server, addr, port, log| - res = TCPSocket.open(addr, port) {|sock| - sock << "GET /#{CtrlSeq}#{CRLF}#{CRLF}" - sock.close_write - sock.read - } - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, res) - s = log.call.each_line.grep(/ERROR bad URI/)[0] - assert_match(DumpPat, s) - assert_not_match(CtrlPat, s) - } - end - - def test_bad_header - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR bad header/, log[0]) - } - start_cgi_server(log_tester) {|server, addr, port, log| - res = TCPSocket.open(addr, port) {|sock| - sock << "GET / HTTP/1.0#{CRLF}#{CtrlSeq}#{CRLF}#{CRLF}" - sock.close_write - sock.read - } - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, res) - s = log.call.each_line.grep(/ERROR bad header/)[0] - assert_match(DumpPat, s) - assert_not_match(CtrlPat, s) - } - end -end diff --git a/tool/test/webrick/test_config.rb b/tool/test/webrick/test_config.rb deleted file mode 100644 index a54a667452..0000000000 --- a/tool/test/webrick/test_config.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/config" - -class TestWEBrickConfig < Test::Unit::TestCase - def test_server_name_default - config = WEBrick::Config::General.dup - assert_equal(false, config.key?(:ServerName)) - assert_equal(WEBrick::Utils.getservername, config[:ServerName]) - assert_equal(true, config.key?(:ServerName)) - end - - def test_server_name_set_nil - config = WEBrick::Config::General.dup.update(ServerName: nil) - assert_equal(nil, config[:ServerName]) - end -end diff --git a/tool/test/webrick/test_cookie.rb b/tool/test/webrick/test_cookie.rb deleted file mode 100644 index e46185f127..0000000000 --- a/tool/test/webrick/test_cookie.rb +++ /dev/null @@ -1,141 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/cookie" - -class TestWEBrickCookie < Test::Unit::TestCase - def test_new - cookie = WEBrick::Cookie.new("foo","bar") - assert_equal("foo", cookie.name) - assert_equal("bar", cookie.value) - assert_equal("foo=bar", cookie.to_s) - end - - def test_time - cookie = WEBrick::Cookie.new("foo","bar") - t = 1000000000 - cookie.max_age = t - assert_match(t.to_s, cookie.to_s) - - cookie = WEBrick::Cookie.new("foo","bar") - t = Time.at(1000000000) - cookie.expires = t - assert_equal(Time, cookie.expires.class) - assert_equal(t, cookie.expires) - ts = t.httpdate - cookie.expires = ts - assert_equal(Time, cookie.expires.class) - assert_equal(t, cookie.expires) - assert_match(ts, cookie.to_s) - end - - def test_parse - data = "" - data << '$Version="1"; ' - data << 'Customer="WILE_E_COYOTE"; $Path="/acme"; ' - data << 'Part_Number="Rocket_Launcher_0001"; $Path="/acme"; ' - data << 'Shipping="FedEx"; $Path="/acme"' - cookies = WEBrick::Cookie.parse(data) - assert_equal(3, cookies.size) - assert_equal(1, cookies[0].version) - assert_equal("Customer", cookies[0].name) - assert_equal("WILE_E_COYOTE", cookies[0].value) - assert_equal("/acme", cookies[0].path) - assert_equal(1, cookies[1].version) - assert_equal("Part_Number", cookies[1].name) - assert_equal("Rocket_Launcher_0001", cookies[1].value) - assert_equal(1, cookies[2].version) - assert_equal("Shipping", cookies[2].name) - assert_equal("FedEx", cookies[2].value) - - data = "hoge=moge; __div__session=9865ecfd514be7f7" - cookies = WEBrick::Cookie.parse(data) - assert_equal(2, cookies.size) - assert_equal(0, cookies[0].version) - assert_equal("hoge", cookies[0].name) - assert_equal("moge", cookies[0].value) - assert_equal("__div__session", cookies[1].name) - assert_equal("9865ecfd514be7f7", cookies[1].value) - - # don't allow ,-separator - data = "hoge=moge, __div__session=9865ecfd514be7f7" - cookies = WEBrick::Cookie.parse(data) - assert_equal(1, cookies.size) - assert_equal(0, cookies[0].version) - assert_equal("hoge", cookies[0].name) - assert_equal("moge, __div__session=9865ecfd514be7f7", cookies[0].value) - end - - def test_parse_no_whitespace - data = [ - '$Version="1"; ', - 'Customer="WILE_E_COYOTE";$Path="/acme";', # no SP between cookie-string - 'Part_Number="Rocket_Launcher_0001";$Path="/acme";', # no SP between cookie-string - 'Shipping="FedEx";$Path="/acme"' - ].join - cookies = WEBrick::Cookie.parse(data) - assert_equal(1, cookies.size) - end - - def test_parse_too_much_whitespaces - # According to RFC6265, - # cookie-string = cookie-pair *( ";" SP cookie-pair ) - # So single 0x20 is needed after ';'. We allow multiple spaces here for - # compatibility with older WEBrick versions. - data = [ - '$Version="1"; ', - 'Customer="WILE_E_COYOTE";$Path="/acme"; ', # no SP between cookie-string - 'Part_Number="Rocket_Launcher_0001";$Path="/acme"; ', # no SP between cookie-string - 'Shipping="FedEx";$Path="/acme"' - ].join - cookies = WEBrick::Cookie.parse(data) - assert_equal(3, cookies.size) - end - - def test_parse_set_cookie - data = %(Customer="WILE_E_COYOTE"; Version="1"; Path="/acme") - cookie = WEBrick::Cookie.parse_set_cookie(data) - assert_equal("Customer", cookie.name) - assert_equal("WILE_E_COYOTE", cookie.value) - assert_equal(1, cookie.version) - assert_equal("/acme", cookie.path) - - data = %(Shipping="FedEx"; Version="1"; Path="/acme"; Secure) - cookie = WEBrick::Cookie.parse_set_cookie(data) - assert_equal("Shipping", cookie.name) - assert_equal("FedEx", cookie.value) - assert_equal(1, cookie.version) - assert_equal("/acme", cookie.path) - assert_equal(true, cookie.secure) - end - - def test_parse_set_cookies - data = %(Shipping="FedEx"; Version="1"; Path="/acme"; Secure) - data << %(, CUSTOMER=WILE_E_COYOTE; path=/; expires=Wednesday, 09-Nov-99 23:12:40 GMT; path=/; Secure) - data << %(, name="Aaron"; Version="1"; path="/acme") - cookies = WEBrick::Cookie.parse_set_cookies(data) - assert_equal(3, cookies.length) - - fed_ex = cookies.find { |c| c.name == 'Shipping' } - assert_not_nil(fed_ex) - assert_equal("Shipping", fed_ex.name) - assert_equal("FedEx", fed_ex.value) - assert_equal(1, fed_ex.version) - assert_equal("/acme", fed_ex.path) - assert_equal(true, fed_ex.secure) - - name = cookies.find { |c| c.name == 'name' } - assert_not_nil(name) - assert_equal("name", name.name) - assert_equal("Aaron", name.value) - assert_equal(1, name.version) - assert_equal("/acme", name.path) - - customer = cookies.find { |c| c.name == 'CUSTOMER' } - assert_not_nil(customer) - assert_equal("CUSTOMER", customer.name) - assert_equal("WILE_E_COYOTE", customer.value) - assert_equal(0, customer.version) - assert_equal("/", customer.path) - assert_equal(Time.utc(1999, 11, 9, 23, 12, 40), customer.expires) - end -end diff --git a/tool/test/webrick/test_do_not_reverse_lookup.rb b/tool/test/webrick/test_do_not_reverse_lookup.rb deleted file mode 100644 index efcb5a9299..0000000000 --- a/tool/test/webrick/test_do_not_reverse_lookup.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick" -require_relative "utils" - -class TestDoNotReverseLookup < Test::Unit::TestCase - class DNRL < WEBrick::GenericServer - def run(sock) - sock << sock.do_not_reverse_lookup.to_s - end - end - - @@original_do_not_reverse_lookup_value = Socket.do_not_reverse_lookup - - def teardown - Socket.do_not_reverse_lookup = @@original_do_not_reverse_lookup_value - end - - def do_not_reverse_lookup?(config) - result = nil - TestWEBrick.start_server(DNRL, config) do |server, addr, port, log| - TCPSocket.open(addr, port) do |sock| - result = {'true' => true, 'false' => false}[sock.gets] - end - end - result - end - - # +--------------------------------------------------------------------------+ - # | Expected interaction between Socket.do_not_reverse_lookup | - # | and WEBrick::Config::General[:DoNotReverseLookup] | - # +----------------------------+---------------------------------------------+ - # | |WEBrick::Config::General[:DoNotReverseLookup]| - # +----------------------------+--------------+---------------+--------------+ - # |Socket.do_not_reverse_lookup| TRUE | FALSE | NIL | - # +----------------------------+--------------+---------------+--------------+ - # | TRUE | true | false | true | - # +----------------------------+--------------+---------------+--------------+ - # | FALSE | true | false | false | - # +----------------------------+--------------+---------------+--------------+ - - def test_socket_dnrl_true_server_dnrl_true - Socket.do_not_reverse_lookup = true - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => true)) - end - - def test_socket_dnrl_true_server_dnrl_false - Socket.do_not_reverse_lookup = true - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => false)) - end - - def test_socket_dnrl_true_server_dnrl_nil - Socket.do_not_reverse_lookup = true - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => nil)) - end - - def test_socket_dnrl_false_server_dnrl_true - Socket.do_not_reverse_lookup = false - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => true)) - end - - def test_socket_dnrl_false_server_dnrl_false - Socket.do_not_reverse_lookup = false - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => false)) - end - - def test_socket_dnrl_false_server_dnrl_nil - Socket.do_not_reverse_lookup = false - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => nil)) - end -end diff --git a/tool/test/webrick/test_filehandler.rb b/tool/test/webrick/test_filehandler.rb deleted file mode 100644 index 9c5b83e300..0000000000 --- a/tool/test/webrick/test_filehandler.rb +++ /dev/null @@ -1,403 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require_relative "utils.rb" -require "webrick" -require "stringio" -require "tmpdir" - -class WEBrick::TestFileHandler < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def default_file_handler(filename) - klass = WEBrick::HTTPServlet::DefaultFileHandler - klass.new(WEBrick::Config::HTTP, filename) - end - - def windows? - File.directory?("\\") - end - - def get_res_body(res) - sio = StringIO.new - sio.binmode - res.send_body(sio) - sio.string - end - - def make_range_request(range_spec) - msg = <<-END_OF_REQUEST - GET / HTTP/1.0 - Range: #{range_spec} - - END_OF_REQUEST - return StringIO.new(msg.gsub(/^ {6}/, "")) - end - - def make_range_response(file, range_spec) - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(make_range_request(range_spec)) - res = WEBrick::HTTPResponse.new(WEBrick::Config::HTTP) - size = File.size(file) - handler = default_file_handler(file) - handler.make_partial_content(req, res, file, size) - return res - end - - def test_make_partial_content - filename = __FILE__ - filesize = File.size(filename) - - res = make_range_response(filename, "bytes=#{filesize-100}-") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=-100") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-99") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=100-199") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-0") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(1, get_res_body(res).size) - - res = make_range_response(filename, "bytes=-1") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(1, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-0, -2") - assert_match(%r{^multipart/byteranges}, res["content-type"]) - body = get_res_body(res) - boundary = /; boundary=(.+)/.match(res['content-type'])[1] - off = filesize - 2 - last = filesize - 1 - - exp = "--#{boundary}\r\n" \ - "Content-Type: text/plain\r\n" \ - "Content-Range: bytes 0-0/#{filesize}\r\n" \ - "\r\n" \ - "#{File.read(__FILE__, 1)}\r\n" \ - "--#{boundary}\r\n" \ - "Content-Type: text/plain\r\n" \ - "Content-Range: bytes #{off}-#{last}/#{filesize}\r\n" \ - "\r\n" \ - "#{File.read(__FILE__, 2, off)}\r\n" \ - "--#{boundary}--\r\n" - assert_equal exp, body - end - - def test_filehandler - config = { :DocumentRoot => File.dirname(__FILE__), } - this_file = File.basename(__FILE__) - filesize = File.size(__FILE__) - this_data = File.binread(__FILE__) - range = nil - bug2593 = '[ruby-dev:40030]' - - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - begin - server[:DocumentRootOptions][:NondisclosureName] = [] - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/html", res.content_type, log.call) - assert_match(/HREF="#{this_file}"/, res.body, log.call) - } - req = Net::HTTP::Get.new("/#{this_file}") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_equal(this_data, res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=#{filesize-100}-") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-100)..(filesize-1), range, log.call) - assert_equal(this_data[-100..-1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=-100") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-100)..(filesize-1), range, log.call) - assert_equal(this_data[-100..-1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-99") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(0..99, range, log.call) - assert_equal(this_data[0..99], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=100-199") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(100..199, range, log.call) - assert_equal(this_data[100..199], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-0") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(0..0, range, log.call) - assert_equal(this_data[0..0], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=-1") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-1)..(filesize-1), range, log.call) - assert_equal(this_data[-1, 1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-0, -2") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("multipart/byteranges", res.content_type, log.call) - } - ensure - server[:DocumentRootOptions].delete :NondisclosureName - end - end - end - - def test_non_disclosure_name - config = { :DocumentRoot => File.dirname(__FILE__), } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } - assert_equal([], log) - } - this_file = File.basename(__FILE__) - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - doc_root_opts = server[:DocumentRootOptions] - doc_root_opts[:NondisclosureName] = %w(.ht* *~ test_*) - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/html", res.content_type, log.call) - assert_no_match(/HREF="#{File.basename(__FILE__)}"/, res.body) - } - req = Net::HTTP::Get.new("/#{this_file}") - http.request(req){|res| - assert_equal("404", res.code, log.call) - } - doc_root_opts[:NondisclosureName] = %w(.ht* *~ TEST_*) - http.request(req){|res| - assert_equal("404", res.code, log.call) - } - end - end - - def test_directory_traversal - return if File.executable?(__FILE__) # skip on strange file system - - config = { :DocumentRoot => File.dirname(__FILE__), } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR bad URI/ =~ s } - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/../../") - http.request(req){|res| assert_equal("400", res.code, log.call) } - req = Net::HTTP::Get.new("/..%5c../#{File.basename(__FILE__)}") - http.request(req){|res| assert_equal(windows? ? "200" : "404", res.code, log.call) } - req = Net::HTTP::Get.new("/..%5c..%5cruby.c") - http.request(req){|res| assert_equal("404", res.code, log.call) } - end - end - - def test_unwise_in_path - if windows? - config = { :DocumentRoot => File.dirname(__FILE__), } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/..%5c..") - http.request(req){|res| assert_equal("301", res.code, log.call) } - end - end - end - - def test_short_filename - return if File.executable?(__FILE__) # skip on strange file system - return if /mswin/ =~ RUBY_PLATFORM && ENV.key?('GITHUB_ACTIONS') # not working from the beginning - - config = { - :CGIInterpreter => TestWEBrick::RubyBin, - :DocumentRoot => File.dirname(__FILE__), - :CGIPathEnv => ENV['PATH'], - } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - if windows? - root = config[:DocumentRoot].tr("/", "\\") - fname = IO.popen(%W[dir /x #{root}\\webrick_long_filename.cgi], encoding: "binary", &:read) - fname.sub!(/\A.*$^$.*$^$/m, '') - if fname - fname = fname[/\s(w.+?cgi)\s/i, 1] - fname.downcase! - end - else - fname = "webric~1.cgi" - end - req = Net::HTTP::Get.new("/#{fname}/test") - http.request(req) do |res| - if windows? - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - else - assert_equal("404", res.code, log.call) - end - end - - req = Net::HTTP::Get.new("/.htaccess") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - req = Net::HTTP::Get.new("/htacce~1") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - req = Net::HTTP::Get.new("/HTACCE~1") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - end - end - - def test_multibyte_char_in_path - if Encoding.default_external == Encoding.find('US-ASCII') - reset_encoding = true - verb = $VERBOSE - $VERBOSE = false - Encoding.default_external = Encoding.find('UTF-8') - end - - c = "\u00a7" - begin - c = c.encode('filesystem') - rescue EncodingError - c = c.b - end - Dir.mktmpdir(c) do |dir| - basename = "#{c}.txt" - File.write("#{dir}/#{basename}", "test_multibyte_char_in_path") - Dir.mkdir("#{dir}/#{c}") - File.write("#{dir}/#{c}/#{basename}", "nested") - config = { - :DocumentRoot => dir, - :DirectoryIndex => [basename], - } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - path = "/#{basename}" - req = Net::HTTP::Get.new(WEBrick::HTTPUtils::escape(path)) - http.request(req){|res| assert_equal("200", res.code, log.call + "\nFilesystem encoding is #{Encoding.find('filesystem')}") } - path = "/#{c}/#{basename}" - req = Net::HTTP::Get.new(WEBrick::HTTPUtils::escape(path)) - http.request(req){|res| assert_equal("200", res.code, log.call) } - req = Net::HTTP::Get.new('/') - http.request(req){|res| - assert_equal("test_multibyte_char_in_path", res.body, log.call) - } - end - end - ensure - if reset_encoding - Encoding.default_external = Encoding.find('US-ASCII') - $VERBOSE = verb - end - end - - def test_script_disclosure - return if File.executable?(__FILE__) # skip on strange file system - - config = { - :CGIInterpreter => TestWEBrick::RubyBinArray, - :DocumentRoot => File.dirname(__FILE__), - :CGIPathEnv => ENV['PATH'], - :RequestCallback => Proc.new{|req, res| - def req.meta_vars - meta = super - meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) - meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] - return meta - end - }, - } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - http.read_timeout = EnvUtil.apply_timeout_scale(60) - http.write_timeout = EnvUtil.apply_timeout_scale(60) if http.respond_to?(:write_timeout=) - - req = Net::HTTP::Get.new("/webrick.cgi/test") - http.request(req) do |res| - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - end - - resok = windows? - response_assertion = Proc.new do |res| - if resok - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - else - assert_equal("404", res.code, log.call) - end - end - req = Net::HTTP::Get.new("/webrick.cgi%20/test") - http.request(req, &response_assertion) - req = Net::HTTP::Get.new("/webrick.cgi./test") - http.request(req, &response_assertion) - resok &&= File.exist?(__FILE__+"::$DATA") - req = Net::HTTP::Get.new("/webrick.cgi::$DATA/test") - http.request(req, &response_assertion) - end - end - - def test_erbhandler - config = { :DocumentRoot => File.dirname(__FILE__) } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.rhtml") - http.request(req) do |res| - assert_equal("200", res.code, log.call) - assert_match %r!\Areq to http://[^/]+/webrick\.rhtml {}\n!, res.body - end - end - end -end diff --git a/tool/test/webrick/test_htgroup.rb b/tool/test/webrick/test_htgroup.rb deleted file mode 100644 index 8749711df5..0000000000 --- a/tool/test/webrick/test_htgroup.rb +++ /dev/null @@ -1,19 +0,0 @@ -require "tempfile" -require "test/unit" -require "webrick/httpauth/htgroup" - -class TestHtgroup < Test::Unit::TestCase - def test_htgroup - Tempfile.create('test_htgroup') do |tmpfile| - tmpfile.close - tmp_group = WEBrick::HTTPAuth::Htgroup.new(tmpfile.path) - tmp_group.add 'superheroes', %w[spiderman batman] - tmp_group.add 'supervillains', %w[joker] - tmp_group.flush - - htgroup = WEBrick::HTTPAuth::Htgroup.new(tmpfile.path) - assert_equal(htgroup.members('superheroes'), %w[spiderman batman]) - assert_equal(htgroup.members('supervillains'), %w[joker]) - end - end -end diff --git a/tool/test/webrick/test_htmlutils.rb b/tool/test/webrick/test_htmlutils.rb deleted file mode 100644 index ae1b8efa95..0000000000 --- a/tool/test/webrick/test_htmlutils.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/htmlutils" - -class TestWEBrickHTMLUtils < Test::Unit::TestCase - include WEBrick::HTMLUtils - - def test_escape - assert_equal("foo", escape("foo")) - assert_equal("foo bar", escape("foo bar")) - assert_equal("foo&bar", escape("foo&bar")) - assert_equal("foo"bar", escape("foo\"bar")) - assert_equal("foo>bar", escape("foo>bar")) - assert_equal("foo<bar", escape("foo<bar")) - assert_equal("\u{3053 3093 306B 3061 306F}", escape("\u{3053 3093 306B 3061 306F}")) - bug8425 = '[Bug #8425] [ruby-core:55052]' - assert_nothing_raised(ArgumentError, Encoding::CompatibilityError, bug8425) { - assert_equal("\u{3053 3093 306B}\xff<", escape("\u{3053 3093 306B}\xff<")) - } - end -end diff --git a/tool/test/webrick/test_httpauth.rb b/tool/test/webrick/test_httpauth.rb deleted file mode 100644 index 9fe8af8be2..0000000000 --- a/tool/test/webrick/test_httpauth.rb +++ /dev/null @@ -1,366 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "tempfile" -require "webrick" -require "webrick/httpauth/basicauth" -require "stringio" -require_relative "utils" - -class TestWEBrickHTTPAuth < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_basic_auth - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, log[0]) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth" - - server.mount_proc(path){|req, res| - WEBrick::HTTPAuth.basic_auth(req, res, realm){|user, pass| - user == "webrick" && pass == "supersecretpassword" - } - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("webrick", "supersecretpassword") - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - g.basic_auth("webrick", "not super") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - } - end - - def test_basic_auth_sha - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.puts("webrick:{SHA}GJYFRpBbdchp595jlh3Bhfmgp8k=") - tmpfile.flush - assert_raise(NotImplementedError){ - WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path) - } - } - end - - def test_basic_auth_md5 - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.puts("webrick:$apr1$IOVMD/..$rmnOSPXr0.wwrLPZHBQZy0") - tmpfile.flush - assert_raise(NotImplementedError){ - WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path) - } - } - end - - [nil, :crypt, :bcrypt].each do |hash_algo| - # OpenBSD does not support insecure DES-crypt - next if /openbsd/ =~ RUBY_PLATFORM && hash_algo != :bcrypt - - begin - case hash_algo - when :crypt - # require 'string/crypt' - when :bcrypt - require 'bcrypt' - end - rescue LoadError - next - end - - define_method(:"test_basic_auth_htpasswd_#{hash_algo}") do - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Basic WEBrick's realm: webrick: password unmatch\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth2" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htpasswd = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - users = [] - htpasswd.each{|user, pass| users << user } - assert_equal(2, users.size, log.call) - assert(users.member?("webrick"), log.call) - assert(users.member?("foo"), log.call) - - server.mount_proc(path){|req, res| - auth = WEBrick::HTTPAuth::BasicAuth.new( - :Realm => realm, :UserDB => htpasswd, - :Logger => server.logger - ) - auth.authenticate(req, res) - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("webrick", "supersecretpassword") - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - g.basic_auth("webrick", "not super") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - } - } - end - - define_method(:"test_basic_auth_bad_username_htpasswd_#{hash_algo}") do - log_tester = lambda {|log, access_log| - assert_equal(2, log.length) - assert_match(/ERROR Basic WEBrick's realm: foo\\ebar: the user is not allowed\./, log[0]) - assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, log[1]) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htpasswd = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - users = [] - htpasswd.each{|user, pass| users << user } - server.mount_proc(path){|req, res| - auth = WEBrick::HTTPAuth::BasicAuth.new( - :Realm => realm, :UserDB => htpasswd, - :Logger => server.logger - ) - auth.authenticate(req, res) - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("foo\ebar", "passwd") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call) } - } - } - end - end - - DIGESTRES_ = / - ([a-zA-Z\-]+) - [ \t]*(?:\r\n[ \t]*)* - = - [ \t]*(?:\r\n[ \t]*)* - (?: - "((?:[^"]+|\\[\x00-\x7F])*)" | - ([!\#$%&'*+\-.0-9A-Z^_`a-z|~]+) - )/x - - def test_digest_auth - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Digest WEBrick's realm: no credentials in the request\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/, - /ERROR Digest WEBrick's realm: webrick: digest unmatch\./ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/digest_auth" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htdigest = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - users = [] - htdigest.each{|user, pass| users << user } - assert_equal(2, users.size, log.call) - assert(users.member?("webrick"), log.call) - assert(users.member?("foo"), log.call) - - auth = WEBrick::HTTPAuth::DigestAuth.new( - :Realm => realm, :UserDB => htdigest, - :Algorithm => 'MD5', - :Logger => server.logger - ) - server.mount_proc(path){|req, res| - auth.authenticate(req, res) - res.body = "hoge" - } - - Net::HTTP.start(addr, port) do |http| - g = Net::HTTP::Get.new(path) - params = {} - http.request(g) do |res| - assert_equal('401', res.code, log.call) - res["www-authenticate"].scan(DIGESTRES_) do |key, quoted, token| - params[key.downcase] = token || quoted.delete('\\') - end - params['uri'] = "http://#{addr}:#{port}#{path}" - end - - g['Authorization'] = credentials_for_request('webrick', "supersecretpassword", params) - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - - params['algorithm'].downcase! #4936 - g['Authorization'] = credentials_for_request('webrick', "supersecretpassword", params) - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - - g['Authorization'] = credentials_for_request('webrick', "not super", params) - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - end - } - } - end - - def test_digest_auth_int - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Digest wb auth-int realm: no credentials in the request\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/, - /ERROR Digest wb auth-int realm: foo: digest unmatch\./ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "wb auth-int realm" - path = "/digest_auth_int" - - Tempfile.create("test_webrick_auth_int") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - tmp_pass.set_passwd(realm, "foo", "Hunter2") - tmp_pass.flush - - htdigest = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - users = [] - htdigest.each{|user, pass| users << user } - assert_equal %w(foo), users - - auth = WEBrick::HTTPAuth::DigestAuth.new( - :Realm => realm, :UserDB => htdigest, - :Algorithm => 'MD5', - :Logger => server.logger, - :Qop => %w(auth-int), - ) - server.mount_proc(path){|req, res| - auth.authenticate(req, res) - res.body = "bbb" - } - Net::HTTP.start(addr, port) do |http| - post = Net::HTTP::Post.new(path) - params = {} - data = 'hello=world' - body = StringIO.new(data) - post.content_length = data.bytesize - post['Content-Type'] = 'application/x-www-form-urlencoded' - post.body_stream = body - - http.request(post) do |res| - assert_equal('401', res.code, log.call) - res["www-authenticate"].scan(DIGESTRES_) do |key, quoted, token| - params[key.downcase] = token || quoted.delete('\\') - end - params['uri'] = "http://#{addr}:#{port}#{path}" - end - - body.rewind - cred = credentials_for_request('foo', 'Hunter3', params, body) - post['Authorization'] = cred - post.body_stream = body - http.request(post){|res| - assert_equal('401', res.code, log.call) - assert_not_equal("bbb", res.body, log.call) - } - - body.rewind - cred = credentials_for_request('foo', 'Hunter2', params, body) - post['Authorization'] = cred - post.body_stream = body - http.request(post){|res| assert_equal("bbb", res.body, log.call)} - end - } - } - end - - def test_digest_auth_invalid - digest_auth = WEBrick::HTTPAuth::DigestAuth.new(Realm: 'realm', UserDB: '') - - def digest_auth.error(fmt, *) - end - - def digest_auth.try_bad_request(len) - request = {"Authorization" => %[Digest a="#{'\b'*len}]} - authenticate request, nil - end - - bad_request = WEBrick::HTTPStatus::BadRequest - t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - assert_raise(bad_request) {digest_auth.try_bad_request(10)} - limit = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) - [20, 50, 100, 200].each do |len| - assert_raise(bad_request) do - Timeout.timeout(len*limit) {digest_auth.try_bad_request(len)} - end - end - end - - private - def credentials_for_request(user, password, params, body = nil) - cnonce = "hoge" - nonce_count = 1 - ha1 = "#{user}:#{params['realm']}:#{password}" - if body - dig = Digest::MD5.new - while buf = body.read(16384) - dig.update(buf) - end - body.rewind - ha2 = "POST:#{params['uri']}:#{dig.hexdigest}" - else - ha2 = "GET:#{params['uri']}" - end - - request_digest = - "#{Digest::MD5.hexdigest(ha1)}:" \ - "#{params['nonce']}:#{'%08x' % nonce_count}:#{cnonce}:#{params['qop']}:" \ - "#{Digest::MD5.hexdigest(ha2)}" - "Digest username=\"#{user}\"" \ - ", realm=\"#{params['realm']}\"" \ - ", nonce=\"#{params['nonce']}\"" \ - ", uri=\"#{params['uri']}\"" \ - ", qop=#{params['qop']}" \ - ", nc=#{'%08x' % nonce_count}" \ - ", cnonce=\"#{cnonce}\"" \ - ", response=\"#{Digest::MD5.hexdigest(request_digest)}\"" \ - ", opaque=\"#{params['opaque']}\"" \ - ", algorithm=#{params['algorithm']}" - end -end diff --git a/tool/test/webrick/test_httpproxy.rb b/tool/test/webrick/test_httpproxy.rb deleted file mode 100644 index 66dae6f6f6..0000000000 --- a/tool/test/webrick/test_httpproxy.rb +++ /dev/null @@ -1,467 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require "webrick/httpproxy" -begin - require "webrick/ssl" - require "net/https" -rescue LoadError - # test_connect will be skipped -end -require File.expand_path("utils.rb", File.dirname(__FILE__)) - -class TestWEBrickHTTPProxy < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_fake_proxy - assert_nil(WEBrick::FakeProxyURI.scheme) - assert_nil(WEBrick::FakeProxyURI.host) - assert_nil(WEBrick::FakeProxyURI.port) - assert_nil(WEBrick::FakeProxyURI.path) - assert_nil(WEBrick::FakeProxyURI.userinfo) - assert_raise(NoMethodError){ WEBrick::FakeProxyURI.foo } - end - - def test_proxy - # Testing GET or POST to the proxy server - # Note that the proxy server works as the origin server. - # +------+ - # V | - # client -------> proxy ---+ - # GET / POST GET / POST - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - http = Net::HTTP.new(addr, port, addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_equal("GET / ", res.body, log.call) - } - assert_equal(1, proxy_handler_called, log.call) - assert_equal(2, request_handler_called, log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_nil(res.body, log.call) - } - assert_equal(2, proxy_handler_called, log.call) - assert_equal(4, request_handler_called, log.call) - - req = Net::HTTP::Post.new("/") - req.body = "post-data" - req.content_type = "application/x-www-form-urlencoded" - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_equal("POST / post-data", res.body, log.call) - } - assert_equal(3, proxy_handler_called, log.call) - assert_equal(6, request_handler_called, log.call) - } - end - - def test_no_proxy - # Testing GET or POST to the proxy server without proxy request. - # - # client -------> proxy - # GET / POST - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - http = Net::HTTP.new(addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_equal("GET / ", res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(1, request_handler_called, log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_nil(res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(2, request_handler_called, log.call) - - req = Net::HTTP::Post.new("/") - req.content_type = "application/x-www-form-urlencoded" - req.body = "post-data" - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_equal("POST / post-data", res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(3, request_handler_called, log.call) - } - end - - def test_big_bodies - require 'digest/md5' - rand_str = File.read(__FILE__) - rand_str.freeze - nr = 1024 ** 2 / rand_str.size # bigger works, too - exp = Digest::MD5.new - nr.times { exp.update(rand_str) } - exp = exp.hexdigest - TestWEBrick.start_httpserver do |o_server, o_addr, o_port, o_log| - o_server.mount_proc('/') do |req, res| - case req.request_method - when 'GET' - res['content-type'] = 'application/octet-stream' - if req.path == '/length' - res['content-length'] = (nr * rand_str.size).to_s - else - res.chunked = true - end - res.body = ->(socket) { nr.times { socket.write(rand_str) } } - when 'POST' - dig = Digest::MD5.new - req.body { |buf| dig.update(buf); buf.clear } - res['content-type'] = 'text/plain' - res['content-length'] = '32' - res.body = dig.hexdigest - end - end - - http = Net::HTTP.new(o_addr, o_port) - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Transfer-Encoding' => 'chunked', - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - - TestWEBrick.start_httpproxy do |p_server, p_addr, p_port, p_log| - http = Net::HTTP.new(o_addr, o_port, p_addr, p_port) - http.request_get('/length') do |res| - assert_equal(nr * rand_str.size, res.content_length) - dig = Digest::MD5.new - res.read_body { |buf| dig.update(buf); buf.clear } - assert_equal exp, dig.hexdigest - end - http.request_get('/') do |res| - assert_predicate res, :chunked? - dig = Digest::MD5.new - res.read_body { |buf| dig.update(buf); buf.clear } - assert_equal exp, dig.hexdigest - end - - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Content-Length' => (nr * rand_str.size).to_s, - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Transfer-Encoding' => 'chunked', - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - end - end - end if RUBY_VERSION >= '2.5' - - def test_http10_proxy_chunked - # Testing HTTP/1.0 client request and HTTP/1.1 chunked response - # from origin server. - # +------+ - # V | - # client -------> proxy ---+ - # GET GET - # HTTP/1.0 HTTP/1.1 - # non-chunked chunked - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - log_tester = lambda {|log, access_log| - log.reject! {|str| - %r{WARN chunked is set for an HTTP/1\.0 request\. \(ignored\)} =~ str - } - assert_equal([], log) - } - TestWEBrick.start_httpproxy(config, log_tester){|server, addr, port, log| - body = nil - server.mount_proc("/"){|req, res| - body = "#{req.request_method} #{req.path} #{req.body}" - res.chunked = true - res.body = -> (socket) { body.each_char {|c| socket.write c } } - } - - # Don't use Net::HTTP because it uses HTTP/1.1. - TCPSocket.open(addr, port) {|s| - s.write "GET / HTTP/1.0\r\nHost: localhost.localdomain\r\n\r\n" - response = s.read - assert_equal(body, response[/.*\z/]) - } - } - end - - def make_certificate(key, cn) - subject = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=#{cn}") - exts = [ - ["keyUsage", "keyEncipherment,digitalSignature", true], - ] - cert = OpenSSL::X509::Certificate.new - cert.version = 2 - cert.serial = 1 - cert.subject = subject - cert.issuer = subject - cert.public_key = key - cert.not_before = Time.now - 3600 - cert.not_after = Time.now + 3600 - ef = OpenSSL::X509::ExtensionFactory.new(cert, cert) - exts.each {|args| cert.add_extension(ef.create_extension(*args)) } - cert.sign(key, "sha256") - return cert - end if defined?(OpenSSL::SSL) - - def test_connect - # Testing CONNECT to proxy server - # - # client -----------> proxy -----------> https - # 1. CONNECT establish TCP - # 2. ---- establish SSL session ---> - # 3. ------- GET or POST ----------> - # - key = TEST_KEY_RSA2048 - cert = make_certificate(key, "127.0.0.1") - s_config = { - :SSLEnable =>true, - :ServerName => "localhost", - :SSLCertificate => cert, - :SSLPrivateKey => key, - } - config = { - :ServerName => "localhost.localdomain", - :RequestCallback => Proc.new{|req, res| - assert_equal("CONNECT", req.request_method) - }, - } - TestWEBrick.start_httpserver(s_config){|s_server, s_addr, s_port, s_log| - s_server.mount_proc("/"){|req, res| - res.body = "SSL #{req.request_method} #{req.path} #{req.body}" - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - http = Net::HTTP.new("127.0.0.1", s_port, addr, port) - http.use_ssl = true - http.verify_callback = Proc.new do |preverify_ok, store_ctx| - store_ctx.current_cert.to_der == cert.to_der - end - - req = Net::HTTP::Get.new("/") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req){|res| - assert_equal("SSL GET / ", res.body, s_log.call + log.call) - } - - req = Net::HTTP::Post.new("/") - req["Content-Type"] = "application/x-www-form-urlencoded" - req.body = "post-data" - http.request(req){|res| - assert_equal("SSL POST / post-data", res.body, s_log.call + log.call) - } - } - } - end if defined?(OpenSSL::SSL) - - def test_upstream_proxy - return if /mswin/ =~ RUBY_PLATFORM && ENV.key?('GITHUB_ACTIONS') # not working from the beginning - # Testing GET or POST through the upstream proxy server - # Note that the upstream proxy server works as the origin server. - # +------+ - # V | - # client -------> proxy -------> proxy ---+ - # GET / POST GET / POST GET / POST - # - up_proxy_handler_called = up_request_handler_called = 0 - proxy_handler_called = request_handler_called = 0 - up_config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| up_proxy_handler_called += 1}, - :RequestCallback => Proc.new{|req, res| up_request_handler_called += 1} - } - TestWEBrick.start_httpproxy(up_config){|up_server, up_addr, up_port, up_log| - up_server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - config = { - :ServerName => "localhost.localdomain", - :ProxyURI => URI.parse("http://localhost:#{up_port}"), - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1}, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1}, - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - http = Net::HTTP.new(up_addr, up_port, addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - skip res.message unless res.code == '200' - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_equal("GET / ", res.body) - } - assert_equal(1, up_proxy_handler_called, up_log.call + log.call) - assert_equal(2, up_request_handler_called, up_log.call + log.call) - assert_equal(1, proxy_handler_called, up_log.call + log.call) - assert_equal(1, request_handler_called, up_log.call + log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_nil(res.body, up_log.call + log.call) - } - assert_equal(2, up_proxy_handler_called, up_log.call + log.call) - assert_equal(4, up_request_handler_called, up_log.call + log.call) - assert_equal(2, proxy_handler_called, up_log.call + log.call) - assert_equal(2, request_handler_called, up_log.call + log.call) - - req = Net::HTTP::Post.new("/") - req.body = "post-data" - req.content_type = "application/x-www-form-urlencoded" - http.request(req){|res| - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_equal("POST / post-data", res.body, up_log.call + log.call) - } - assert_equal(3, up_proxy_handler_called, up_log.call + log.call) - assert_equal(6, up_request_handler_called, up_log.call + log.call) - assert_equal(3, proxy_handler_called, up_log.call + log.call) - assert_equal(3, request_handler_called, up_log.call + log.call) - - if defined?(OpenSSL::SSL) - # Testing CONNECT to the upstream proxy server - # - # client -------> proxy -------> proxy -------> https - # 1. CONNECT CONNECT establish TCP - # 2. -------- establish SSL session ------> - # 3. ---------- GET or POST --------------> - # - key = TEST_KEY_RSA2048 - cert = make_certificate(key, "127.0.0.1") - s_config = { - :SSLEnable =>true, - :ServerName => "localhost", - :SSLCertificate => cert, - :SSLPrivateKey => key, - } - TestWEBrick.start_httpserver(s_config){|s_server, s_addr, s_port, s_log| - s_server.mount_proc("/"){|req2, res| - res.body = "SSL #{req2.request_method} #{req2.path} #{req2.body}" - } - http = Net::HTTP.new("127.0.0.1", s_port, addr, port, up_log.call + log.call + s_log.call) - http.use_ssl = true - http.verify_callback = Proc.new do |preverify_ok, store_ctx| - store_ctx.current_cert.to_der == cert.to_der - end - - req2 = Net::HTTP::Get.new("/") - http.request(req2){|res| - assert_equal("SSL GET / ", res.body, up_log.call + log.call + s_log.call) - } - - req2 = Net::HTTP::Post.new("/") - req2.body = "post-data" - req2.content_type = "application/x-www-form-urlencoded" - http.request(req2){|res| - assert_equal("SSL POST / post-data", res.body, up_log.call + log.call + s_log.call) - } - } - end - } - } - end - - if defined?(OpenSSL::SSL) - TEST_KEY_RSA2048 = OpenSSL::PKey.read <<-_end_of_pem_ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAuV9ht9J7k4NBs38jOXvvTKY9gW8nLICSno5EETR1cuF7i4pN -s9I1QJGAFAX0BEO4KbzXmuOvfCpD3CU+Slp1enenfzq/t/e/1IRW0wkJUJUFQign -4CtrkJL+P07yx18UjyPlBXb81ApEmAB5mrJVSrWmqbjs07JbuS4QQGGXLc+Su96D -kYKmSNVjBiLxVVSpyZfAY3hD37d60uG+X8xdW5v68JkRFIhdGlb6JL8fllf/A/bl -NwdJOhVr9mESHhwGjwfSeTDPfd8ZLE027E5lyAVX9KZYcU00mOX+fdxOSnGqS/8J -DRh0EPHDL15RcJjV2J6vZjPb0rOYGDoMcH+94wIDAQABAoIBAAzsamqfYQAqwXTb -I0CJtGg6msUgU7HVkOM+9d3hM2L791oGHV6xBAdpXW2H8LgvZHJ8eOeSghR8+dgq -PIqAffo4x1Oma+FOg3A0fb0evyiACyrOk+EcBdbBeLo/LcvahBtqnDfiUMQTpy6V -seSoFCwuN91TSCeGIsDpRjbG1vxZgtx+uI+oH5+ytqJOmfCksRDCkMglGkzyfcl0 -Xc5CUhIJ0my53xijEUQl19rtWdMnNnnkdbG8PT3LZlOta5Do86BElzUYka0C6dUc -VsBDQ0Nup0P6rEQgy7tephHoRlUGTYamsajGJaAo1F3IQVIrRSuagi7+YpSpCqsW -wORqorkCgYEA7RdX6MDVrbw7LePnhyuaqTiMK+055/R1TqhB1JvvxJ1CXk2rDL6G -0TLHQ7oGofd5LYiemg4ZVtWdJe43BPZlVgT6lvL/iGo8JnrncB9Da6L7nrq/+Rvj -XGjf1qODCK+LmreZWEsaLPURIoR/Ewwxb9J2zd0CaMjeTwafJo1CZvcCgYEAyCgb -aqoWvUecX8VvARfuA593Lsi50t4MEArnOXXcd1RnXoZWhbx5rgO8/ATKfXr0BK/n -h2GF9PfKzHFm/4V6e82OL7gu/kLy2u9bXN74vOvWFL5NOrOKPM7Kg+9I131kNYOw -Ivnr/VtHE5s0dY7JChYWE1F3vArrOw3T00a4CXUCgYEA0SqY+dS2LvIzW4cHCe9k -IQqsT0yYm5TFsUEr4sA3xcPfe4cV8sZb9k/QEGYb1+SWWZ+AHPV3UW5fl8kTbSNb -v4ng8i8rVVQ0ANbJO9e5CUrepein2MPL0AkOATR8M7t7dGGpvYV0cFk8ZrFx0oId -U0PgYDotF/iueBWlbsOM430CgYEAqYI95dFyPI5/AiSkY5queeb8+mQH62sdcCCr -vd/w/CZA/K5sbAo4SoTj8dLk4evU6HtIa0DOP63y071eaxvRpTNqLUOgmLh+D6gS -Cc7TfLuFrD+WDBatBd5jZ+SoHccVrLR/4L8jeodo5FPW05A+9gnKXEXsTxY4LOUC -9bS4e1kCgYAqVXZh63JsMwoaxCYmQ66eJojKa47VNrOeIZDZvd2BPVf30glBOT41 -gBoDG3WMPZoQj9pb7uMcrnvs4APj2FIhMU8U15LcPAj59cD6S6rWnAxO8NFK7HQG -4Jxg3JNNf8ErQoCHb1B3oVdXJkmbJkARoDpBKmTCgKtP8ADYLmVPQw== ------END RSA PRIVATE KEY----- - _end_of_pem_ - end -end diff --git a/tool/test/webrick/test_httprequest.rb b/tool/test/webrick/test_httprequest.rb deleted file mode 100644 index 3c0ea937d9..0000000000 --- a/tool/test/webrick/test_httprequest.rb +++ /dev/null @@ -1,488 +0,0 @@ -# frozen_string_literal: false -require "webrick" -require "stringio" -require "test/unit" - -class TestWEBrickHTTPRequest < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_simple_request - msg = <<-_end_of_message_ -GET / - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert(req.meta_vars) # fails if @header was not initialized and iteration is attempted on the nil reference - end - - def test_parse_09 - msg = <<-_end_of_message_ - GET / - foobar # HTTP/0.9 request don't have header nor entity body. - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/", req.unparsed_uri) - assert_equal(WEBrick::HTTPVersion.new("0.9"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(false, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_parse_10 - msg = <<-_end_of_message_ - GET / HTTP/1.0 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/", req.unparsed_uri) - assert_equal(WEBrick::HTTPVersion.new("1.0"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(false, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_parse_11 - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/path", req.unparsed_uri) - assert_equal("", req.script_name) - assert_equal("/path", req.path_info) - assert_equal(WEBrick::HTTPVersion.new("1.1"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(true, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_request_uri_too_large - msg = <<-_end_of_message_ - GET /#{"a"*2084} HTTP/1.1 - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - assert_raise(WEBrick::HTTPStatus::RequestURITooLarge){ - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - } - end - - def test_parse_headers - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: test.ruby-lang.org:8080 - Connection: close - Accept: text/*;q=0.3, text/html;q=0.7, text/html;level=1, - text/html;level=2;q=0.4, */*;q=0.5 - Accept-Encoding: compress;q=0.5 - Accept-Encoding: gzip;q=1.0, identity; q=0.4, *;q=0 - Accept-Language: en;q=0.5, *; q=0 - Accept-Language: ja - Content-Type: text/plain - Content-Length: 7 - X-Empty-Header: - - foobar - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal( - URI.parse("http://test.ruby-lang.org:8080/path"), req.request_uri) - assert_equal("test.ruby-lang.org", req.host) - assert_equal(8080, req.port) - assert_equal(false, req.keep_alive?) - assert_equal( - %w(text/html;level=1 text/html */* text/html;level=2 text/*), - req.accept) - assert_equal(%w(gzip compress identity *), req.accept_encoding) - assert_equal(%w(ja en *), req.accept_language) - assert_equal(7, req.content_length) - assert_equal("text/plain", req.content_type) - assert_equal("foobar\n", req.body) - assert_equal("", req["x-empty-header"]) - assert_equal(nil, req["x-no-header"]) - assert(req.query.empty?) - end - - def test_parse_header2() - msg = <<-_end_of_message_ - POST /foo/bar/../baz?q=a HTTP/1.0 - Content-Length: 9 - User-Agent: - FOO BAR - BAZ - - hogehoge - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("POST", req.request_method) - assert_equal("/foo/baz", req.path) - assert_equal("", req.script_name) - assert_equal("/foo/baz", req.path_info) - assert_equal("9", req['content-length']) - assert_equal("FOO BAR BAZ", req['user-agent']) - assert_equal("hogehoge\n", req.body) - end - - def test_parse_headers3 - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: test.ruby-lang.org - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://test.ruby-lang.org/path"), req.request_uri) - assert_equal("test.ruby-lang.org", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: 192.168.1.1 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://192.168.1.1/path"), req.request_uri) - assert_equal("192.168.1.1", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: [fe80::208:dff:feef:98c7] - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://[fe80::208:dff:feef:98c7]/path"), - req.request_uri) - assert_equal("[fe80::208:dff:feef:98c7]", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: 192.168.1.1:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://192.168.1.1:8080/path"), req.request_uri) - assert_equal("192.168.1.1", req.host) - assert_equal(8080, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: [fe80::208:dff:feef:98c7]:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://[fe80::208:dff:feef:98c7]:8080/path"), - req.request_uri) - assert_equal("[fe80::208:dff:feef:98c7]", req.host) - assert_equal(8080, req.port) - end - - def test_parse_get_params - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - GET /path?#{param} HTTP/1.1 - Host: test.ruby-lang.org:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - query = req.query - assert_equal("1", query["foo"]) - assert_equal(["1", "2", "3"], query["foo"].to_ary) - assert_equal(["1", "2", "3"], query["foo"].list) - assert_equal("x", query["bar"]) - assert_equal(["x"], query["bar"].list) - end - - def test_parse_post_params - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Length: #{param.size} - Content-Type: application/x-www-form-urlencoded - - #{param} - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - query = req.query - assert_equal("1", query["foo"]) - assert_equal(["1", "2", "3"], query["foo"].to_ary) - assert_equal(["1", "2", "3"], query["foo"].list) - assert_equal("x", query["bar"]) - assert_equal(["x"], query["bar"].list) - end - - def test_chunked - crlf = "\x0d\x0a" - expect = File.binread(__FILE__).freeze - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - Host: test.ruby-lang.org:8080 - Transfer-Encoding: chunked - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - File.open(__FILE__){|io| - while chunk = io.read(100) - msg << chunk.size.to_s(16) << crlf - msg << chunk << crlf - end - } - msg << "0" << crlf - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal(expect, req.body) - - # chunked req.body_reader - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - dst = StringIO.new - IO.copy_stream(req.body_reader, dst) - assert_equal(expect, dst.string) - end - - def test_forwarded - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - User-Agent: w3m/0.5.2 - X-Forwarded-For: 123.123.123.123 - X-Forwarded-Host: forward.example.com - X-Forwarded-Server: server.example.com - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("http://forward.example.com/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(80, req.port) - assert_equal("123.123.123.123", req.remote_ip) - assert(!req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - User-Agent: w3m/0.5.2 - X-Forwarded-For: 192.168.1.10, 172.16.1.1, 123.123.123.123 - X-Forwarded-Host: forward.example.com:8080 - X-Forwarded-Server: server.example.com - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("http://forward.example.com:8080/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(8080, req.port) - assert_equal("123.123.123.123", req.remote_ip) - assert(!req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https, http - X-Forwarded-For: 192.168.1.10, 10.0.0.1, 123.123.123.123 - X-Forwarded-Host: forward.example.com - X-Forwarded-Server: server.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("https://forward.example.com/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(443, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: forward1.example.com:1234, forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://forward1.example.com:1234/foo", req.request_uri.to_s) - assert_equal("forward1.example.com", req.host) - assert_equal(1234, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: [fd20:8b1e:b255:8154:250:56ff:fea8:4d84], forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]/foo", req.request_uri.to_s) - assert_equal("[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]", req.host) - assert_equal(443, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: [fd20:8b1e:b255:8154:250:56ff:fea8:4d84]:1234, forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]:1234/foo", req.request_uri.to_s) - assert_equal("[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]", req.host) - assert_equal(1234, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - end - - def test_continue_sent - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - Expect: 100-continue - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert req['expect'] - l = msg.size - req.continue - assert_not_equal l, msg.size - assert_match(/HTTP\/1.1 100 continue\r\n\r\n\z/, msg) - assert !req['expect'] - end - - def test_continue_not_sent - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert !req['expect'] - l = msg.size - req.continue - assert_equal l, msg.size - end - - def test_empty_post - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Type: application/x-www-form-urlencoded - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - end - - def test_bad_messages - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Type: application/x-www-form-urlencoded - - #{param} - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::LengthRequired){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Length: 100000 - - body is too short. - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::BadRequest){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Transfer-Encoding: foobar - - body is too short. - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::NotImplemented){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - end - - def test_eof_raised_when_line_is_nil - assert_raise(WEBrick::HTTPStatus::EOFError) { - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new("")) - } - end -end diff --git a/tool/test/webrick/test_httpresponse.rb b/tool/test/webrick/test_httpresponse.rb deleted file mode 100644 index 4410f63e89..0000000000 --- a/tool/test/webrick/test_httpresponse.rb +++ /dev/null @@ -1,282 +0,0 @@ -# frozen_string_literal: false -require "webrick" -require "test/unit" -require "stringio" -require "net/http" - -module WEBrick - class TestHTTPResponse < Test::Unit::TestCase - class FakeLogger - attr_reader :messages - - def initialize - @messages = [] - end - - def warn msg - @messages << msg - end - end - - attr_reader :config, :logger, :res - - def setup - super - @logger = FakeLogger.new - @config = Config::HTTP - @config[:Logger] = logger - @res = HTTPResponse.new config - @res.keep_alive = true - end - - def test_prevent_response_splitting_headers_crlf - res['X-header'] = "malicious\r\nCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_crlf - user_input = "malicious\r\nCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_headers_cr - res['X-header'] = "malicious\rCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_cr - user_input = "malicious\rCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_headers_lf - res['X-header'] = "malicious\nCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_lf - user_input = "malicious\nCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_set_redirect_response_splitting - url = "malicious\r\nCookie: cracked_indicator_for_test" - assert_raise(URI::InvalidURIError) do - res.set_redirect(WEBrick::HTTPStatus::MultipleChoices, url) - end - end - - def test_set_redirect_html_injection - url = 'http://example.com////?a</a><head></head><body><img src=1></body>' - assert_raise(WEBrick::HTTPStatus::MultipleChoices) do - res.set_redirect(WEBrick::HTTPStatus::MultipleChoices, url) - end - res.status = 300 - io = StringIO.new - res.send_response(io) - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '300', res.code - refute_match(/<img/, io.string) - end - - def test_304_does_not_log_warning - res.status = 304 - res.setup_header - assert_equal 0, logger.messages.length - end - - def test_204_does_not_log_warning - res.status = 204 - res.setup_header - - assert_equal 0, logger.messages.length - end - - def test_1xx_does_not_log_warnings - res.status = 105 - res.setup_header - - assert_equal 0, logger.messages.length - end - - def test_200_chunked_does_not_set_content_length - res.chunked = false - res["Transfer-Encoding"] = 'chunked' - res.setup_header - assert_nil res.header.fetch('content-length', nil) - end - - def test_send_body_io - IO.pipe {|body_r, body_w| - body_w.write 'hello' - body_w.close - - @res.body = body_r - - IO.pipe {|r, w| - - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string - @res.body = 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_io - @res.body = StringIO.new 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_io_chunked - @res.chunked = true - - IO.pipe {|body_r, body_w| - - body_w.write 'hello' - body_w.close - - @res.body = body_r - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_chunked - @res.chunked = true - - @res.body = 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_io_chunked - @res.chunked = true - - @res.body = StringIO.new 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_proc - @res.body = Proc.new { |out| out.write('hello') } - IO.pipe do |r, w| - @res.send_body(w) - w.close - r.binmode - assert_equal 'hello', r.read - end - assert_equal 0, logger.messages.length - end - - def test_send_body_proc_chunked - @res.body = Proc.new { |out| out.write('hello') } - @res.chunked = true - IO.pipe do |r, w| - @res.send_body(w) - w.close - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - end - assert_equal 0, logger.messages.length - end - - def test_set_error - status = 400 - message = 'missing attribute' - @res.status = status - error = WEBrick::HTTPStatus[status].new(message) - body = @res.set_error(error) - assert_match(/#{@res.reason_phrase}/, body) - assert_match(/#{message}/, body) - end - - def test_no_extraneous_space - [200, 300, 400, 500].each do |status| - @res.status = status - assert_match(/\S\r\n/, @res.status_line) - end - end - end -end diff --git a/tool/test/webrick/test_https.rb b/tool/test/webrick/test_https.rb deleted file mode 100644 index ec0aac354a..0000000000 --- a/tool/test/webrick/test_https.rb +++ /dev/null @@ -1,112 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require "webrick/https" -require "webrick/utils" -require_relative "utils" - -class TestWEBrickHTTPS < Test::Unit::TestCase - empty_log = Object.new - def empty_log.<<(str) - assert_equal('', str) - self - end - NoLog = WEBrick::Log.new(empty_log, WEBrick::BasicLog::WARN) - - class HTTPSNITest < ::Net::HTTP - attr_accessor :sni_hostname - - def ssl_socket_connect(s, timeout) - s.hostname = sni_hostname - super - end - end - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def https_get(addr, port, hostname, path, verifyname = nil) - subject = nil - http = HTTPSNITest.new(addr, port) - http.use_ssl = true - http.verify_mode = OpenSSL::SSL::VERIFY_NONE - http.verify_callback = proc { |x, store| subject = store.chain[0].subject.to_s; x } - http.sni_hostname = hostname - req = Net::HTTP::Get.new(path) - req["Host"] = "#{hostname}:#{port}" - response = http.start { http.request(req).body } - assert_equal("/CN=#{verifyname || hostname}", subject) - response - end - - def test_sni - config = { - :ServerName => "localhost", - :SSLEnable => true, - :SSLCertName => "/CN=localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc("/") {|req, res| res.body = "master" } - - # catch stderr in create_self_signed_cert - stderr_buffer = StringIO.new - old_stderr, $stderr = $stderr, stderr_buffer - - begin - vhost_config1 = { - :ServerName => "vhost1", - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :SSLEnable => true, - :SSLCertName => "/CN=vhost1", - } - vhost1 = WEBrick::HTTPServer.new(vhost_config1) - vhost1.mount_proc("/") {|req, res| res.body = "vhost1" } - server.virtual_host(vhost1) - - vhost_config2 = { - :ServerName => "vhost2", - :ServerAlias => ["vhost2alias"], - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :SSLEnable => true, - :SSLCertName => "/CN=vhost2", - } - vhost2 = WEBrick::HTTPServer.new(vhost_config2) - vhost2.mount_proc("/") {|req, res| res.body = "vhost2" } - server.virtual_host(vhost2) - ensure - # restore stderr - $stderr = old_stderr - end - - assert_match(/\A([.+*]+\n)+\z/, stderr_buffer.string) - assert_equal("master", https_get(addr, port, "localhost", "/localhost")) - assert_equal("master", https_get(addr, port, "unknown", "/unknown", "localhost")) - assert_equal("vhost1", https_get(addr, port, "vhost1", "/vhost1")) - assert_equal("vhost2", https_get(addr, port, "vhost2", "/vhost2")) - assert_equal("vhost2", https_get(addr, port, "vhost2alias", "/vhost2alias", "vhost2")) - } - end - - def test_check_ssl_virtual - config = { - :ServerName => "localhost", - :SSLEnable => true, - :SSLCertName => "/CN=localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - assert_raise ArgumentError do - vhost = WEBrick::HTTPServer.new({:DoNotListen => true, :Logger => NoLog}) - server.virtual_host(vhost) - end - } - end -end diff --git a/tool/test/webrick/test_httpserver.rb b/tool/test/webrick/test_httpserver.rb deleted file mode 100644 index 4133be85ad..0000000000 --- a/tool/test/webrick/test_httpserver.rb +++ /dev/null @@ -1,543 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require_relative "utils" - -class TestWEBrickHTTPServer < Test::Unit::TestCase - empty_log = Object.new - def empty_log.<<(str) - assert_equal('', str) - self - end - NoLog = WEBrick::Log.new(empty_log, WEBrick::BasicLog::WARN) - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_mount - httpd = WEBrick::HTTPServer.new( - :Logger => NoLog, - :DoNotListen=>true - ) - httpd.mount("/", :Root) - httpd.mount("/foo", :Foo) - httpd.mount("/foo/bar", :Bar, :bar1) - httpd.mount("/foo/bar/baz", :Baz, :baz1, :baz2) - - serv, opts, script_name, path_info = httpd.search_servlet("/") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/sub") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/sub", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/sub/") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/sub/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/sub") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("/sub", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/bar") - assert_equal(:Bar, serv) - assert_equal([:bar1], opts) - assert_equal("/foo/bar", script_name) - assert_equal("", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/bar/baz") - assert_equal(:Baz, serv) - assert_equal([:baz1, :baz2], opts) - assert_equal("/foo/bar/baz", script_name) - assert_equal("", path_info) - end - - class Req - attr_reader :port, :host - def initialize(addr, port, host) - @addr, @port, @host = addr, port, host - end - def addr - [0,0,0,@addr] - end - end - - def httpd(addr, port, host, ali) - config ={ - :Logger => NoLog, - :DoNotListen => true, - :BindAddress => addr, - :Port => port, - :ServerName => host, - :ServerAlias => ali, - } - return WEBrick::HTTPServer.new(config) - end - - def assert_eql?(v1, v2) - assert_equal(v1.object_id, v2.object_id) - end - - def test_lookup_server - addr1 = "192.168.100.1" - addr2 = "192.168.100.2" - addrz = "192.168.100.254" - local = "127.0.0.1" - port1 = 80 - port2 = 8080 - port3 = 10080 - portz = 32767 - name1 = "www.example.com" - name2 = "www2.example.com" - name3 = "www3.example.com" - namea = "www.example.co.jp" - nameb = "www.example.jp" - namec = "www2.example.co.jp" - named = "www2.example.jp" - namez = "foobar.example.com" - alias1 = [namea, nameb] - alias2 = [namec, named] - - host1 = httpd(nil, port1, name1, nil) - hosts = [ - host2 = httpd(addr1, port1, name1, nil), - host3 = httpd(addr1, port1, name2, alias1), - host4 = httpd(addr1, port2, name1, nil), - host5 = httpd(addr1, port2, name2, alias1), - httpd(addr1, port2, name3, alias2), - host7 = httpd(addr2, nil, name1, nil), - host8 = httpd(addr2, nil, name2, alias1), - httpd(addr2, nil, name3, alias2), - host10 = httpd(local, nil, nil, nil), - host11 = httpd(nil, port3, nil, nil), - ].sort_by{ rand } - hosts.each{|h| host1.virtual_host(h) } - - # connect to addr1 - assert_eql?(host2, host1.lookup_server(Req.new(addr1, port1, name1))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, name2))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, namea))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, port1, namez))) - assert_eql?(host4, host1.lookup_server(Req.new(addr1, port2, name1))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, name2))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, namea))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, port2, namez))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, name1))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, name2))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, namea))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, namez))) - - # connect to addr2 - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port1, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, port1, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port2, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, port2, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port3, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addr2, port3, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, portz, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, portz, namez))) - - # connect to addrz - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, namez))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, name1))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, name2))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, namea))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, namez))) - - # connect to localhost - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, namez))) - end - - def test_callbacks - accepted = started = stopped = 0 - requested0 = requested1 = 0 - config = { - :ServerName => "localhost", - :AcceptCallback => Proc.new{ accepted += 1 }, - :StartCallback => Proc.new{ started += 1 }, - :StopCallback => Proc.new{ stopped += 1 }, - :RequestCallback => Proc.new{|req, res| requested0 += 1 }, - } - log_tester = lambda {|log, access_log| - assert(log.find {|s| %r{ERROR `/' not found\.} =~ s }) - assert_equal([], log.reject {|s| %r{ERROR `/' not found\.} =~ s }) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - vhost_config = { - :ServerName => "myhostname", - :BindAddress => addr, - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :RequestCallback => Proc.new{|req, res| requested1 += 1 }, - } - server.virtual_host(WEBrick::HTTPServer.new(vhost_config)) - - Thread.pass while server.status != :Running - sleep 1 if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # server.status behaves unexpectedly with --jit-wait - assert_equal(1, started, log.call) - assert_equal(0, stopped, log.call) - assert_equal(0, accepted, log.call) - - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req["Host"] = "myhostname:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - req["Host"] = "localhost:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - assert_equal(6, accepted, log.call) - assert_equal(3, requested0, log.call) - assert_equal(3, requested1, log.call) - } - assert_equal(started, 1) - assert_equal(stopped, 1) - end - - class CustomRequest < ::WEBrick::HTTPRequest; end - class CustomResponse < ::WEBrick::HTTPResponse; end - class CustomServer < ::WEBrick::HTTPServer - def create_request(config) - CustomRequest.new(config) - end - - def create_response(config) - CustomResponse.new(config) - end - end - - def test_custom_server_request_and_response - config = { :ServerName => "localhost" } - TestWEBrick.start_server(CustomServer, config){|server, addr, port, log| - server.mount_proc("/", lambda {|req, res| - assert_kind_of(CustomRequest, req) - assert_kind_of(CustomResponse, res) - res.body = "via custom response" - }) - Thread.pass while server.status != :Running - - Net::HTTP.start(addr, port) do |http| - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("via custom response", res.body) - } - server.shutdown - end - } - end - - # This class is needed by test_response_io_with_chunked_set method - class EventManagerForChunkedResponseTest - def initialize - @listeners = [] - end - def add_listener( &block ) - @listeners << block - end - def raise_str_event( str ) - @listeners.each{ |e| e.call( :str, str ) } - end - def raise_close_event() - @listeners.each{ |e| e.call( :cls ) } - end - end - def test_response_io_with_chunked_set - evt_man = EventManagerForChunkedResponseTest.new - t = Thread.new do - begin - config = { - :ServerName => "localhost" - } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - body_strs = [ 'aaaaaa', 'bb', 'cccc' ] - server.mount_proc( "/", ->( req, res ){ - # Test for setting chunked... - res.chunked = true - r,w = IO.pipe - evt_man.add_listener do |type,str| - type == :cls ? ( w.close ) : ( w << str ) - end - res.body = r - } ) - Thread.pass while server.status != :Running - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - http.request(req) do |res| - i = 0 - evt_man.raise_str_event( body_strs[i] ) - res.read_body do |s| - assert_equal( body_strs[i], s ) - i += 1 - if i < body_strs.length - evt_man.raise_str_event( body_strs[i] ) - else - evt_man.raise_close_event() - end - end - assert_equal( body_strs.length, i ) - end - end - rescue => err - flunk( 'exception raised in thread: ' + err.to_s ) - end - end - if t.join( 3 ).nil? - evt_man.raise_close_event() - flunk( 'timeout' ) - if t.join( 1 ).nil? - Thread.kill t - end - end - end - - def test_response_io_without_chunked_set - config = { - :ServerName => "localhost" - } - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/WARN Could not determine content-length of response body./, log[0]) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - server.mount_proc("/", lambda { |req, res| - r,w = IO.pipe - # Test for not setting chunked... - # res.chunked = true - res.body = r - w << "foo" - w.close - }) - Thread.pass while server.status != :Running - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req['Connection'] = 'Keep-Alive' - begin - Timeout.timeout(2) do - http.request(req){|res| assert_equal("foo", res.body) } - end - rescue Timeout::Error - flunk('corrupted response') - end - } - end - - def test_request_handler_callback_is_deprecated - requested = 0 - config = { - :ServerName => "localhost", - :RequestHandler => Proc.new{|req, res| requested += 1 }, - } - log_tester = lambda {|log, access_log| - assert_equal(2, log.length) - assert_match(/WARN :RequestHandler is deprecated, please use :RequestCallback/, log[0]) - assert_match(%r{ERROR `/' not found\.}, log[1]) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - Thread.pass while server.status != :Running - - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req["Host"] = "localhost:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - assert_match(%r{:RequestHandler is deprecated, please use :RequestCallback$}, log.call, log.call) - } - assert_equal(1, requested) - end - - def test_shutdown_with_busy_keepalive_connection - requested = 0 - config = { - :ServerName => "localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc("/", lambda {|req, res| res.body = "heffalump" }) - Thread.pass while server.status != :Running - - Net::HTTP.start(addr, port) do |http| - req = Net::HTTP::Get.new("/") - http.request(req){|res| assert_equal('Keep-Alive', res['Connection'], log.call) } - server.shutdown - begin - 10.times {|n| http.request(req); requested += 1 } - rescue - # Errno::ECONNREFUSED or similar - end - end - } - assert_equal(0, requested, "Server responded to #{requested} requests after shutdown") - end - - def test_cntrl_in_path - log_ary = [] - access_log_ary = [] - config = { - :Port => 0, - :BindAddress => '127.0.0.1', - :Logger => WEBrick::Log.new(log_ary, WEBrick::BasicLog::WARN), - :AccessLog => [[access_log_ary, '']], - } - s = WEBrick::HTTPServer.new(config) - s.mount('/foo', WEBrick::HTTPServlet::FileHandler, __FILE__) - th = Thread.new { s.start } - addr = s.listeners[0].addr - - http = Net::HTTP.new(addr[3], addr[1]) - req = Net::HTTP::Get.new('/notexist%0a/foo') - http.request(req) { |res| assert_equal('404', res.code) } - exp = %Q(ERROR `/notexist\\n/foo' not found.\n) - assert_equal 1, log_ary.size - assert_include log_ary[0], exp - ensure - s&.shutdown - th&.join - end - - def test_gigantic_request_header - log_tester = lambda {|log, access_log| - assert_equal 1, log.size - assert_include log[0], 'ERROR headers too large' - } - TestWEBrick.start_httpserver({}, log_tester){|server, addr, port, log| - server.mount('/', WEBrick::HTTPServlet::FileHandler, __FILE__) - TCPSocket.open(addr, port) do |c| - c.write("GET / HTTP/1.0\r\n") - junk = -"X-Junk: #{' ' * 1024}\r\n" - assert_raise(Errno::ECONNRESET, Errno::EPIPE, Errno::EPROTOTYPE) do - loop { c.write(junk) } - end - end - } - end - - def test_eof_in_chunk - log_tester = lambda do |log, access_log| - assert_equal 1, log.size - assert_include log[0], 'ERROR bad chunk data size' - end - TestWEBrick.start_httpserver({}, log_tester){|server, addr, port, log| - server.mount_proc('/', ->(req, res) { res.body = req.body }) - TCPSocket.open(addr, port) do |c| - c.write("POST / HTTP/1.1\r\nHost: example.com\r\n" \ - "Transfer-Encoding: chunked\r\n\r\n5\r\na") - c.shutdown(Socket::SHUT_WR) # trigger EOF in server - res = c.read - assert_match %r{\AHTTP/1\.1 400 }, res - end - } - end - - def test_big_chunks - nr_out = 3 - buf = 'big' # 3 bytes is bigger than 2! - config = { :InputBufferSize => 2 }.freeze - total = 0 - all = '' - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc('/', ->(req, res) { - err = [] - ret = req.body do |chunk| - n = chunk.bytesize - n > config[:InputBufferSize] and err << "#{n} > :InputBufferSize" - total += n - all << chunk - end - ret.nil? or err << 'req.body should return nil' - (buf * nr_out) == all or err << 'input body does not match expected' - res.header['connection'] = 'close' - res.body = err.join("\n") - }) - TCPSocket.open(addr, port) do |c| - c.write("POST / HTTP/1.1\r\nHost: example.com\r\n" \ - "Transfer-Encoding: chunked\r\n\r\n") - chunk = "#{buf.bytesize.to_s(16)}\r\n#{buf}\r\n" - nr_out.times { c.write(chunk) } - c.write("0\r\n\r\n") - head, body = c.read.split("\r\n\r\n") - assert_match %r{\AHTTP/1\.1 200 OK}, head - assert_nil body - end - } - end -end diff --git a/tool/test/webrick/test_httpstatus.rb b/tool/test/webrick/test_httpstatus.rb deleted file mode 100644 index fd0570d5c6..0000000000 --- a/tool/test/webrick/test_httpstatus.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick" - -class TestWEBrickHTTPStatus < Test::Unit::TestCase - def test_info? - assert WEBrick::HTTPStatus.info?(100) - refute WEBrick::HTTPStatus.info?(200) - end - - def test_success? - assert WEBrick::HTTPStatus.success?(200) - refute WEBrick::HTTPStatus.success?(300) - end - - def test_redirect? - assert WEBrick::HTTPStatus.redirect?(300) - refute WEBrick::HTTPStatus.redirect?(400) - end - - def test_error? - assert WEBrick::HTTPStatus.error?(400) - refute WEBrick::HTTPStatus.error?(600) - end - - def test_client_error? - assert WEBrick::HTTPStatus.client_error?(400) - refute WEBrick::HTTPStatus.client_error?(500) - end - - def test_server_error? - assert WEBrick::HTTPStatus.server_error?(500) - refute WEBrick::HTTPStatus.server_error?(600) - end -end diff --git a/tool/test/webrick/test_httputils.rb b/tool/test/webrick/test_httputils.rb deleted file mode 100644 index 00f297bd09..0000000000 --- a/tool/test/webrick/test_httputils.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/httputils" - -class TestWEBrickHTTPUtils < Test::Unit::TestCase - include WEBrick::HTTPUtils - - def test_normilize_path - assert_equal("/foo", normalize_path("/foo")) - assert_equal("/foo/bar/", normalize_path("/foo/bar/")) - - assert_equal("/", normalize_path("/foo/../")) - assert_equal("/", normalize_path("/foo/..")) - assert_equal("/", normalize_path("/foo/bar/../../")) - assert_equal("/", normalize_path("/foo/bar/../..")) - assert_equal("/", normalize_path("/foo/bar/../..")) - assert_equal("/baz", normalize_path("/foo/bar/../../baz")) - assert_equal("/baz", normalize_path("/foo/../bar/../baz")) - assert_equal("/baz/", normalize_path("/foo/../bar/../baz/")) - assert_equal("/...", normalize_path("/bar/../...")) - assert_equal("/.../", normalize_path("/bar/../.../")) - - assert_equal("/foo/", normalize_path("/foo/./")) - assert_equal("/foo/", normalize_path("/foo/.")) - assert_equal("/foo/", normalize_path("/foo/././")) - assert_equal("/foo/", normalize_path("/foo/./.")) - assert_equal("/foo/bar", normalize_path("/foo/./bar")) - assert_equal("/foo/bar/", normalize_path("/foo/./bar/.")) - assert_equal("/foo/bar/", normalize_path("/./././foo/./bar/.")) - - assert_equal("/foo/bar/", normalize_path("//foo///.//bar/.///.//")) - assert_equal("/", normalize_path("//foo///..///bar/.///..//.//")) - - assert_raise(RuntimeError){ normalize_path("foo/bar") } - assert_raise(RuntimeError){ normalize_path("..") } - assert_raise(RuntimeError){ normalize_path("/..") } - assert_raise(RuntimeError){ normalize_path("/./..") } - assert_raise(RuntimeError){ normalize_path("/./../") } - assert_raise(RuntimeError){ normalize_path("/./../..") } - assert_raise(RuntimeError){ normalize_path("/./../../") } - assert_raise(RuntimeError){ normalize_path("/./../") } - assert_raise(RuntimeError){ normalize_path("/../..") } - assert_raise(RuntimeError){ normalize_path("/../../") } - assert_raise(RuntimeError){ normalize_path("/../../..") } - assert_raise(RuntimeError){ normalize_path("/../../../") } - assert_raise(RuntimeError){ normalize_path("/../foo/../") } - assert_raise(RuntimeError){ normalize_path("/../foo/../../") } - assert_raise(RuntimeError){ normalize_path("/foo/bar/../../../../") } - assert_raise(RuntimeError){ normalize_path("/foo/../bar/../../") } - assert_raise(RuntimeError){ normalize_path("/./../bar/") } - assert_raise(RuntimeError){ normalize_path("/./../") } - end - - def test_split_header_value - assert_equal(['foo', 'bar'], split_header_value('foo, bar')) - assert_equal(['"foo"', 'bar'], split_header_value('"foo", bar')) - assert_equal(['foo', '"bar"'], split_header_value('foo, "bar"')) - assert_equal(['*'], split_header_value('*')) - assert_equal(['W/"xyzzy"', 'W/"r2d2xxxx"', 'W/"c3piozzzz"'], - split_header_value('W/"xyzzy", W/"r2d2xxxx", W/"c3piozzzz"')) - end - - def test_escape - assert_equal("/foo/bar", escape("/foo/bar")) - assert_equal("/~foo/bar", escape("/~foo/bar")) - assert_equal("/~foo%20bar", escape("/~foo bar")) - assert_equal("/~foo%20bar", escape("/~foo bar")) - assert_equal("/~foo%09bar", escape("/~foo\tbar")) - assert_equal("/~foo+bar", escape("/~foo+bar")) - bug8425 = '[Bug #8425] [ruby-core:55052]' - assert_nothing_raised(ArgumentError, Encoding::CompatibilityError, bug8425) { - assert_equal("%E3%83%AB%E3%83%93%E3%83%BC%E3%81%95%E3%82%93", escape("\u{30EB 30D3 30FC 3055 3093}")) - } - end - - def test_escape_form - assert_equal("%2Ffoo%2Fbar", escape_form("/foo/bar")) - assert_equal("%2F~foo%2Fbar", escape_form("/~foo/bar")) - assert_equal("%2F~foo+bar", escape_form("/~foo bar")) - assert_equal("%2F~foo+%2B+bar", escape_form("/~foo + bar")) - end - - def test_unescape - assert_equal("/foo/bar", unescape("%2ffoo%2fbar")) - assert_equal("/~foo/bar", unescape("/%7efoo/bar")) - assert_equal("/~foo/bar", unescape("%2f%7efoo%2fbar")) - assert_equal("/~foo+bar", unescape("/%7efoo+bar")) - end - - def test_unescape_form - assert_equal("//foo/bar", unescape_form("/%2Ffoo/bar")) - assert_equal("//foo/bar baz", unescape_form("/%2Ffoo/bar+baz")) - assert_equal("/~foo/bar baz", unescape_form("/%7Efoo/bar+baz")) - end - - def test_escape_path - assert_equal("/foo/bar", escape_path("/foo/bar")) - assert_equal("/foo/bar/", escape_path("/foo/bar/")) - assert_equal("/%25foo/bar/", escape_path("/%foo/bar/")) - end -end diff --git a/tool/test/webrick/test_httpversion.rb b/tool/test/webrick/test_httpversion.rb deleted file mode 100644 index e50ee17971..0000000000 --- a/tool/test/webrick/test_httpversion.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/httpversion" - -class TestWEBrickHTTPVersion < Test::Unit::TestCase - def setup - @v09 = WEBrick::HTTPVersion.new("0.9") - @v10 = WEBrick::HTTPVersion.new("1.0") - @v11 = WEBrick::HTTPVersion.new("1.001") - end - - def test_to_s() - assert_equal("0.9", @v09.to_s) - assert_equal("1.0", @v10.to_s) - assert_equal("1.1", @v11.to_s) - end - - def test_major() - assert_equal(0, @v09.major) - assert_equal(1, @v10.major) - assert_equal(1, @v11.major) - end - - def test_minor() - assert_equal(9, @v09.minor) - assert_equal(0, @v10.minor) - assert_equal(1, @v11.minor) - end - - def test_compar() - assert_equal(0, @v09 <=> "0.9") - assert_equal(0, @v09 <=> "0.09") - - assert_equal(-1, @v09 <=> @v10) - assert_equal(-1, @v09 <=> "1.00") - - assert_equal(1, @v11 <=> @v09) - assert_equal(1, @v11 <=> "1.0") - assert_equal(1, @v11 <=> "0.9") - end -end diff --git a/tool/test/webrick/test_server.rb b/tool/test/webrick/test_server.rb deleted file mode 100644 index 815cc3ce39..0000000000 --- a/tool/test/webrick/test_server.rb +++ /dev/null @@ -1,191 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "tempfile" -require "webrick" -require_relative "utils" - -class TestWEBrickServer < Test::Unit::TestCase - class Echo < WEBrick::GenericServer - def run(sock) - while line = sock.gets - sock << line - end - end - end - - def test_server - TestWEBrick.start_server(Echo){|server, addr, port, log| - TCPSocket.open(addr, port){|sock| - sock.puts("foo"); assert_equal("foo\n", sock.gets, log.call) - sock.puts("bar"); assert_equal("bar\n", sock.gets, log.call) - sock.puts("baz"); assert_equal("baz\n", sock.gets, log.call) - sock.puts("qux"); assert_equal("qux\n", sock.gets, log.call) - } - } - end - - def test_start_exception - stopped = 0 - - log = [] - logger = WEBrick::Log.new(log, WEBrick::BasicLog::WARN) - - assert_raise(SignalException) do - listener = Object.new - def listener.to_io # IO.select invokes #to_io. - raise SignalException, 'SIGTERM' # simulate signal in main thread - end - def listener.shutdown - end - def listener.close - end - - server = WEBrick::HTTPServer.new({ - :BindAddress => "127.0.0.1", :Port => 0, - :StopCallback => Proc.new{ stopped += 1 }, - :Logger => logger, - }) - server.listeners[0].close - server.listeners[0] = listener - - server.start - end - - assert_equal(1, stopped) - assert_equal(1, log.length) - assert_match(/FATAL SignalException: SIGTERM/, log[0]) - end - - def test_callbacks - accepted = started = stopped = 0 - config = { - :AcceptCallback => Proc.new{ accepted += 1 }, - :StartCallback => Proc.new{ started += 1 }, - :StopCallback => Proc.new{ stopped += 1 }, - } - TestWEBrick.start_server(Echo, config){|server, addr, port, log| - true while server.status != :Running - sleep 1 if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # server.status behaves unexpectedly with --jit-wait - assert_equal(1, started, log.call) - assert_equal(0, stopped, log.call) - assert_equal(0, accepted, log.call) - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - assert_equal(3, accepted, log.call) - } - assert_equal(1, started) - assert_equal(1, stopped) - end - - def test_daemon - begin - r, w = IO.pipe - pid1 = Process.fork{ - r.close - WEBrick::Daemon.start - w.puts(Process.pid) - sleep 10 - } - pid2 = r.gets.to_i - assert(Process.kill(:KILL, pid2)) - assert_not_equal(pid1, pid2) - rescue NotImplementedError - # snip this test - ensure - Process.wait(pid1) if pid1 - r.close - w.close - end - end - - def test_restart_after_shutdown - address = '127.0.0.1' - port = 0 - log = [] - config = { - :BindAddress => address, - :Port => port, - :Logger => WEBrick::Log.new(log, WEBrick::BasicLog::WARN), - } - server = Echo.new(config) - client_proc = lambda {|str| - begin - ret = server.listeners.first.connect_address.connect {|s| - s.write(str) - s.close_write - s.read - } - assert_equal(str, ret) - ensure - server.shutdown - end - } - server_thread = Thread.new { server.start } - client_thread = Thread.new { client_proc.call("a") } - assert_join_threads([client_thread, server_thread]) - server.listen(address, port) - server_thread = Thread.new { server.start } - client_thread = Thread.new { client_proc.call("b") } - assert_join_threads([client_thread, server_thread]) - assert_equal([], log) - end - - def test_restart_after_stop - log = Object.new - class << log - include Test::Unit::Assertions - def <<(msg) - flunk "unexpected log: #{msg.inspect}" - end - end - client_thread = nil - wakeup = -> {client_thread.wakeup} - warn_flunk = WEBrick::Log.new(log, WEBrick::BasicLog::WARN) - server = WEBrick::HTTPServer.new( - :StartCallback => wakeup, - :StopCallback => wakeup, - :BindAddress => '0.0.0.0', - :Port => 0, - :Logger => warn_flunk) - 2.times { - server_thread = Thread.start { - server.start - } - client_thread = Thread.start { - sleep 0.1 until server.status == :Running || !server_thread.status - server.stop - sleep 0.1 until server.status == :Stop || !server_thread.status - } - assert_join_threads([client_thread, server_thread]) - } - end - - def test_port_numbers - config = { - :BindAddress => '0.0.0.0', - :Logger => WEBrick::Log.new([], WEBrick::BasicLog::WARN), - } - - ports = [0, "0"] - - ports.each do |port| - config[:Port]= port - server = WEBrick::GenericServer.new(config) - server_thread = Thread.start { server.start } - client_thread = Thread.start { - sleep 0.1 until server.status == :Running || !server_thread.status - server_port = server.listeners[0].addr[1] - server.stop - assert_equal server.config[:Port], server_port - sleep 0.1 until server.status == :Stop || !server_thread.status - } - assert_join_threads([client_thread, server_thread]) - end - - assert_raise(ArgumentError) do - config[:Port]= "FOO" - WEBrick::GenericServer.new(config) - end - end -end diff --git a/tool/test/webrick/test_ssl_server.rb b/tool/test/webrick/test_ssl_server.rb deleted file mode 100644 index 4e52598bf5..0000000000 --- a/tool/test/webrick/test_ssl_server.rb +++ /dev/null @@ -1,67 +0,0 @@ -require "test/unit" -require "webrick" -require "webrick/ssl" -require_relative "utils" -require 'timeout' - -class TestWEBrickSSLServer < Test::Unit::TestCase - class Echo < WEBrick::GenericServer - def run(sock) - while line = sock.gets - sock << line - end - end - end - - def test_self_signed_cert_server - assert_self_signed_cert( - :SSLEnable => true, - :SSLCertName => [["C", "JP"], ["O", "www.ruby-lang.org"], ["CN", "Ruby"]], - ) - end - - def test_self_signed_cert_server_with_string - assert_self_signed_cert( - :SSLEnable => true, - :SSLCertName => "/C=JP/O=www.ruby-lang.org/CN=Ruby", - ) - end - - def assert_self_signed_cert(config) - TestWEBrick.start_server(Echo, config){|server, addr, port, log| - io = TCPSocket.new(addr, port) - sock = OpenSSL::SSL::SSLSocket.new(io) - sock.connect - sock.puts(server.ssl_context.cert.subject.to_s) - assert_equal("/C=JP/O=www.ruby-lang.org/CN=Ruby\n", sock.gets, log.call) - sock.close - io.close - } - end - - def test_slow_connect - poke = lambda do |io, msg| - begin - sock = OpenSSL::SSL::SSLSocket.new(io) - sock.connect - sock.puts(msg) - assert_equal "#{msg}\n", sock.gets, msg - ensure - sock&.close - io.close - end - end - config = { - :SSLEnable => true, - :SSLCertName => "/C=JP/O=www.ruby-lang.org/CN=Ruby", - } - EnvUtil.timeout(10) do - TestWEBrick.start_server(Echo, config) do |server, addr, port, log| - outer = TCPSocket.new(addr, port) - inner = TCPSocket.new(addr, port) - poke.call(inner, 'fast TLS negotiation') - poke.call(outer, 'slow TLS negotiation') - end - end - end -end diff --git a/tool/test/webrick/test_utils.rb b/tool/test/webrick/test_utils.rb deleted file mode 100644 index c2b7a36e8a..0000000000 --- a/tool/test/webrick/test_utils.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/utils" - -class TestWEBrickUtils < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def assert_expired(m) - Thread.handle_interrupt(Timeout::Error => :never, EX => :never) do - assert_empty(m::TimeoutHandler.instance.instance_variable_get(:@timeout_info)) - end - end - - def assert_not_expired(m) - Thread.handle_interrupt(Timeout::Error => :never, EX => :never) do - assert_not_empty(m::TimeoutHandler.instance.instance_variable_get(:@timeout_info)) - end - end - - EX = Class.new(StandardError) - - def test_no_timeout - m = WEBrick::Utils - assert_equal(:foo, m.timeout(10){ :foo }) - assert_expired(m) - end - - def test_nested_timeout_outer - m = WEBrick::Utils - i = 0 - assert_raise(Timeout::Error){ - m.timeout(1){ - assert_raise(Timeout::Error){ m.timeout(0.1){ i += 1; sleep(1) } } - assert_not_expired(m) - i += 1 - sleep(2) - } - } - assert_equal(2, i) - assert_expired(m) - end - - def test_timeout_default_exception - m = WEBrick::Utils - assert_raise(Timeout::Error){ m.timeout(0.01){ sleep } } - assert_expired(m) - end - - def test_timeout_custom_exception - m = WEBrick::Utils - ex = EX - assert_raise(ex){ m.timeout(0.01, ex){ sleep } } - assert_expired(m) - end - - def test_nested_timeout_inner_custom_exception - m = WEBrick::Utils - ex = EX - i = 0 - assert_raise(ex){ - m.timeout(10){ - m.timeout(0.01, ex){ i += 1; sleep } - } - sleep - } - assert_equal(1, i) - assert_expired(m) - end - - def test_nested_timeout_outer_custom_exception - m = WEBrick::Utils - ex = EX - i = 0 - assert_raise(Timeout::Error){ - m.timeout(0.01){ - m.timeout(1.0, ex){ i += 1; sleep } - } - sleep - } - assert_equal(1, i) - assert_expired(m) - end - - def test_create_listeners - addr = listener_address(0) - port = addr.slice!(1) - assert_kind_of(Integer, port, "dynamically chosen port number") - assert_equal(["AF_INET", "127.0.0.1", "127.0.0.1"], addr) - - assert_equal(["AF_INET", port, "127.0.0.1", "127.0.0.1"], - listener_address(port), - "specific port number") - - assert_equal(["AF_INET", port, "127.0.0.1", "127.0.0.1"], - listener_address(port.to_s), - "specific port number string") - end - - def listener_address(port) - listeners = WEBrick::Utils.create_listeners("127.0.0.1", port) - srv = listeners.first - assert_kind_of TCPServer, srv - srv.addr - ensure - listeners.each(&:close) if listeners - end -end diff --git a/tool/test/webrick/utils.rb b/tool/test/webrick/utils.rb deleted file mode 100644 index a8568d0a43..0000000000 --- a/tool/test/webrick/utils.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: false -require "webrick" -begin - require "webrick/https" -rescue LoadError -end -require "webrick/httpproxy" - -module TestWEBrick - NullWriter = Object.new - def NullWriter.<<(msg) - puts msg if $DEBUG - return self - end - - class WEBrick::HTTPServlet::CGIHandler - remove_const :Ruby - require "envutil" unless defined?(EnvUtil) - Ruby = EnvUtil.rubybin - remove_const :CGIRunner - CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: - remove_const :CGIRunnerArray - CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb"] # :nodoc: - end - - RubyBin = "\"#{EnvUtil.rubybin}\"" - RubyBin << " --disable-gems" - RubyBin << " \"-I#{File.expand_path("../..", File.dirname(__FILE__))}/lib\"" - RubyBin << " \"-I#{File.dirname(EnvUtil.rubybin)}/.ext/common\"" - RubyBin << " \"-I#{File.dirname(EnvUtil.rubybin)}/.ext/#{RUBY_PLATFORM}\"" - - RubyBinArray = [EnvUtil.rubybin] - RubyBinArray << "--disable-gems" - RubyBinArray << "-I" << "#{File.expand_path("../..", File.dirname(__FILE__))}/lib" - RubyBinArray << "-I" << "#{File.dirname(EnvUtil.rubybin)}/.ext/common" - RubyBinArray << "-I" << "#{File.dirname(EnvUtil.rubybin)}/.ext/#{RUBY_PLATFORM}" - - require "test/unit" unless defined?(Test::Unit) - include Test::Unit::Assertions - extend Test::Unit::Assertions - include Test::Unit::CoreAssertions - extend Test::Unit::CoreAssertions - - module_function - - DefaultLogTester = lambda {|log, access_log| assert_equal([], log) } - - def start_server(klass, config={}, log_tester=DefaultLogTester, &block) - log_ary = [] - access_log_ary = [] - log = proc { "webrick log start:\n" + (log_ary+access_log_ary).join.gsub(/^/, " ").chomp + "\nwebrick log end" } - config = ({ - :BindAddress => "127.0.0.1", :Port => 0, - :ServerType => Thread, - :Logger => WEBrick::Log.new(log_ary, WEBrick::BasicLog::WARN), - :AccessLog => [[access_log_ary, ""]] - }.update(config)) - server = capture_output {break klass.new(config)} - server_thread = server.start - server_thread2 = Thread.new { - server_thread.join - if log_tester - log_tester.call(log_ary, access_log_ary) - end - } - addr = server.listeners[0].addr - client_thread = Thread.new { - begin - block.yield([server, addr[3], addr[1], log]) - ensure - server.shutdown - end - } - assert_join_threads([client_thread, server_thread2]) - end - - def start_httpserver(config={}, log_tester=DefaultLogTester, &block) - start_server(WEBrick::HTTPServer, config, log_tester, &block) - end - - def start_httpproxy(config={}, log_tester=DefaultLogTester, &block) - start_server(WEBrick::HTTPProxyServer, config, log_tester, &block) - end -end diff --git a/tool/test/webrick/webrick.cgi b/tool/test/webrick/webrick.cgi deleted file mode 100644 index a294fa72f9..0000000000 --- a/tool/test/webrick/webrick.cgi +++ /dev/null @@ -1,38 +0,0 @@ -#!ruby -require "webrick/cgi" - -class TestApp < WEBrick::CGI - def do_GET(req, res) - res["content-type"] = "text/plain" - if req.path_info == "/dumpenv" - res.body = Marshal.dump(ENV.to_hash) - elsif (p = req.path_info) && p.length > 0 - res.body = p - elsif (q = req.query).size > 0 - res.body = q.keys.sort.collect{|key| - q[key].list.sort.collect{|v| - "#{key}=#{v}" - }.join(", ") - }.join(", ") - elsif %r{/$} =~ req.request_uri.to_s - res.body = "" - res.body << req.request_uri.to_s << "\n" - res.body << req.script_name - elsif !req.cookies.empty? - res.body = req.cookies.inject(""){|result, cookie| - result << "%s=%s\n" % [cookie.name, cookie.value] - } - res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") - res.cookies << WEBrick::Cookie.new("Shipping", "FedEx") - else - res.body = req.script_name - end - end - - def do_POST(req, res) - do_GET(req, res) - end -end - -cgi = TestApp.new -cgi.start diff --git a/tool/test/webrick/webrick.rhtml b/tool/test/webrick/webrick.rhtml deleted file mode 100644 index a7bbe43fb5..0000000000 --- a/tool/test/webrick/webrick.rhtml +++ /dev/null @@ -1,4 +0,0 @@ -req to <%= -servlet_request.request_uri -%> <%= -servlet_request.query.inspect %> diff --git a/tool/test/webrick/webrick_long_filename.cgi b/tool/test/webrick/webrick_long_filename.cgi deleted file mode 100644 index 43c1af825c..0000000000 --- a/tool/test/webrick/webrick_long_filename.cgi +++ /dev/null @@ -1,36 +0,0 @@ -#!ruby -require "webrick/cgi" - -class TestApp < WEBrick::CGI - def do_GET(req, res) - res["content-type"] = "text/plain" - if (p = req.path_info) && p.length > 0 - res.body = p - elsif (q = req.query).size > 0 - res.body = q.keys.sort.collect{|key| - q[key].list.sort.collect{|v| - "#{key}=#{v}" - }.join(", ") - }.join(", ") - elsif %r{/$} =~ req.request_uri.to_s - res.body = "" - res.body << req.request_uri.to_s << "\n" - res.body << req.script_name - elsif !req.cookies.empty? - res.body = req.cookies.inject(""){|result, cookie| - result << "%s=%s\n" % [cookie.name, cookie.value] - } - res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") - res.cookies << WEBrick::Cookie.new("Shipping", "FedEx") - else - res.body = req.script_name - end - end - - def do_POST(req, res) - do_GET(req, res) - end -end - -cgi = TestApp.new -cgi.start diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb index b19f68bac4..1257a92d38 100644 --- a/tool/transcode-tblgen.rb +++ b/tool/transcode-tblgen.rb @@ -1078,11 +1078,7 @@ if __FILE__ == $0 end libs1 = $".dup - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%') - else - erb = ERB.new(src, nil, '%') - end + erb = ERB.new(src, trim_mode: '%') erb.filename = arg erb_result = erb.result(binding) libs2 = $".dup diff --git a/tool/transform_mjit_header.rb b/tool/transform_mjit_header.rb deleted file mode 100644 index 2180236824..0000000000 --- a/tool/transform_mjit_header.rb +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright (C) 2017 Vladimir Makarov, <vmakarov@redhat.com> -# This is a script to transform functions to static inline. -# Usage: transform_mjit_header.rb <c-compiler> <header file> <out> - -require 'fileutils' -require 'tempfile' - -PROGRAM = File.basename($0, ".*") - -module MJITHeader - ATTR_VALUE_REGEXP = /[^()]|\([^()]*\)/ - ATTR_REGEXP = /__attribute__\s*\(\(#{ATTR_VALUE_REGEXP}*\)\)/ - # Example: - # VALUE foo(int bar) - # VALUE __attribute__ ((foo)) bar(int baz) - # __attribute__ ((foo)) VALUE bar(int baz) - FUNC_HEADER_REGEXP = /\A[^\[{(]*(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/ - TARGET_NAME_REGEXP = /\A(rb|ruby|vm|insn|attr|Init)_/ - - # Predefined macros for compilers which are already supported by MJIT. - # We're going to support cl.exe too (WIP) but `cl.exe -E` can't produce macro. - SUPPORTED_CC_MACROS = [ - '__GNUC__', # gcc - '__clang__', # clang - ] - - # These macros are relied on this script's transformation - PREFIXED_MACROS = [ - 'ALWAYS_INLINE', - 'COLDFUNC', - 'inline', - 'RBIMPL_ATTR_COLD', - ] - - # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times. - RECURSIVE_MACROS = %w[ - RASCTRYINFO - RASIPADDR - ] - - IGNORED_FUNCTIONS = [ - 'rb_vm_search_method_slowpath', # This increases the time to compile when inlined. So we use it as external function. - 'rb_equal_opt', # Not used from VM and not compilable - 'ruby_abi_version', - ] - - ALWAYS_INLINED_FUNCTIONS = [ - 'vm_opt_plus', - 'vm_opt_minus', - 'vm_opt_mult', - 'vm_opt_div', - 'vm_opt_mod', - 'vm_opt_neq', - 'vm_opt_lt', - 'vm_opt_le', - 'vm_opt_gt', - 'vm_opt_ge', - 'vm_opt_ltlt', - 'vm_opt_and', - 'vm_opt_or', - 'vm_opt_aref', - 'vm_opt_aset', - 'vm_opt_aref_with', - 'vm_opt_aset_with', - 'vm_opt_not', - ] - - COLD_FUNCTIONS = %w[ - setup_parameters_complex - vm_call_iseq_setup - vm_call_iseq_setup_2 - vm_call_iseq_setup_tailcall - vm_call_method_each_type - vm_ic_update - ] - - # Return start..stop of last decl in CODE ending STOP - def self.find_decl(code, stop) - level = 0 - i = stop - while i = code.rindex(/[;{}]/, i) - if level == 0 && stop != i && decl_found?($&, i) - return decl_start($&, i)..stop - end - case $& - when '}' - level += 1 - when '{' - level -= 1 - end - i -= 1 - end - nil - end - - def self.decl_found?(code, i) - i == 0 || code == ';' || code == '}' - end - - def self.decl_start(code, i) - if i == 0 && code != ';' && code != '}' - 0 - else - i + 1 - end - end - - # Given DECL return the name of it, nil if failed - def self.decl_name_of(decl) - ident_regex = /\w+/ - decl = decl.gsub(/^#.+$/, '') # remove macros - reduced_decl = decl.gsub(ATTR_REGEXP, '') # remove attributes - su1_regex = /{[^{}]*}/ - su2_regex = /{([^{}]|#{su1_regex})*}/ - su3_regex = /{([^{}]|#{su2_regex})*}/ # 3 nested structs/unions is probably enough - reduced_decl.gsub!(su3_regex, '') # remove structs/unions in the header - id_seq_regex = /\s*(?:#{ident_regex}(?:\s+|\s*[*]+\s*))*/ - # Process function header: - match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl) - return match[:name] if match - # Process non-function declaration: - reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization - match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl); - return match[:name] if match - nil - end - - # Return true if CC with CFLAGS compiles successfully the current code. - # Use STAGE in the message in case of a compilation failure - def self.check_code!(code, cc, cflags, stage) - with_code(code) do |path| - cmd = "#{cc} #{cflags} #{path}" - out = IO.popen(cmd, err: [:child, :out], &:read) - unless $?.success? - STDERR.puts "error in #{stage} header file:\n#{out}" - exit false - end - end - end - - # Remove unpreprocessable macros - def self.remove_harmful_macros!(code) - code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '') - end - - # -dD outputs those macros, and it produces redefinition warnings or errors - # This assumes common.mk passes `-DMJIT_HEADER` first when it creates rb_mjit_header.h. - def self.remove_predefined_macros!(code) - code.sub!(/\A(#define [^\n]+|\n)*(#define MJIT_HEADER 1\n)/, '\2') - end - - # Return [macro, others]. But others include PREFIXED_MACROS to be used in code. - def self.separate_macro_and_code(code) - code.lines.partition do |l| - l.start_with?('#') && PREFIXED_MACROS.all? { |m| !l.start_with?("#define #{m}") } - end.map! { |lines| lines.join('') } - end - - def self.write(code, out) - # create with strict permission, then will install proper - # permission - FileUtils.mkdir_p(File.dirname(out), mode: 0700) - File.binwrite("#{out}.new", code, perm: 0600) - FileUtils.mv("#{out}.new", out) - end - - # Note that this checks runruby. This conservatively covers platform names. - def self.windows? - RUBY_PLATFORM =~ /mswin|mingw|msys/ - end - - # If code has macro which only supported compilers predefine, return true. - def self.supported_header?(code) - SUPPORTED_CC_MACROS.any? { |macro| code =~ /^#\s*define\s+#{Regexp.escape(macro)}\b/ } - end - - # This checks if syntax check outputs one of the following messages. - # "error: conflicting types for 'restrict'" - # "error: redefinition of parameter 'restrict'" - # If it's true, this script regards platform as AIX or Solaris and adds -std=c99 as workaround. - def self.conflicting_types?(code, cc, cflags) - with_code(code) do |path| - cmd = "#{cc} #{cflags} #{path}" - out = IO.popen(cmd, err: [:child, :out], &:read) - !$?.success? && - (out.match?(/error: conflicting types for '[^']+'/) || - out.match?(/error: redefinition of parameter '[^']+'/)) - end - end - - def self.with_code(code) - # for `system_header` pragma which can't be in the main file. - Tempfile.open(['', '.h'], mode: File::BINARY) do |f| - f.puts code - f.close - Tempfile.open(['', '.c'], mode: File::BINARY) do |c| - c.puts <<SRC -#include "#{f.path}" -SRC - c.close - return yield(c.path) - end - end - end - private_class_method :with_code -end - -if ARGV.size != 3 - abort "Usage: #{$0} <c-compiler> <header file> <out>" -end - -if STDOUT.tty? - require_relative 'lib/colorize' - color = Colorize.new -end -cc = ARGV[0] -code = File.binread(ARGV[1]) # Current version of the header file. -outfile = ARGV[2] -cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors' - -if !MJITHeader.supported_header?(code) - puts "This compiler (#{cc}) looks not supported for MJIT. Giving up to generate MJIT header." - MJITHeader.write("#error MJIT does not support '#{cc}' yet", outfile) - exit -end - -MJITHeader.remove_predefined_macros!(code) - -if MJITHeader.windows? # transformation is broken on Windows and the platform is not supported - MJITHeader.remove_harmful_macros!(code) - MJITHeader.check_code!(code, cc, cflags, 'initial') - puts "\nSkipped transforming external functions to static on Windows." - MJITHeader.write(code, outfile) - exit -end - -macro, code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW -code = <<header + code -#ifdef __GNUC__ -# pragma GCC system_header -#endif -header -code_to_check = "#{code}#{macro}" # macro should not affect code again - -if MJITHeader.conflicting_types?(code_to_check, cc, cflags) - cflags = "#{cflags} -std=c99" # For AIX gcc -end - -# Check initial file correctness in the manner of final output. -MJITHeader.check_code!(code_to_check, cc, cflags, 'initial') - -stop_pos = -1 -extern_names = [] -transform_logs = Hash.new { |h, k| h[k] = [] } - -# This loop changes function declarations to static inline. -while (decl_range = MJITHeader.find_decl(code, stop_pos)) - stop_pos = decl_range.begin - 1 - decl = code[decl_range] - decl_name = MJITHeader.decl_name_of(decl) - - if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - transform_logs[:def_to_decl] << decl_name - code[decl_range] = decl.sub(/{.+}/m, ';') - elsif MJITHeader::COLD_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - header = match[0].sub(/{\z/, '').strip - header = "static #{header.sub(/\A((static|inline) )+/, '')}" - decl[match.begin(0)...match.end(0)] = '{' # remove header - code[decl_range] = "\nCOLDFUNC #{header} #{decl}" - elsif MJITHeader::ALWAYS_INLINED_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - header = match[0].sub(/{\z/, '').strip - header = "static inline #{header.sub(/\A((static|inline) )+/, '')}" - decl[match.begin(0)...match.end(0)] = '{' # remove header - code[decl_range] = "\nALWAYS_INLINE(#{header});\n#{header} #{decl}" - elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/) - decl.sub!(/(extern|static|inline) /, ' ') - unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. - transform_logs[:static_inline_decl] << decl_name - end - - code[decl_range] = "static inline #{decl}" - elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/ - unless decl_name.match(MJITHeader::TARGET_NAME_REGEXP) - transform_logs[:skipped] << decl_name - next - end - - extern_names << decl_name - decl[match.begin(0)...match.end(0)] = '' - - if decl =~ /\bstatic\b/ - abort "#{PROGRAM}: a static decl was found inside external definition #{decl_name.dump}" - end - - header.sub!(/(extern|inline) /, ' ') - unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. - transform_logs[:static_inline_def] << decl_name - end - code[decl_range] = "static inline #{header}#{decl}" - end -end - -code << macro - -# Check the final file correctness -MJITHeader.check_code!(code, cc, cflags, 'final') - -MJITHeader.write(code, outfile) - -messages = { - def_to_decl: 'changing definition to declaration', - static_inline_def: 'making external definition static inline', - static_inline_decl: 'making declaration static inline', - skipped: 'SKIPPED to transform', -} -transform_logs.each do |key, decl_names| - decl_names = decl_names.map { |s| color.bold(s) } if color - puts("#{PROGRAM}: #{messages.fetch(key)}: #{decl_names.join(', ')}") -end diff --git a/tool/update-NEWS-gemlist.rb b/tool/update-NEWS-gemlist.rb index aa766b8846..0b5503580d 100755 --- a/tool/update-NEWS-gemlist.rb +++ b/tool/update-NEWS-gemlist.rb @@ -1,4 +1,4 @@ -#!/usr/bin/ruby +#!/usr/bin/env ruby require 'json' news = File.read("NEWS.md") prev = news[/since the \*+(\d+\.\d+\.\d+)\*+/, 1] @@ -6,34 +6,47 @@ prevs = [prev, prev.sub(/\.\d+\z/, '')] update = ->(list, type, desc = "updated") do item = ->(mark = "* ") do - "The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + + "### The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + list.map {|g, v|"#{mark}#{g} #{v}\n"}.join("") + "\n" end - news.sub!(/^(?:\*( +))?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do + news.sub!(/^(?:\*( +)|#+ *)?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do item["#{$1&.<< " "}*#{$2 || ' '}"] end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+( *\* *).*)*)*\n+\K/) do item[$1 || "* "] end end -ARGV.each do |type| - last = JSON.parse(File.read("#{type}_gems.json"))['gems'].filter_map do |g| + +load_gems_json = ->(type) do + JSON.parse(File.read("#{type}_gems.json"))['gems'].filter_map do |g| v = g['versions'].values_at(*prevs).compact.first g = g['gem'] g = 'RubyGems' if g == 'rubygems' [g, v] if v end.to_h +end + +ARGV.each do |type| + last = load_gems_json[type] changed = File.foreach("gems/#{type}_gems").filter_map do |l| next if l.start_with?("#") g, v = l.split(" ", 3) next unless v [g, v] unless last[g] == v end - if type == 'bundled' - changed, added = changed.partition {|g, _| last[g]} - end + changed, added = changed.partition {|g, _| last[g]} update[changed, type] or next if added and !added.empty? - update[added, 'default', 'now bundled'] or next + if type == 'bundled' + default_gems = load_gems_json['default'] + promoted = {} + added.delete_if do |k, v| + default_gems.key?(k) && promoted[k] = v + end + update[added, type, 'added'] + update[promoted, type, 'promoted from default gems'] or next + else + update[added, type, 'added'] or next + end end File.write("NEWS.md", news) end diff --git a/tool/update-NEWS-refs.rb b/tool/update-NEWS-refs.rb index 2b19f0fdaa..f48cac5ee1 100644 --- a/tool/update-NEWS-refs.rb +++ b/tool/update-NEWS-refs.rb @@ -13,8 +13,9 @@ if links.empty? || lines.last != "" raise "NEWS.md must end with a sequence of links" end -labels = links.keys.select {|k| !(k.start_with?("Feature") || k.start_with?("Bug"))} -new_src = lines.join("\n").gsub(/\[?\[((?:Feature|Bug)\s+#(\d+))\]\]?/) do +trackers = ["Feature", "Bug", "Misc"] +labels = links.keys.reject {|k| k.start_with?(*trackers)} +new_src = lines.join("\n").gsub(/\[?\[(#{Regexp.union(trackers)}\s+#(\d+))\]\]?/) do links[$1] ||= "https://bugs.ruby-lang.org/issues/#$2" "[[#$1]]" end.gsub(/\[\[#{Regexp.union(labels)}\]\]?/) do @@ -22,7 +23,7 @@ end.gsub(/\[\[#{Regexp.union(labels)}\]\]?/) do end.chomp + "\n\n" label_width = links.max_by {|k, _| k.size}.first.size + 4 -redmine_links, non_redmine_links = links.partition {|k,| k =~ /\A(Feature|Bug)\s+#\d+\z/ } +redmine_links, non_redmine_links = links.partition {|k,| k =~ /\A#{Regexp.union(trackers)}\s+#\d+\z/ } (redmine_links.sort_by {|k,| k[/\d+/].to_i } + non_redmine_links.reverse).each do |k, v| new_src << "[#{k}]:".ljust(label_width) << v << "\n" diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb index c8927b878d..dec6b49cee 100755 --- a/tool/update-bundled_gems.rb +++ b/tool/update-bundled_gems.rb @@ -1,23 +1,41 @@ -#!ruby -pla +#!ruby -alpF\s+|#.* BEGIN { require 'rubygems' + date = nil + # STDOUT is not usable in inplace edit mode + output = $-i ? STDOUT : STDERR } -unless /^[^#]/ !~ (gem = $F[0]) +output = STDERR if ARGF.file == STDIN +END { + output.print date.strftime("latest_date=%F") if date +} +if gem = $F[0] + ver = Gem::Version.new($F[1]) (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| s.platform == "ruby" && s.name == gem } - gem = src.fetch_spec(gem) - uri = gem.metadata["source_code_uri"] || gem.homepage - uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") - if $F[3] - if $F[3].include?($F[1]) - $F[3][$F[1]] = gem.version.to_s - elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] - $F[3..-1] = [] + if gem.version > ver + gem = src.fetch_spec(gem) + if ENV["UPDATE_BUNDLED_GEMS_ALL"] + uri = gem.metadata["source_code_uri"] || gem.homepage + uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") + else + uri = $F[2] + end + if (!date or gem.date && gem.date > date) and gem.date.to_i != 315_619_200 + # DEFAULT_SOURCE_DATE_EPOCH is meaningless + date = gem.date + end + if $F[3] + if $F[3].include?($F[1]) + $F[3][$F[1]] = gem.version.to_s + elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] + $F[3..-1] = [] + end end + f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] + $_.gsub!(/\S+\s*(?=\s|$)/) {|s| (f.shift || "").ljust(s.size)} + $_ = [$_, *f].join(" ") unless f.empty? + $_.rstrip! end - f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] - $_.gsub!(/\S+\s*/) {|s| (f.shift || "").ljust(s.size)} - $_ = [$_, *f].join(" ") unless f.empty? - $_.rstrip! end diff --git a/tool/update-deps b/tool/update-deps index 445f9a00e3..2d4a5674be 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -17,6 +17,14 @@ # 3. Use --fix to fix makefiles. # Ex. ./ruby tool/update-deps --fix # +# Usage to create a depend file initially: +# 1. Copy the dependency section from the Makefile generated by extconf.rb. +# Ex. ext/cgi/escape/Makefile +# 2. Add `# AUTOGENERATED DEPENDENCIES START` and `# AUTOGENERATED DEPENDENCIES END` +# sections to top and end of the depend file. +# 3. Run tool/update-deps --fix to fix the depend file. +# 4. Commit the depend file. +# # Other usages: # * Fix makefiles using previously detected dependency problems # Ex. ruby tool/update-deps --actual-fix [file] @@ -88,6 +96,15 @@ result.each {|k,v| # They can be referenced as $(top_srcdir)/filename. # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' FILES_IN_SOURCE_DIRECTORY = %w[ + prism/api_node.c + prism/ast.h + prism/diagnostic.c + prism/diagnostic.h + prism/node.c + prism/prettyprint.c + prism/serialize.c + prism/token_type.c + prism/version.h ] # Files built in the build directory (except extconf.h). @@ -111,6 +128,7 @@ FILES_NEED_VPATH = %w[ ext/ripper/eventids1.c ext/ripper/eventids2table.c ext/ripper/ripper.c + ext/ripper/ripper_init.c golf_prelude.c id.c id.h @@ -119,10 +137,8 @@ FILES_NEED_VPATH = %w[ known_errors.inc lex.c miniprelude.c - mjit_sp_inc.inc newline.c node_name.inc - opt_sc.inc optinsn.inc optunifs.inc parse.c @@ -167,13 +183,17 @@ FILES_SAME_NAME_TOP = %w[ version.h ] +# Files that may or may not exist on CI for some reason. +# Windows build generally seems to have missing dependencies. +UNSTABLE_FILES = %r{\Awin32/[^/]+\.o\z} + # Other source files exist in the source directory. def in_makefile(target, source) target = target.to_s source = source.to_s case target - when %r{\A[^/]*\z}, %r{\Acoroutine/} + when %r{\A[^/]*\z}, %r{\Acoroutine/}, %r{\Aprism/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" @@ -193,7 +213,7 @@ def in_makefile(target, source) when %r{\Acoroutine/} then source2 = "{$(VPATH)}$(COROUTINE_H)" else source2 = "$(top_srcdir)/#{source}" end - ["common.mk", target2, source2] + ["depend", target2, source2] when %r{\Aenc/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source @@ -230,6 +250,10 @@ def in_makefile(target, source) else source2 = "$(top_srcdir)/#{source}" end ["#{File.dirname(target)}/depend", target2, source2] + # Files that may or may not exist on CI for some reason. + # Windows build generally seems to have missing dependencies. + when UNSTABLE_FILES + warn "warning: ignoring: #{target}" else raise "unexpected target: #{target}" end @@ -301,6 +325,9 @@ def read_make_deps(cwd) deps = deps.scan(%r{[/0-9a-zA-Z._-]+}) deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp next if /\.o\z/ !~ target.to_s + next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source) + next if /libzjit.o\z/ =~ target.to_s # skip ZJIT Rust object (no corresponding C source) + next if /target\/release\/libruby.o\z/ =~ target.to_s # skip YJIT+ZJIT Rust object (no corresponding C source) next if /\.bundle\// =~ target.to_s next if /\A\./ =~ target.to_s # skip rules such as ".c.o" #p [curdir, target, deps] @@ -333,24 +360,25 @@ end # raise ArgumentError, "can not find #{filename} (hint: #{hint0})" #end -def read_single_cc_deps(path_i, cwd) +def read_single_cc_deps(path_i, cwd, fn_o) files = {} compiler_wd = nil path_i.each_line {|line| next if /\A\# \d+ "(.*)"/ !~ line dep = $1 + next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc. next if /\.e?rb\z/ =~ dep - compiler_wd ||= dep + # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. + if /\/\/\z/ =~ dep + compiler_wd = Pathname(dep.sub(%r{//\z}, '')) + next + end + files[dep] = true } - # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. - if %r{\A/.*//\z} =~ compiler_wd - files.delete compiler_wd - compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, '')) - elsif !(compiler_wd = yield) - raise "compiler working directory not found: #{path_i}" - end + compiler_wd ||= fn_o.to_s.start_with?("enc/") ? cwd : path_i.parent + deps = [] files.each_key {|dep| dep = Pathname(dep) @@ -382,13 +410,7 @@ def read_cc_deps(cwd) end path_o = cwd + fn_o path_i = cwd + fn_i - deps[path_o] = read_single_cc_deps(path_i, cwd) do - if fn_o.to_s.start_with?("enc/") - cwd - else - path_i.parent - end - end + deps[path_o] = read_single_cc_deps(path_i, cwd, fn_o) } deps end @@ -476,7 +498,7 @@ def compare_deps(make_deps, cc_deps, out=$stdout) } } - makefiles.keys.sort.each {|makefile| + makefiles.keys.compact.sort.each {|makefile| cc_lines = cc_lines_hash[makefile] || Hash.new(false) make_lines = make_lines_hash[makefile] || Hash.new(false) content = begin diff --git a/tool/ytab.sed b/tool/ytab.sed deleted file mode 100755 index 95a9b3e1eb..0000000000 --- a/tool/ytab.sed +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sed -f -# This file is used when generating code for the Ruby parser. -/^int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^extern int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^yydestruct.*yymsg/,/{/{ - /^yydestruct/{ - /,$/N - /[, *]p)/!{ - H - s/^/ruby_parser_&/ - s/)$/, p)/ - /\*/s/p)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /yydestruct/{ - i\ -\ struct parser_params *p; - } - x - } - /^{/{ - x - /yydestruct/{ - i\ -#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, p) - } - x - } -} -/^yy_stack_print /,/{/{ - /^yy_stack_print/{ - /[, *]p)/!{ - H - s/^/ruby_parser_&/ - s/)$/, p)/ - /\*/s/p)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /yy_stack_print/{ - i\ -\ struct parser_params *p; - } - x - } - /^{/{ - x - /yy_stack_print/{ - i\ -#define yy_stack_print(b, t) ruby_parser_yy_stack_print(b, t, p) - } - x - } -} -/^yy_reduce_print/,/^}/{ - s/fprintf *(stderr,/YYFPRINTF (p,/g -} -s/^yysyntax_error (/&struct parser_params *p, / -s/ yysyntax_error (/&p, / -s/\( YYFPRINTF *(\)yyoutput,/\1p,/ -s/\( YYFPRINTF *(\)yyo,/\1p,/ -s/\( YYFPRINTF *(\)stderr,/\1p,/ -s/\( YYDPRINTF *((\)stderr,/\1p,/ -s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/ -s!^ *extern char \*getenv();!/* & */! -s/^\(#.*\)".*\.tab\.c"/\1"parse.c"/ -/^\(#.*\)".*\.y"/s:\\\\:/:g diff --git a/tool/zjit_bisect.rb b/tool/zjit_bisect.rb new file mode 100755 index 0000000000..997c572f51 --- /dev/null +++ b/tool/zjit_bisect.rb @@ -0,0 +1,158 @@ +#!/usr/bin/env ruby +require 'logger' +require 'optparse' +require 'shellwords' +require 'tempfile' +require 'timeout' + +ARGS = {timeout: 5} +OptionParser.new do |opts| + opts.banner += " <path_to_ruby> -- <options>" + opts.on("--timeout=TIMEOUT_SEC", "Seconds until child process is killed") do |timeout| + ARGS[:timeout] = Integer(timeout) + end + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end +end.parse! + +usage = "Usage: zjit_bisect.rb <path_to_ruby> -- <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +raise(usage) if OPTIONS.empty? +LOGGER = Logger.new($stdout) + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +# Attempt to reduce the `items` argument as much as possible, returning the +# shorter version. `fixed` will always be used as part of the items when +# running `command`. +# `command` should return True if the command succeeded (the failure did not +# reproduce) and False if the command failed (the failure reproduced). +def bisect_impl(command, fixed, items, indent="") + LOGGER.info("#{indent}step fixed[#{fixed.length}] and items[#{items.length}]") + while items.length > 1 + LOGGER.info("#{indent}#{fixed.length + items.length} candidates") + # Return two halves of the given list. For odd-length lists, the second + # half will be larger. + half = items.length / 2 + left = items[0...half] + right = items[half..] + if !command.call(fixed + left) + items = left + next + end + if !command.call(fixed + right) + items = right + next + end + # We need something from both halves to trigger the failure. Try + # holding each half fixed and bisecting the other half to reduce the + # candidates. + new_right = bisect_impl(command, fixed + left, right, indent + "< ") + new_left = bisect_impl(command, fixed + new_right, left, indent + "> ") + return new_left + new_right + end + items +end + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +def run_bisect(command, items) + LOGGER.info("Verifying items") + if command.call(items) + raise StandardError.new("Command succeeded with full items") + end + if !command.call([]) + raise StandardError.new("Command failed with empty items") + end + bisect_impl(command, [], items) +end + +def add_zjit_options cmd + if RUBY == "make" + # Automatically detect that we're running a make command instead of a Ruby + # one. Pass the bisection options via RUN_OPTS/SPECOPTS instead. + zjit_opts = cmd.select { |arg| arg.start_with?("--zjit") } + run_opts_index = cmd.find_index { |arg| arg.start_with?("RUN_OPTS=") } + specopts_index = cmd.find_index { |arg| arg.start_with?("SPECOPTS=") } + if run_opts_index + run_opts = Shellwords.split(cmd[run_opts_index].delete_prefix("RUN_OPTS=")) + run_opts.concat(zjit_opts) + cmd[run_opts_index] = "RUN_OPTS=#{run_opts.shelljoin}" + elsif specopts_index + specopts = Shellwords.split(cmd[specopts_index].delete_prefix("SPECOPTS=")) + specopts.concat(zjit_opts) + cmd[specopts_index] = "SPECOPTS=#{specopts.shelljoin}" + else + raise "Expected RUN_OPTS or SPECOPTS to be present in make command" + end + cmd = cmd - zjit_opts + end + cmd +end + +def run_ruby *cmd + cmd = add_zjit_options(cmd) + pid = Process.spawn(*cmd, { + in: :close, + out: [File::NULL, File::RDWR], + err: [File::NULL, File::RDWR], + }) + begin + status = Timeout.timeout(ARGS[:timeout]) do + Process::Status.wait(pid) + end + rescue Timeout::Error + Process.kill("KILL", pid) + LOGGER.warn("Timed out after #{ARGS[:timeout]} seconds") + status = Process::Status.wait(pid) + end + + status +end + +def run_with_jit_list(ruby, options, jit_list) + # Make a new temporary file containing the JIT list + Tempfile.create("jit_list") do |temp_file| + temp_file.write(jit_list.join("\n")) + temp_file.flush + temp_file.close + # Run the JIT with the temporary file + run_ruby ruby, "--zjit-allowed-iseqs=#{temp_file.path}", *options + end +end + +# Try running with no JIT list to get a stable baseline +unless run_with_jit_list(RUBY, OPTIONS, []).success? + cmd = [RUBY, "--zjit-allowed-iseqs=/dev/null", *OPTIONS].shelljoin + raise "The command failed unexpectedly with an empty JIT list. To reproduce, try running the following: `#{cmd}`" +end +# Collect the JIT list from the failing Ruby process +jit_list = nil +Tempfile.create "jit_list" do |temp_file| + run_ruby RUBY, "--zjit-log-compiled-iseqs=#{temp_file.path}", *OPTIONS + jit_list = File.readlines(temp_file.path).map(&:strip).reject(&:empty?) +end +LOGGER.info("Starting with JIT list of #{jit_list.length} items.") +# Try running without the optimizer +status = run_with_jit_list(RUBY, ["--zjit-disable-hir-opt", *OPTIONS], jit_list) +if status.success? + LOGGER.warn "*** Command suceeded with HIR optimizer disabled. HIR optimizer is probably at fault. ***" +end +# Now narrow it down +command = lambda do |items| + run_with_jit_list(RUBY, OPTIONS, items).success? +end +result = run_bisect(command, jit_list) +File.open("jitlist.txt", "w") do |file| + file.puts(result) +end +puts "Run:" +jitlist_path = File.expand_path("jitlist.txt") +puts add_zjit_options([RUBY, "--zjit-allowed-iseqs=#{jitlist_path}", *OPTIONS]).shelljoin +puts "Reduced JIT list (available in jitlist.txt):" +puts result diff --git a/tool/zjit_iongraph.html b/tool/zjit_iongraph.html new file mode 100644 index 0000000000..993cce9045 --- /dev/null +++ b/tool/zjit_iongraph.html @@ -0,0 +1,551 @@ +<!-- Copyright Mozilla and licensed under Mozilla Public License Version 2.0. + Source can be found at https://github.com/mozilla-spidermonkey/iongraph --> +<!-- Generated by `npm run build-www` on + 39b04fa18f23cbf3fd2ca7339a45341ff3351ba1 in tekknolagi/iongraph --> +<!DOCTYPE html> + +<head> + <title>iongraph</title> + <style>/* iongraph-specific styles inspired by Tachyons */ + +:root { + --ig-size-1: 1rem; + --ig-size-2: 2rem; + --ig-size-3: 4rem; + --ig-size-4: 8rem; + --ig-size-5: 16rem; + + --ig-spacing-1: .25rem; + --ig-spacing-2: .5rem; + --ig-spacing-3: 1rem; + --ig-spacing-4: 2rem; + --ig-spacing-5: 4rem; + --ig-spacing-6: 8rem; + --ig-spacing-7: 16rem; + + --ig-text-color: black; + --ig-text-color-dim: #777; + --ig-background-primary: #ffb54e; + --ig-background-light: white; + --ig-border-color: #0c0c0d; + + --ig-block-header-color: #0c0c0d; + --ig-loop-header-color: #1fa411; + --ig-movable-color: #1048af; + --ig-rob-color: #444; + --ig-in-worklist-color: red; + + --ig-block-selected: #ffc863; + --ig-block-last-selected: #ffb54e; + + --ig-highlight-0: #ffb54e; + --ig-highlight-1: #ffb5c5; + --ig-highlight-2: #a4cbff; + --ig-highlight-3: #8be182; + --ig-highlight-4: #d9a4fd; + + --ig-flash-color: #ffb54e; + + /* + * The heatmap of sample counts will effectively be sampled from a gradient: + * + * |----------|---------------------------------------| + * cold "cool" hot + * + * The "cold" color will simply be transparent. Therefore, the "cool" + * threshold indicates where the instruction will be fully colored and + * noticeable to the user. + */ + --ig-hot-color: #ff849e; + --ig-cool-color: #ffe546; + --ig-cool-threshold: 0.2; +} + +a.ig-link-normal { + color: inherit; + text-decoration: inherit; +} + +.ig-flex { + display: flex; +} + +.ig-flex-column { + flex-direction: column; +} + +.ig-flex-basis-0 { + flex-basis: 0; +} + +.ig-flex-grow-1 { + flex-grow: 1; +} + +.ig-flex-shrink-0 { + flex-shrink: 0; +} + +.ig-flex-shrink-1 { + flex-shrink: 1; +} + +.ig-items-center { + align-items: center; +} + +.ig-relative { + position: relative; +} + +.ig-absolute { + position: absolute; +} + +.ig-absolute-fill { + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.ig-g1 { + gap: var(--ig-spacing-1); +} + +.ig-g2 { + gap: var(--ig-spacing-2); +} + +.ig-g3 { + gap: var(--ig-spacing-3); +} + +.ig-w1 { + width: var(--ig-size-1); +} + +.ig-w2 { + width: var(--ig-size-2); +} + +.ig-w3 { + width: var(--ig-size-3); +} + +.ig-w4 { + width: var(--ig-size-4); +} + +.ig-w5 { + width: var(--ig-size-5); +} + +.ig-w-100 { + width: 100%; +} + +.ig-ba { + border-style: solid; + border-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bt { + border-top-style: solid; + border-top-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-br { + border-right-style: solid; + border-right-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bb { + border-bottom-style: solid; + border-bottom-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bl { + border-left-style: solid; + border-left-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-pa1 { + padding: var(--ig-spacing-1); +} + +.ig-pa2 { + padding: var(--ig-spacing-2); +} + +.ig-pa3 { + padding: var(--ig-spacing-3); +} + +.ig-ph1 { + padding-left: var(--ig-spacing-1); + padding-right: var(--ig-spacing-1); +} + +.ig-ph2 { + padding-left: var(--ig-spacing-2); + padding-right: var(--ig-spacing-2); +} + +.ig-ph3 { + padding-left: var(--ig-spacing-3); + padding-right: var(--ig-spacing-3); +} + +.ig-pv1 { + padding-top: var(--ig-spacing-1); + padding-bottom: var(--ig-spacing-1); +} + +.ig-pv2 { + padding-top: var(--ig-spacing-2); + padding-bottom: var(--ig-spacing-2); +} + +.ig-pv3 { + padding-top: var(--ig-spacing-3); + padding-bottom: var(--ig-spacing-3); +} + +.ig-pt1 { + padding-top: var(--ig-spacing-1); +} + +.ig-pt2 { + padding-top: var(--ig-spacing-2); +} + +.ig-pt3 { + padding-top: var(--ig-spacing-3); +} + +.ig-pr1 { + padding-right: var(--ig-spacing-1); +} + +.ig-pr2 { + padding-right: var(--ig-spacing-2); +} + +.ig-pr3 { + padding-right: var(--ig-spacing-3); +} + +.ig-pb1 { + padding-bottom: var(--ig-spacing-1); +} + +.ig-pb2 { + padding-bottom: var(--ig-spacing-2); +} + +.ig-pb3 { + padding-bottom: var(--ig-spacing-3); +} + +.ig-pl1 { + padding-left: var(--ig-spacing-1); +} + +.ig-pl2 { + padding-left: var(--ig-spacing-2); +} + +.ig-pl3 { + padding-left: var(--ig-spacing-3); +} + +.ig-f1 { + font-size: 3rem; +} + +.ig-f2 { + font-size: 2.25rem; +} + +.ig-f3 { + font-size: 1.5rem; +} + +.ig-f4 { + font-size: 1.25rem; +} + +.ig-f5 { + font-size: 1rem; +} + +.ig-f6 { + font-size: .875rem; +} + +.ig-f7 { + font-size: .75rem; +} + +.ig-text-normal { + color: var(--ig-text-color); +} + +.ig-text-dim { + color: var(--ig-text-color-dim); +} + +.ig-tl { + text-align: left; +} + +.ig-tr { + text-align: right; +} + +.ig-tc { + text-align: center; +} + +.ig-bg-white { + background-color: var(--ig-background-light); +} + +.ig-bg-primary { + background-color: var(--ig-background-primary); +} + +.ig-overflow-hidden { + overflow: hidden; +} + +.ig-overflow-auto { + overflow: auto; +} + +.ig-overflow-x-auto { + overflow-x: auto; +} + +.ig-overflow-y-auto { + overflow-y: auto; +} + +.ig-hide-if-empty:empty { + display: none; +} + +/* Non-utility styles */ + +.ig-graph { + color: var(--ig-text-color); + position: absolute; + left: 0; + top: 0; + isolation: isolate; +} + +.ig-block { + position: absolute; + + .ig-block-header { + font-weight: bold; + text-align: center; + background-color: var(--ig-block-header-color); + color: white; + padding: 0 1em; + border: 1px solid var(--ig-border-color); + border-width: 1px 1px 0; + } + + .ig-instructions { + padding: 0.5em; + border: 1px solid var(--ig-border-color); + border-width: 0 1px 1px; + + table { + border-collapse: collapse; + } + + td, + th { + white-space: nowrap; + padding: 0.1em 0.5em; + } + + th { + font-weight: normal; + } + } + + &.ig-selected { + outline: 4px solid var(--ig-block-selected); + } + + &.ig-last-selected { + outline-color: var(--ig-block-last-selected); + } +} + +.ig-block-att-loopheader { + .ig-block-header { + background-color: var(--ig-loop-header-color); + } +} + +.ig-block-att-splitedge { + .ig-instructions { + border-style: dotted; + border-width: 0 2px 2px; + } +} + +.ig-ins-num { + text-align: right; + cursor: pointer; +} + +.ig-ins-type { + text-align: right; +} + +.ig-ins-samples { + font-size: 0.875em; + text-align: right; + cursor: pointer; +} + +.ig-use { + padding: 0 0.25em; + border-radius: 2px; + cursor: pointer; +} + +.ig-edge-label { + position: absolute; + font-size: 0.8em; + line-height: 1; + bottom: -1em; + padding-left: 4px; +} + +.ig-ins-att-RecoveredOnBailout { + color: var(--ig-rob-color); +} + +.ig-ins-att-Movable { + color: var(--ig-movable-color); +} + +.ig-ins-att-Guard { + text-decoration: underline; +} + +.ig-ins-att-InWorklist { + color: var(--ig-in-worklist-color); +} + +.ig-can-flash { + transition: outline-color 1s ease-out; + outline: 3px solid color-mix(in srgb, var(--ig-flash-color) 0%, transparent); +} + +.ig-flash { + transition: outline-color 0s; + outline-color: var(--ig-flash-color); +} + +.ig-hotness { + --ig-hotness: 0; + + --ig-cold-color: color-mix(in srgb, var(--ig-cool-color) 20%, transparent); + background-color: + /* cool <-> hot */ + color-mix(in oklab, + /* cold <-> cool */ + color-mix(in oklab, + /* dead or cold */ + color-mix(in srgb, transparent, var(--ig-cold-color) clamp(0%, calc(var(--ig-hotness) * 100000000%), 100%)), + var(--ig-cool-color) clamp(0%, calc((var(--ig-hotness) / var(--ig-cool-threshold)) * 100%), 100%)), + var(--ig-hot-color) clamp(0%, calc(((var(--ig-hotness) - var(--ig-cool-threshold)) / (1 - var(--ig-cool-threshold))) * 100%), 100%)); +} + +.ig-highlight { + --ig-highlight-color: transparent; + background-color: var(--ig-highlight-color); +}</style> + <style>* { + box-sizing: border-box; +} + +:root { + font-size: 0.875rem; +} + +body { + margin: 0; + background-color: #e5e8ea; + /* Font, and many other styles, taken from the Firefox Profiler/ */ + font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen-Sans, Ubuntu, Noto Sans, Liberation Sans, Cantarell, Helvetica Neue, sans-serif; +} + +#container { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +.tweaks-panel { + position: absolute; + bottom: 0; + right: 0; + padding: 1rem; + border: 1px solid black; + border-width: 1px 0 0 1px; + background-color: white; +}</style> +</head> + +<body> + <script>"use strict";var iongraph=(()=>{var J=Object.defineProperty;var mt=Object.getOwnPropertyDescriptor;var gt=Object.getOwnPropertyNames;var ft=Object.prototype.hasOwnProperty;var bt=(a,t)=>{for(var e in t)J(a,e,{get:t[e],enumerable:!0})},kt=(a,t,e,s)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of gt(t))!ft.call(a,i)&&i!==e&&J(a,i,{get:()=>t[i],enumerable:!(s=mt(t,i))||s.enumerable});return a};var yt=a=>kt(J({},"__esModule",{value:!0}),a);var Yt={};bt(Yt,{StandaloneUI:()=>et,WebUI:()=>tt});function X(a){a.version===void 0&&(a.version=0);for(let t of a.functions)xt(t,a.version);return a.version=1,a}function xt(a,t){for(let e of a.passes){for(let s of e.mir.blocks)wt(s,t);for(let s of e.lir.blocks)It(s,t)}return a}function wt(a,t){t===0&&(a.ptr=(a.id??a.number)+1,a.id=a.number);for(let e of a.instructions)vt(e,t);return a}function vt(a,t){return t===0&&(a.ptr=a.id),a}function It(a,t){t===0&&(a.ptr=a.id??a.number,a.id=a.number);for(let e of a.instructions)Lt(e,t);return a}function Lt(a,t){return t===0&&(a.ptr=a.id,a.mirPtr=null),a}function q(a,t,e){return Math.max(t,Math.min(e,a))}function R(a,t,e,s){return(a-t)*Math.pow(e,s)+t}function k(a,t,e=!1){if(!a)if(e)console.error(t??"Assertion failed");else throw new Error(t??"Assertion failed")}function L(a,t){return k(a,t),a}function Pt(a){return typeof a=="string"?document.createTextNode(a):a}function Bt(a,t){for(let e of t)e&&a.appendChild(Pt(e))}function x(a,t,e,s){let i=document.createElement(a);if(t&&t.length>0){let c=t.filter(l=>!!l);i.classList.add(...c)}return e?.(i),s&&Bt(i,s),i}var st=Object.prototype.hasOwnProperty;function E(a,t){var e,s;if(a===t)return!0;if(a&&t&&(e=a.constructor)===t.constructor){if(e===Date)return a.getTime()===t.getTime();if(e===RegExp)return a.toString()===t.toString();if(e===Array){if((s=a.length)===t.length)for(;s--&&E(a[s],t[s]););return s===-1}if(!e||typeof a=="object"){s=0;for(e in a)if(st.call(a,e)&&++s&&!st.call(t,e)||!(e in t)||!E(a[e],t[e]))return!1;return Object.keys(t).length===s}}return a!==a&&t!==t}function N(a,t,e={}){let s=t,i=[],c={get(){return s},set(l){s=l;for(let o of i)o(s)},valueOf(){return s},toString(){return String(s)},[Symbol.toPrimitive](l){return l==="string"?String(s):s},onChange(l){i.push(l)},initial:t,name:a,min:e.min??0,max:e.max??100,step:e.step??1};return(e.tweaksObject??K).add(c)}var F=class{constructor(t){this.container=t.container,this.tweaks=[],this.callbacks=[]}add(t){let e=this.tweaks.find(r=>r.name===t.name);if(e)return e;this.tweaks.push(t);let s=document.createElement("div");this.container.appendChild(s),s.style.display="flex",s.style.alignItems="center",s.style.justifyContent="end",s.style.gap="0.5rem";let i=t.name.replace(/[a-zA-Z0-9]/g,"_"),c=document.createElement("label");s.appendChild(c),c.innerText=t.name,c.htmlFor=`tweak-${i}-input`;let l=document.createElement("input");s.appendChild(l),l.type="number",l.value=String(t),l.id=`tweak-${i}-input`,l.style.width="4rem",l.addEventListener("input",()=>{t.set(l.valueAsNumber)});let o=document.createElement("input");s.appendChild(o),o.type="range",o.value=String(t),o.min=String(t.min),o.max=String(t.max),o.step=t.step===0?"any":String(t.step),o.addEventListener("input",()=>{t.set(o.valueAsNumber)});let n=document.createElement("button");return s.appendChild(n),n.innerText="Reset",n.disabled=t.get()===t.initial,n.addEventListener("click",()=>{t.set(t.initial)}),t.onChange(r=>{l.value=String(r),o.value=String(r),n.disabled=t.get()===t.initial;for(let d of this.callbacks)d(t)}),t}onTweak(t){this.callbacks.push(t)}},nt=document.createElement("div");nt.classList.add("tweaks-panel");var K=new F({container:nt});K.onTweak(a=>{window.dispatchEvent(new CustomEvent("tweak",{detail:a}))});window.tweaks=K;var Nt=document.createElement("div"),St=new F({container:Nt}),ot=N("Test Value",3,{tweaksObject:St});ot.set(4);ot=4;var dt=N("Debug?",0,{min:0,max:1}),P=20,C=44,I=16,S=60,T=12,H=36,it=16,D=16,Tt=N("Layout Iterations",2,{min:0,max:6}),rt=N("Nearly Straight Threshold",30,{min:0,max:200}),$t=N("Nearly Straight Iterations",8,{min:0,max:10}),at=N("Stop At Pass",30,{min:0,max:30}),Mt=1.5,Et=.01,Ct=1,Ht=.1,z=40;function _(a){return a.mir.attributes.includes("loopheader")}function Dt(a){return a.loopHeight!==void 0}function $(a){if(k(a),Dt(a))return a;throw new Error(`Block ${a.id} is not a pseudo LoopHeader`)}var j=1,lt=2,U=4,O=0,W=1,w=new Proxy(console,{get(a,t){let e=a[t];return typeof e!="function"?e:+dt?e.bind(a):()=>{}}}),V=class{constructor(t,e,s={}){this.viewport=t;let i=t.getBoundingClientRect();this.viewportSize={x:i.width,y:i.height},this.graphContainer=document.createElement("div"),this.graphContainer.classList.add("ig-graph"),this.graphContainer.style.transformOrigin="top left",this.viewport.appendChild(this.graphContainer),this.sampleCounts=s.sampleCounts,this.maxSampleCounts=[0,0],this.heatmapMode=O;for(let[n,r]of this.sampleCounts?.totalLineHits??[])this.maxSampleCounts[O]=Math.max(this.maxSampleCounts[O],r);for(let[n,r]of this.sampleCounts?.selfLineHits??[])this.maxSampleCounts[W]=Math.max(this.maxSampleCounts[W],r);this.size={x:0,y:0},this.numLayers=0,this.zoom=1,this.translation={x:0,y:0},this.animating=!1,this.targetZoom=1,this.targetTranslation={x:0,y:0},this.startMousePos={x:0,y:0},this.lastMousePos={x:0,y:0},this.selectedBlockPtrs=new Set,this.lastSelectedBlockPtr=0,this.nav={visited:[],currentIndex:-1,siblings:[]},this.highlightedInstructions=[],this.instructionPalette=s.instructionPalette??[0,1,2,3,4].map(n=>`var(--ig-highlight-${n})`),this.blocks=e.mir.blocks.map(n=>{let r={ptr:n.ptr,id:n.id,mir:n,lir:e.lir.blocks.find(d=>d.id===n.id)??null,preds:[],succs:[],el:void 0,size:{x:0,y:0},layer:-1,loopID:-1,layoutNode:void 0};if(r.mir.attributes.includes("loopheader")){let d=r;d.loopHeight=0,d.parentLoop=null,d.outgoingEdges=[]}return k(r.ptr,"blocks must always have non-null ptrs"),r}),this.blocksByID=new Map,this.blocksByPtr=new Map,this.insPtrsByID=new Map,this.insIDsByPtr=new Map,this.loops=[];for(let n of this.blocks){this.blocksByID.set(n.id,n),this.blocksByPtr.set(n.ptr,n);for(let r of n.mir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id);if(n.lir)for(let r of n.lir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id)}for(let n of this.blocks)if(n.preds=n.mir.predecessors.map(r=>L(this.blocksByID.get(r))),n.succs=n.mir.successors.map(r=>L(this.blocksByID.get(r))),_(n)){let r=n.preds.filter(d=>d.mir.attributes.includes("backedge"));k(r.length===1),n.backedge=r[0]}for(let n of this.blocks)n.el=this.renderBlock(n);for(let n of this.blocks)n.size={x:n.el.clientWidth,y:n.el.clientHeight};let[c,l,o]=this.layout();this.render(c,l,o),this.addEventListeners()}layout(){let[t,e]=this.findLayoutRoots();w.log("Layout roots:",t.map(l=>l.id));for(let l of[...t,...e]){let o=l;o.loopHeight=0,o.parentLoop=null,o.outgoingEdges=[],Object.defineProperty(o,"backedge",{get(){throw new Error("Accessed .backedge on a pseudo loop header! Don't do that.")},configurable:!0})}for(let l of t)w.group("findLoops"),this.findLoops(l),w.groupEnd();for(let l of t)w.group("layer"),this.layer(l),w.groupEnd();for(let l of e)l.layer=0,l.loopID=l.id;let s=this.makeLayoutNodes();this.straightenEdges(s);let i=this.finagleJoints(s),c=this.verticalize(s,i);return[s,c,i]}findLayoutRoots(){let t=[],e=[],s=this.blocks.filter(i=>i.preds.length===0);for(let i of s){let c=i;if(i.mir.attributes.includes("osr")){k(i.succs.length>0);let l=i.succs[0];c=l;for(let o=0;;o++){if(o>=1e7)throw new Error("likely infinite loop");let n=c.preds.filter(r=>!r.mir.attributes.includes("osr")&&!r.mir.attributes.includes("backedge"));if(n.length===0)break;c=n[0]}c!==l?e.push(i):c=i}t.includes(c)||t.push(c)}return[t,e]}findLoops(t,e=null){if(e===null&&(e=[t.id]),!(t.loopID>=0)){if(w.log("block:",t.id,t.mir.loopDepth,"loopIDsByDepth:",e),w.log(t.mir.attributes),_(t)){let s=e[e.length-1],i=$(this.blocksByID.get(s));t.parentLoop=i,e=[...e,t.id],w.log("Block",t.id,"is true loop header, loopIDsByDepth is now",e)}if(t.mir.loopDepth>e.length-1&&(t.mir.loopDepth=e.length-1,w.log("Block",t.id,"has been forced back to loop depth",t.mir.loopDepth)),t.mir.loopDepth<e.length-1&&(e=e.slice(0,t.mir.loopDepth+1),w.log("Block",t.id,"has low loop depth, therefore we exited a loop. loopIDsByDepth:",e)),t.loopID=e[t.mir.loopDepth],!t.mir.attributes.includes("backedge"))for(let s of t.succs)this.findLoops(s,e)}}layer(t,e=0){if(w.log("block",t.id,"layer",e),t.mir.attributes.includes("backedge")){t.layer=t.succs[0].layer;return}if(e<=t.layer)return;t.layer=Math.max(t.layer,e),this.numLayers=Math.max(t.layer+1,this.numLayers);let s=$(this.blocksByID.get(t.loopID));for(;s;)s.loopHeight=Math.max(s.loopHeight,t.layer-s.layer+1),s=s.parentLoop;for(let i of t.succs)i.mir.loopDepth<t.mir.loopDepth?$(this.blocksByID.get(t.loopID)).outgoingEdges.push(i):this.layer(i,e+1);if(_(t))for(let i of t.outgoingEdges)this.layer(i,e+t.loopHeight)}makeLayoutNodes(){w.group("makeLayoutNodes");function t(o,n,r){o.dstNodes[n]=r,r.srcNodes.includes(o)||r.srcNodes.push(o),w.log("connected",o.id,"to",r.id)}let e;{let o={};for(let n of this.blocks)o[n.layer]||(o[n.layer]=[]),o[n.layer].push(n);e=Object.entries(o).map(([n,r])=>[Number(n),r]).sort((n,r)=>n[0]-r[0]).map(([n,r])=>r)}let s=0,i=e.map(()=>[]),c=[],l=new Map;for(let[o,n]of e.entries()){w.group("layer",o,"blocks",n.map(u=>u.id));let r=[];for(let u of n)for(let h=c.length-1;h>=0;h--){let p=c[h];p.dstBlock===u&&(r.unshift(p),c.splice(h,1))}let d=new Map;for(let u of c){let h,p=d.get(u.dstBlock.id);if(p)t(u.src,u.srcPort,p),h=p;else{let m={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:u.dstBlock,jointOffsets:[],flags:0};t(u.src,u.srcPort,m),i[o].push(m),d.set(u.dstBlock.id,m),h=m,w.log("Created dummy",h.id,"on the way to block",u.dstBlock.id)}u.src=h,u.srcPort=0}let b=[];for(let u of n){let h=$(this.blocksByID.get(u.loopID));for(;_(h);){let p=b.find(f=>f.loopID===h.id);p?p.block=u:b.push({loopID:h.id,block:u});let m=h.parentLoop;if(!m)break;h=m}}let g=[];for(let u of n){let h={id:s++,pos:{x:P,y:P},size:u.size,block:u,srcNodes:[],dstNodes:[],jointOffsets:[],flags:0};for(let p of r)p.dstBlock===u&&t(p.src,p.srcPort,h);i[o].push(h),u.layoutNode=h;for(let p of b.filter(m=>m.block===u)){let m=$(this.blocksByID.get(p.loopID)).backedge,f={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:m,jointOffsets:[],flags:0},y=l.get(m);y?t(f,0,y):(f.flags|=U,t(f,0,m.layoutNode)),i[o].push(f),l.set(m,f)}if(u.mir.attributes.includes("backedge"))t(u.layoutNode,0,u.succs[0].layoutNode);else for(let[p,m]of u.succs.entries())m.mir.attributes.includes("backedge")?g.push({src:h,srcPort:p,dstBlock:m}):c.push({src:h,srcPort:p,dstBlock:m})}for(let u of g){let h=L(l.get(u.dstBlock));t(u.src,u.srcPort,h)}w.groupEnd()}w.log("Pruning backedge dummies");{let o=[];for(let r of At(i))r.srcNodes.length===0&&o.push(r);let n=new Set;for(let r of o){let d=r;for(;d.block===null&&d.srcNodes.length===0;)Ot(d),n.add(d),k(d.dstNodes.length===1),d=d.dstNodes[0]}for(let r of i)for(let d=r.length-1;d>=0;d--)n.has(r[d])&&r.splice(d,1)}w.log("Marking leftmost and rightmost dummies");for(let o of i){for(let n=0;n<o.length&&o[n].block===null;n++)o[n].flags|=j;for(let n=o.length-1;n>=0&&o[n].block===null;n--)o[n].flags|=lt}w.log("Verifying integrity of all nodes");for(let o of i)for(let n of o){n.block?k(n.dstNodes.length===n.block.succs.length,`expected node ${n.id} for block ${n.block.id} to have ${n.block.succs.length} destination nodes, but got ${n.dstNodes.length} instead`):k(n.dstNodes.length===1,`expected dummy node ${n.id} to have only one destination node, but got ${n.dstNodes.length} instead`);for(let r=0;r<n.dstNodes.length;r++)k(n.dstNodes[r]!==void 0,`dst slot ${r} of node ${n.id} was undefined`)}return w.groupEnd(),i}straightenEdges(t){let e=g=>{for(let u=0;u<g.length-1;u++){let h=g[u],p=g[u+1],m=h.block===null&&p.block!==null,f=h.pos.x+h.size.x+(m?I:0)+C;p.pos.x=Math.max(p.pos.x,f)}},s=()=>{for(let g of t)for(let u of g){if(u.block===null)continue;let h=u.block.loopID!==null?$(this.blocksByID.get(u.block.loopID)):null;if(h){let p=h.layoutNode;u.pos.x=Math.max(u.pos.x,p.pos.x)}}},i=()=>{let g=new Map;for(let u of Q(t)){let h=u.dstBlock,p=u.pos.x;g.set(h,Math.max(g.get(h)??0,p))}for(let u of Q(t)){let h=u.dstBlock,p=g.get(h);k(p,`no position for backedge ${h.id}`),u.pos.x=p}for(let u of t)e(u)},c=()=>{let g=new Map;for(let u of t){let h=0,p=0;for(;h<u.length;h++)if(!(u[h].flags&j)){p=u[h].pos.x;break}for(h-=1,p-=C+I;h>=0;h--){let m=u[h];k(m.block===null&&m.flags&j);let f=p;for(let y of m.srcNodes){let v=y.pos.x+y.dstNodes.indexOf(m)*S;v<f&&(f=v)}m.pos.x=f,p=m.pos.x-C,g.set(m.dstBlock,Math.min(g.get(m.dstBlock)??1/0,f))}}for(let u of Q(t)){if(!(u.flags&j))continue;let h=g.get(u.dstBlock);k(h,`no position for run to block ${u.dstBlock.id}`),u.pos.x=h}},l=()=>{for(let g=0;g<t.length-1;g++){let u=t[g];e(u);let h=-1;for(let p of u)for(let[m,f]of p.dstNodes.entries()){let y=t[g+1].indexOf(f);if(y>h&&f.srcNodes[0]===p){let v=I+S*m,B=I,Z=f.pos.x;f.pos.x=Math.max(f.pos.x,p.pos.x+v-B),f.pos.x!==Z&&(h=y)}}}},o=()=>{for(let g of t){for(let u=g.length-1;u>=0;u--){let h=g[u];if(!h.block||h.block.mir.attributes.includes("backedge"))continue;let p=[];for(let m of h.srcNodes){let f=I+m.dstNodes.indexOf(h)*S,y=I;p.push(m.pos.x+f-(h.pos.x+y))}for(let[m,f]of h.dstNodes.entries()){if(f.block===null&&f.dstBlock.mir.attributes.includes("backedge"))continue;let y=I+m*S,v=I;p.push(f.pos.x+v-(h.pos.x+y))}if(!p.includes(0)){p=p.filter(m=>m>0).sort((m,f)=>m-f);for(let m of p){let f=!1;for(let y=u+1;y<g.length;y++){let v=g[y];if(v.flags<)continue;let B=h.pos.x+m,Z=h.pos.x+m+h.size.x,ht=v.pos.x-C,pt=v.pos.x+v.size.x+C;Z>=ht&&B<=pt&&(f=!0)}if(!f){h.pos.x+=m;break}}}}e(g)}},n=()=>{for(let g=t.length-1;g>=0;g--){let u=t[g];e(u);for(let h of u)for(let p of h.srcNodes){if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}},r=()=>{for(let g=0;g<t.length;g++){let u=t[g];e(u);for(let h of u){if(h.dstNodes.length===0)continue;let p=h.dstNodes[0];if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}};function d(g,u){let h=[];for(let p=0;p<u;p++)for(let m of g)h.push(m);return h}let b=[...d([l,s,i],Tt),i,...d([n,r],$t),o,i,c];k(b.length<=(at.initial??1/0),`STOP_AT_PASS was too small - should be at least ${b.length}`),w.group("Running passes");for(let[g,u]of b.entries())g<at&&(w.log(u.name??u.toString()),u());w.groupEnd()}finagleJoints(t){let e=[];for(let s of t){let i=[];for(let r of s)if(r.jointOffsets=new Array(r.dstNodes.length).fill(0),!r.block?.mir.attributes.includes("backedge"))for(let[d,b]of r.dstNodes.entries()){let g=r.pos.x+I+S*d,u=b.pos.x+I;Math.abs(u-g)<2*T||i.push({x1:g,x2:u,src:r,srcPort:d,dst:b})}i.sort((r,d)=>r.x1-d.x1);let c=[],l=[];t:for(let r of i){let d=r.x2-r.x1>=0?c:l,b=null;for(let g=d.length-1;g>=0;g--){let u=d[g],h=!1;for(let p of u){if(r.dst===p.dst){u.push(r);continue t}let m=Math.min(r.x1,r.x2),f=Math.max(r.x1,r.x2),y=Math.min(p.x1,p.x2),v=Math.max(p.x1,p.x2);if(f>=y&&m<=v){h=!0;break}}if(h)break;b=u}b?b.push(r):d.push([r])}let o=Math.max(0,c.length+l.length-1)*it,n=-o/2;for(let r of[...c.reverse(),...l]){for(let d of r)d.src.jointOffsets[d.srcPort]=n;n+=it}e.push(o)}return k(e.length===t.length),e}verticalize(t,e){let s=new Array(t.length),i=P;for(let c=0;c<t.length;c++){let l=t[c],o=0;for(let n of l)n.pos.y=i,o=Math.max(o,n.size.y);s[c]=o,i+=o+H+e[c]+H}return s}renderBlock(t){let e=document.createElement("div");this.graphContainer.appendChild(e),e.classList.add("ig-block","ig-bg-white");for(let o of t.mir.attributes)e.classList.add(`ig-block-att-${o}`);e.setAttribute("data-ig-block-ptr",`${t.ptr}`),e.setAttribute("data-ig-block-id",`${t.id}`);let s="";t.mir.attributes.includes("loopheader")?s=" (loop header)":t.mir.attributes.includes("backedge")?s=" (backedge)":t.mir.attributes.includes("splitedge")&&(s=" (split edge)");let i=document.createElement("div");i.classList.add("ig-block-header"),i.innerText=`Block ${t.id}${s}`,e.appendChild(i);let c=document.createElement("div");c.classList.add("ig-instructions"),e.appendChild(c);let l=document.createElement("table");if(t.lir){l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + ${this.sampleCounts?` + <col style="width: 1px"> + <col style="width: 1px"> + `:""} + </colgroup> + ${this.sampleCounts?` + <thead> + <tr> + <th></th> + <th></th> + <th class="ig-f6">Total</th> + <th class="ig-f6">Self</th> + </tr> + </thead> + `:""} + `;for(let o of t.lir.instructions)l.appendChild(this.renderLIRInstruction(o))}else{l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + <col style="width: 1px"> + </colgroup> + `;for(let o of t.mir.instructions)l.appendChild(this.renderMIRInstruction(o))}if(c.appendChild(l),t.succs.length===2)for(let[o,n]of[1,0].entries()){let r=document.createElement("div");r.innerText=`${n}`,r.classList.add("ig-edge-label"),r.style.left=`${I+S*o}px`,e.appendChild(r)}return i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",o=>{o.stopPropagation(),o.shiftKey||this.selectedBlockPtrs.clear(),this.setSelection([],t.ptr)}),e}render(t,e,s){for(let n of t)for(let r of n)if(r.block!==null){let d=r.block;d.el.style.left=`${r.pos.x}px`,d.el.style.top=`${r.pos.y}px`}let i=0,c=0;for(let n of t)for(let r of n)i=Math.max(i,r.pos.x+r.size.x+P),c=Math.max(c,r.pos.y+r.size.y+P);let l=document.createElementNS("http://www.w3.org/2000/svg","svg");this.graphContainer.appendChild(l);let o=(n,r)=>{for(let d of n)i=Math.max(i,d+P);for(let d of r)c=Math.max(c,d+P)};for(let n=0;n<t.length;n++){let r=t[n];for(let d of r){d.block||k(d.dstNodes.length===1,`dummy nodes must have exactly one destination, but dummy ${d.id} had ${d.dstNodes.length}`),k(d.dstNodes.length===d.jointOffsets.length,"must have a joint offset for each destination");for(let[b,g]of d.dstNodes.entries()){let u=d.pos.x+I+S*b,h=d.pos.y+d.size.y;if(d.block?.mir.attributes.includes("backedge")){let p=d.block.succs[0],m=d.pos.x,f=d.pos.y+D,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=jt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(d.flags&U){let p=L(g.block),m=d.pos.x+I,f=d.pos.y+D+T,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=zt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(g.block===null&&g.dstBlock.mir.attributes.includes("backedge")){let p=g.pos.x+I,m=g.pos.y+(g.flags&U?D+T:0);if(d.block===null){let f=h-H,y=Ft(u,h,p,m,f,!1);l.appendChild(y),o([u,p],[h,m,f])}else{let f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=_t(u,h,p,m,f);l.appendChild(y),o([u,p],[h,m,f])}}else{let p=g.pos.x+I,m=g.pos.y,f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=Rt(u,h,p,m,f,g.block!==null);l.appendChild(y),o([u,p],[h,m,f])}}}}if(l.setAttribute("width",`${i}`),l.setAttribute("height",`${c}`),this.size={x:i,y:c},+dt)for(let n of t)for(let r of n){let d=document.createElement("div");d.innerHTML=`${r.id}<br><- ${r.srcNodes.map(b=>b.id)}<br>-> ${r.dstNodes.map(b=>b.id)}<br>${r.flags}`,d.style.position="absolute",d.style.border="1px solid black",d.style.backgroundColor="white",d.style.left=`${r.pos.x}px`,d.style.top=`${r.pos.y}px`,d.style.whiteSpace="nowrap",this.graphContainer.appendChild(d)}this.updateHighlightedInstructions(),this.updateHotness()}renderMIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-mir","ig-can-flash",...t.attributes.map(o=>`ig-ins-att-${o}`)),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=`v${t.id}`,s.appendChild(i);let c=document.createElement("td");c.innerHTML=e.replace(/(v)(\d+)/g,(o,n,r)=>`<span class="ig-use ig-highlightable" data-ig-use="${r}">${n}${r}</span>`),s.appendChild(c);let l=document.createElement("td");return l.classList.add("ig-ins-type"),l.innerText=t.type==="None"?"":t.type,s.appendChild(l),i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),c.querySelectorAll(".ig-use").forEach(o=>{o.addEventListener("pointerdown",n=>{n.preventDefault(),n.stopPropagation()}),o.addEventListener("click",n=>{let r=parseInt(L(o.getAttribute("data-ig-use")),10);this.jumpToInstruction(r,{zoom:1})})}),s}renderLIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-lir","ig-hotness"),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=String(t.id),s.appendChild(i);let c=document.createElement("td");if(c.innerText=e,s.appendChild(c),this.sampleCounts){let l=this.sampleCounts?.totalLineHits.get(t.id)??0,o=this.sampleCounts?.selfLineHits.get(t.id)??0,n=document.createElement("td");n.classList.add("ig-ins-samples"),n.classList.toggle("ig-text-dim",l===0),n.innerText=`${l}`,n.title="Color by total count",s.appendChild(n);let r=document.createElement("td");r.classList.add("ig-ins-samples"),r.classList.toggle("ig-text-dim",o===0),r.innerText=`${o}`,r.title="Color by self count",s.appendChild(r);for(let[d,b]of[n,r].entries())b.addEventListener("pointerdown",g=>{g.preventDefault(),g.stopPropagation()}),b.addEventListener("click",()=>{k(d===O||d===W),this.heatmapMode=d,this.updateHotness()})}return i.addEventListener("pointerdown",l=>{l.preventDefault(),l.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),s}renderSelection(){this.graphContainer.querySelectorAll(".ig-block").forEach(t=>{let e=parseInt(L(t.getAttribute("data-ig-block-ptr")),10);t.classList.toggle("ig-selected",this.selectedBlockPtrs.has(e)),t.classList.toggle("ig-last-selected",this.lastSelectedBlockPtr===e)})}removeNonexistentHighlights(){this.highlightedInstructions=this.highlightedInstructions.filter(t=>this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`))}updateHighlightedInstructions(){for(let t of this.highlightedInstructions)k(this.highlightedInstructions.filter(e=>e.ptr===t.ptr).length===1,`instruction ${t.ptr} was highlighted more than once`);this.graphContainer.querySelectorAll(".ig-ins, .ig-use").forEach(t=>{Vt(t)});for(let t of this.highlightedInstructions){let e=this.instructionPalette[t.paletteColor%this.instructionPalette.length],s=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`);if(s){ct(s,e);let i=this.insIDsByPtr.get(t.ptr);this.graphContainer.querySelectorAll(`.ig-use[data-ig-use="${i}"]`).forEach(c=>{ct(c,e)})}}}updateHotness(){this.graphContainer.querySelectorAll(".ig-ins-lir").forEach(t=>{k(t.classList.contains("ig-hotness"));let e=parseInt(L(t.getAttribute("data-ig-ins-id")),10),s=0;this.sampleCounts&&(s=((this.heatmapMode===O?this.sampleCounts.totalLineHits:this.sampleCounts.selfLineHits).get(e)??0)/this.maxSampleCounts[this.heatmapMode]),t.style.setProperty("--ig-hotness",`${s}`)})}addEventListeners(){this.viewport.addEventListener("wheel",e=>{e.preventDefault();let s=this.zoom;if(e.ctrlKey){s=Math.max(Ht,Math.min(Ct,this.zoom*Math.pow(Mt,-e.deltaY*Et)));let c=s/this.zoom-1;this.zoom=s;let{x:l,y:o}=this.viewport.getBoundingClientRect(),n=e.clientX-l-this.translation.x,r=e.clientY-o-this.translation.y;this.translation.x-=n*c,this.translation.y-=r*c}else this.translation.x-=e.deltaX,this.translation.y-=e.deltaY;let i=this.clampTranslation(this.translation,s);this.translation.x=i.x,this.translation.y=i.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerdown",e=>{e.pointerType==="mouse"&&!(e.button===0||e.button===1)||(e.preventDefault(),this.viewport.setPointerCapture(e.pointerId),this.startMousePos={x:e.clientX,y:e.clientY},this.lastMousePos={x:e.clientX,y:e.clientY},this.animating=!1)}),this.viewport.addEventListener("pointermove",e=>{if(!this.viewport.hasPointerCapture(e.pointerId))return;let s=e.clientX-this.lastMousePos.x,i=e.clientY-this.lastMousePos.y;this.translation.x+=s,this.translation.y+=i,this.lastMousePos={x:e.clientX,y:e.clientY};let c=this.clampTranslation(this.translation,this.zoom);this.translation.x=c.x,this.translation.y=c.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerup",e=>{this.viewport.releasePointerCapture(e.pointerId);let s=2,i=this.startMousePos.x-e.clientX,c=this.startMousePos.y-e.clientY;Math.abs(i)<=s&&Math.abs(c)<=s&&this.setSelection([]),this.animating=!1}),new ResizeObserver(e=>{k(e.length===1);let s=e[0].contentRect;this.viewportSize.x=s.width,this.viewportSize.y=s.height}).observe(this.viewport)}setSelection(t,e=0){this.setSelectionRaw(t,e),e?this.nav={visited:[e],currentIndex:0,siblings:[e]}:this.nav={visited:[],currentIndex:-1,siblings:[]}}setSelectionRaw(t,e){this.selectedBlockPtrs.clear();for(let s of[...t,e])this.blocksByPtr.has(s)&&this.selectedBlockPtrs.add(s);this.lastSelectedBlockPtr=this.blocksByPtr.has(e)?e:0,this.renderSelection()}navigate(t){let e=this.lastSelectedBlockPtr;if(t==="down"||t==="up")if(e){let s=L(this.blocksByPtr.get(e)),i=(t==="down"?s.succs:s.preds).map(l=>l.ptr);s.ptr!==this.nav.visited[this.nav.currentIndex]&&(this.nav.visited=[s.ptr],this.nav.currentIndex=0);let c=this.nav.currentIndex+(t==="down"?1:-1);if(0<=c&&c<this.nav.visited.length)this.nav.currentIndex=c,this.nav.siblings=i;else{let l=i[0];l!==void 0&&(t==="down"?(this.nav.visited.push(l),this.nav.currentIndex+=1,k(this.nav.currentIndex===this.nav.visited.length-1)):(this.nav.visited.unshift(l),k(this.nav.currentIndex===0)),this.nav.siblings=i)}this.setSelectionRaw([],this.nav.visited[this.nav.currentIndex])}else{let s=[...this.blocks].sort((n,r)=>n.id-r.id),i=s.filter(n=>n.preds.length===0),c=s.filter(n=>n.succs.length===0),l=t==="down"?i:c,o=l[0];k(o),this.setSelectionRaw([],o.ptr),this.nav={visited:[o.ptr],currentIndex:0,siblings:l.map(n=>n.ptr)}}else if(e!==void 0){let s=this.nav.siblings.indexOf(e);k(s>=0,"currently selected node should be in siblings array");let i=s+(t==="right"?1:-1);0<=i&&i<this.nav.siblings.length&&this.setSelectionRaw([],this.nav.siblings[i])}k(this.nav.visited.length===0||this.nav.siblings.includes(this.nav.visited[this.nav.currentIndex]),"expected currently visited node to be in the siblings array"),k(this.lastSelectedBlockPtr===0||this.nav.siblings.includes(this.lastSelectedBlockPtr),"expected currently selected block to be in siblings array")}toggleInstructionHighlight(t,e){this.removeNonexistentHighlights();let s=this.highlightedInstructions.findIndex(c=>c.ptr===t),i=s>=0;if(e!==void 0&&(i=!e),i)s>=0&&this.highlightedInstructions.splice(s,1);else if(s<0){let c=0;for(;;){if(this.highlightedInstructions.find(l=>l.paletteColor===c)){c+=1;continue}break}this.highlightedInstructions.push({ptr:t,paletteColor:c})}this.updateHighlightedInstructions()}clampTranslation(t,e){let s=z-this.size.x*e,i=this.viewportSize.x-z,c=z-this.size.y*e,l=this.viewportSize.y-z,o=q(t.x,s,i),n=q(t.y,c,l);return{x:o,y:n}}updatePanAndZoom(){let t=this.clampTranslation(this.translation,this.zoom);this.graphContainer.style.transform=`translate(${t.x}px, ${t.y}px) scale(${this.zoom})`}graph2viewport(t,e=this.translation,s=this.zoom){return{x:t.x*s+e.x,y:t.y*s+e.y}}viewport2graph(t,e=this.translation,s=this.zoom){return{x:(t.x-e.x)/s,y:(t.y-e.y)/s}}async goToGraphCoordinates(t,{zoom:e=this.zoom,animate:s=!0}){let i={x:-t.x*e,y:-t.y*e};if(!s){this.animating=!1,this.translation.x=i.x,this.translation.y=i.y,this.zoom=e,this.updatePanAndZoom(),await new Promise(l=>setTimeout(l,0));return}if(this.targetTranslation=i,this.targetZoom=e,this.animating)return;this.animating=!0;let c=performance.now();for(;this.animating;){let l=await new Promise(h=>requestAnimationFrame(h)),o=(l-c)/1e3;c=l;let n=1,r=.01,d=1e-6,b=this.targetTranslation.x-this.translation.x,g=this.targetTranslation.y-this.translation.y,u=this.targetZoom-this.zoom;if(this.translation.x=R(this.translation.x,this.targetTranslation.x,d,o),this.translation.y=R(this.translation.y,this.targetTranslation.y,d,o),this.zoom=R(this.zoom,this.targetZoom,d,o),this.updatePanAndZoom(),Math.abs(b)<=n&&Math.abs(g)<=n&&Math.abs(u)<=r){this.translation.x=this.targetTranslation.x,this.translation.y=this.targetTranslation.y,this.zoom=this.targetZoom,this.animating=!1,this.updatePanAndZoom();break}}await new Promise(l=>setTimeout(l,0))}jumpToBlock(t,{zoom:e=this.zoom,animate:s=!0,viewportPos:i}={}){let c=this.blocksByPtr.get(t);if(!c)return Promise.resolve();let l;return i?l={x:c.layoutNode.pos.x-i.x/e,y:c.layoutNode.pos.y-i.y/e}:l=this.graphPosToCenterRect(c.layoutNode.pos,c.layoutNode.size,e),this.goToGraphCoordinates(l,{zoom:e,animate:s})}async jumpToInstruction(t,{zoom:e=this.zoom,animate:s=!0}){let i=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-id="${t}"]`);if(!i)return;let c=i.getBoundingClientRect(),l=this.graphContainer.getBoundingClientRect(),o=(c.x-l.x)/this.zoom,n=(c.y-l.y)/this.zoom,r=c.width/this.zoom,d=c.height/this.zoom,b=this.graphPosToCenterRect({x:o,y:n},{x:r,y:d},e);i.classList.add("ig-flash"),await this.goToGraphCoordinates(b,{zoom:e,animate:s}),i.classList.remove("ig-flash")}graphPosToCenterRect(t,e,s){let i=this.viewportSize.x/s,c=this.viewportSize.y/s,l=Math.max(20/s,(i-e.x)/2),o=Math.max(20/s,(c-e.y)/2),n=t.x-l,r=t.y-o;return{x:n,y:r}}exportState(){let t={translation:this.translation,zoom:this.zoom,heatmapMode:this.heatmapMode,highlightedInstructions:this.highlightedInstructions,selectedBlockPtrs:this.selectedBlockPtrs,lastSelectedBlockPtr:this.lastSelectedBlockPtr,viewportPosOfSelectedBlock:void 0};return this.lastSelectedBlockPtr&&(t.viewportPosOfSelectedBlock=this.graph2viewport(L(this.blocksByPtr.get(this.lastSelectedBlockPtr)).layoutNode.pos)),t}restoreState(t,e){this.translation.x=t.translation.x,this.translation.y=t.translation.y,this.zoom=t.zoom,this.heatmapMode=t.heatmapMode,this.highlightedInstructions=t.highlightedInstructions,this.setSelection(Array.from(t.selectedBlockPtrs),t.lastSelectedBlockPtr),this.updatePanAndZoom(),this.updateHotness(),this.updateHighlightedInstructions(),e.preserveSelectedBlockPosition&&this.jumpToBlock(this.lastSelectedBlockPtr,{zoom:this.zoom,animate:!1,viewportPos:t.viewportPosOfSelectedBlock})}};function Ot(a){for(let t of a.dstNodes){let e=t.srcNodes.indexOf(a);k(e!==-1),t.srcNodes.splice(e,1)}}function*Q(a){for(let t of a)for(let e of t)e.block===null&&(yield e)}function*At(a){for(let t of a)for(let e of t)e.block===null&&e.dstBlock.mir.attributes.includes("backedge")&&(yield e)}function Rt(a,t,e,s,i,c,l=1){let o=T;k(t+o<=i&&i<s-o,`downward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i-o} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${e} ${i+o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,180);r.appendChild(b)}return r}function Ft(a,t,e,s,i,c,l=1){let o=T;k(s+o<=i&&i<=t-o,`upward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i+o} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${e} ${i-o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,0);r.appendChild(b)}return r}function zt(a,t,e,s,i=1){let c=T;k(t-c>=s&&a-c>=e,`to backedge: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, r = ${c} `,!0),i%2===1&&(a+=.5,s+=.5);let l="";l+=`M ${a} ${t} `,l+=`A ${c} ${c} 0 0 0 ${a-c} ${s} `,l+=`L ${e} ${s} `;let o=document.createElementNS("http://www.w3.org/2000/svg","g"),n=document.createElementNS("http://www.w3.org/2000/svg","path");n.setAttribute("d",l),n.setAttribute("fill","none"),n.setAttribute("stroke","black"),n.setAttribute("stroke-width",`${i} `),o.appendChild(n);let r=G(e,s,270);return o.appendChild(r),o}function _t(a,t,e,s,i,c=1){let l=T;k(t+l<=i&&a<=e&&s<=t,`block to backedge dummy: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${l} `,!0),c%2===1&&(a+=.5,e+=.5,i+=.5);let o="";o+=`M ${a} ${t} `,o+=`L ${a} ${i-l} `,o+=`A ${l} ${l} 0 0 0 ${a+l} ${i} `,o+=`L ${e-l} ${i} `,o+=`A ${l} ${l} 0 0 0 ${e} ${i-l} `,o+=`L ${e} ${s} `;let n=document.createElementNS("http://www.w3.org/2000/svg","g"),r=document.createElementNS("http://www.w3.org/2000/svg","path");return r.setAttribute("d",o),r.setAttribute("fill","none"),r.setAttribute("stroke","black"),r.setAttribute("stroke-width",`${c} `),n.appendChild(r),n}function jt(a,t,e,s,i=1){k(e<a&&s===t,`x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s} `,!0),i%2===1&&(t+=.5,s+=.5);let c="";c+=`M ${a} ${t} `,c+=`L ${e} ${s} `;let l=document.createElementNS("http://www.w3.org/2000/svg","g"),o=document.createElementNS("http://www.w3.org/2000/svg","path");o.setAttribute("d",c),o.setAttribute("fill","none"),o.setAttribute("stroke","black"),o.setAttribute("stroke-width",`${i} `),l.appendChild(o);let n=G(e,s,270);return l.appendChild(n),l}function G(a,t,e,s=5){let i=document.createElementNS("http://www.w3.org/2000/svg","path");return i.setAttribute("d",`M 0 0 L ${-s} ${s*1.5} L ${s} ${s*1.5} Z`),i.setAttribute("transform",`translate(${a}, ${t}) rotate(${e})`),i}function ct(a,t){a.classList.add("ig-highlight"),a.style.setProperty("--ig-highlight-color",t)}function Vt(a){a.classList.remove("ig-highlight"),a.style.setProperty("--ig-highlight-color","transparent")}var A=class{constructor(t,{func:e,pass:s=0,sampleCounts:i}){this.graph=null,this.func=e,this.passNumber=s,this.sampleCounts=i,this.keyPasses=[null,null,null,null];{let c=null;for(let[l,o]of e.passes.entries())o.mir.blocks.length>0&&(this.keyPasses[0]===null&&(this.keyPasses[0]=l),o.lir.blocks.length===0&&(this.keyPasses[1]=l)),o.lir.blocks.length>0&&(c?.lir.blocks.length===0&&(this.keyPasses[2]=l),this.keyPasses[3]=l),c=o}this.redundantPasses=[];{let c=null;for(let[l,o]of e.passes.entries()){if(c===null){c=o;continue}E(c.mir,o.mir)&&E(c.lir,o.lir)&&this.redundantPasses.push(l),c=o}}this.viewport=x("div",["ig-flex-grow-1","ig-overflow-hidden"],c=>{c.style.position="relative"}),this.sidebarLinks=e.passes.map((c,l)=>x("a",["ig-link-normal","ig-pv1","ig-ph2","ig-flex","ig-g2"],o=>{o.href="#",o.addEventListener("click",n=>{n.preventDefault(),this.switchPass(l)})},[x("div",["ig-w1","ig-tr","ig-f6","ig-text-dim"],o=>{o.style.paddingTop="0.08rem"},[`${l}`]),x("div",[this.redundantPasses.includes(l)&&"ig-text-dim"],()=>{},[c.name])])),this.container=x("div",["ig-absolute","ig-absolute-fill","ig-flex"],()=>{},[x("div",["ig-w5","ig-br","ig-flex-shrink-0","ig-overflow-y-auto","ig-bg-white"],()=>{},[...this.sidebarLinks]),this.viewport]),t.appendChild(this.container),this.keydownHandler=this.keydownHandler.bind(this),this.tweakHandler=this.tweakHandler.bind(this),window.addEventListener("keydown",this.keydownHandler),window.addEventListener("tweak",this.tweakHandler),this.update()}destroy(){this.container.remove(),window.removeEventListener("keydown",this.keydownHandler),window.removeEventListener("tweak",this.tweakHandler)}update(){for(let[s,i]of this.sidebarLinks.entries())i.classList.toggle("ig-bg-primary",this.passNumber===s);let t=this.graph?.exportState();this.viewport.innerHTML="",this.graph=null;let e=this.func.passes[this.passNumber];if(e)try{this.graph=new V(this.viewport,e,{sampleCounts:this.sampleCounts}),t&&this.graph.restoreState(t,{preserveSelectedBlockPosition:!0})}catch(s){this.viewport.innerHTML="An error occurred while laying out the graph. See console.",console.error(s)}}switchPass(t){this.passNumber=t,this.update()}keydownHandler(t){switch(t.key){case"w":case"s":this.graph?.navigate(t.key==="s"?"down":"up"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"a":case"d":this.graph?.navigate(t.key==="d"?"right":"left"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"f":for(let e=this.passNumber+1;e<this.func.passes.length;e++)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"r":for(let e=this.passNumber-1;e>=0;e--)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"1":case"2":case"3":case"4":{let e=["1","2","3","4"].indexOf(t.key),s=this.keyPasses[e];typeof s=="number"&&this.switchPass(s)}break;case"c":{let e=this.graph?.blocksByPtr.get(this.graph?.lastSelectedBlockPtr??-1);e&&this.graph?.jumpToBlock(e.ptr,{zoom:1})}break}}tweakHandler(){this.update()}};var M=new URL(window.location.toString()).searchParams,Gt=M.has("func")?parseInt(M.get("func"),10):void 0,ut=M.has("pass")?parseInt(M.get("pass"),10):void 0,Y=class{constructor(t){this.exportButton=null,this.ionjson=null,this.funcIndex=Gt??0,this.funcSelected=t.funcSelected,this.funcSelector=x("div",[],()=>{},["Function",x("input",["ig-w3"],e=>{e.type="number",e.min="1",e.addEventListener("input",()=>{this.switchFunc(parseInt(e.value,10)-1)})},[])," / ",x("span",["num-functions"])]),this.funcSelectorNone=x("div",[],()=>{},["No functions to display."]),this.funcName=x("div"),this.root=x("div",["ig-bb","ig-flex","ig-bg-white"],()=>{},[x("div",["ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center","ig-br","ig-hide-if-empty"],()=>{},[t.browse&&x("div",[],()=>{},[x("input",[],e=>{e.type="file",e.addEventListener("change",s=>{let i=s.target;i.files?.length&&this.fileSelected(i.files[0])})})]),this.funcSelector,this.funcSelectorNone]),x("div",["ig-flex-grow-1","ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center"],()=>{},[this.funcName,x("div",["ig-flex-grow-1"]),t.export&&x("div",[],()=>{},[x("button",[],e=>{this.exportButton=e,e.addEventListener("click",()=>{this.exportStandalone()})},["Export"])])])]),this.update()}async fileSelected(t){let e=JSON.parse(await t.text());this.ionjson=X(e),this.switchFunc(0),this.update()}switchIonJSON(t){this.ionjson=t,this.switchFunc(this.funcIndex)}switchFunc(t){t=Math.max(0,Math.min(this.numFunctions()-1,t)),this.funcIndex=isNaN(t)?0:t,this.funcSelected(this.ionjson?.functions[this.funcIndex]??null),this.update()}numFunctions(){return this.ionjson?.functions.length??0}update(){let t=0<=this.funcIndex&&this.funcIndex<this.numFunctions();this.funcSelector.hidden=this.numFunctions()<=1,this.funcSelectorNone.hidden=!(this.ionjson&&this.numFunctions()===0);let e=this.funcSelector.querySelector("input");e.max=`${this.numFunctions()}`,e.value=`${this.funcIndex+1}`,this.funcSelector.querySelector(".num-functions").innerHTML=`${this.numFunctions()}`,this.funcName.hidden=!t,this.funcName.innerText=`${this.ionjson?.functions[this.funcIndex].name??""}`,this.exportButton&&(this.exportButton.disabled=!this.ionjson||!t)}async exportStandalone(){let t=L(this.ionjson),e=t.functions[this.funcIndex].name,s={version:1,functions:[t.functions[this.funcIndex]]},c=(await(await fetch("./standalone.html")).text()).replace(/\{\{\s*IONJSON\s*\}\}/,JSON.stringify(s)),l=URL.createObjectURL(new Blob([c],{type:"text/html;charset=utf-8"})),o=document.createElement("a");o.href=l,o.download=`iongraph-${e}.html`,document.body.appendChild(o),o.click(),o.remove(),URL.revokeObjectURL(l)}},tt=class{constructor(){this.menuBar=new Y({browse:!0,export:!0,funcSelected:t=>this.switchFunc(t)}),this.func=null,this.sampleCountsFromFile=void 0,this.graph=null,this.loadStuffFromQueryParams(),this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer]),this.update()}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut,sampleCounts:this.sampleCountsFromFile}))}loadStuffFromQueryParams(){(async()=>{let t=M.get("file");if(t){let s=await(await fetch(t)).json(),i=X(s);this.menuBar.switchIonJSON(i)}})(),(async()=>{let t=M.get("sampleCounts");if(t){let s=await(await fetch(t)).json();this.sampleCountsFromFile={selfLineHits:new Map(s.selfLineHits),totalLineHits:new Map(s.totalLineHits)},this.update()}})()}switchFunc(t){this.func=t,this.update()}},et=class{constructor(){this.menuBar=new Y({funcSelected:t=>this.switchFunc(t)}),this.func=null,this.graph=null,this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer])}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut}))}setIonJSON(t){this.menuBar.switchIonJSON(t)}switchFunc(t){this.func=t,this.update()}};return yt(Yt);})(); +</script> + <script>window.__exportedIonJSON = {{ IONJSON }}</script> + <script> + const ui = new iongraph.StandaloneUI(); + document.body.appendChild(ui.root); + ui.setIonJSON(window.__exportedIonJSON); + </script> +</body> diff --git a/tool/zjit_iongraph.rb b/tool/zjit_iongraph.rb new file mode 100755 index 0000000000..0cb7701614 --- /dev/null +++ b/tool/zjit_iongraph.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +require 'json' +require 'logger' + +LOGGER = Logger.new($stderr) + +def run_ruby *cmd + # Find the first --zjit* option and add --zjit-dump-hir-iongraph after it + zjit_index = cmd.find_index { |arg| arg.start_with?("--zjit") } + raise "No --zjit option found in command" unless zjit_index + cmd.insert(zjit_index + 1, "--zjit-dump-hir-iongraph") + pid = Process.spawn(*cmd) + _, status = Process.wait2(pid) + if status.exitstatus != 0 + LOGGER.warn("Command failed with exit status #{status.exitstatus}") + end + pid +end + +usage = "Usage: zjit_iongraph.rb <path_to_ruby> <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +pid = run_ruby(RUBY, *OPTIONS) +functions = Dir["/tmp/zjit-iongraph-#{pid}/fun*.json"].map do |path| + JSON.parse(File.read(path)) +end + +if functions.empty? + LOGGER.warn("No iongraph functions found for PID #{pid}") +end + +json = JSON.dump({version: 1, functions: functions}) +# Get zjit_iongraph.html from the sibling file next to this script +html = File.read(File.join(File.dirname(__FILE__), "zjit_iongraph.html")) +html.sub!("{{ IONJSON }}", json) +output_path = "zjit_iongraph_#{pid}.html" +File.write(output_path, html) +puts "Wrote iongraph to #{output_path}" |