diff options
Diffstat (limited to 'tool')
342 files changed, 13952 insertions, 16887 deletions
diff --git a/tool/annocheck/Dockerfile b/tool/annocheck/Dockerfile index 138adc48de..d1fb1839c9 100644 --- a/tool/annocheck/Dockerfile +++ b/tool/annocheck/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/fedora:latest +FROM ghcr.io/ruby/fedora:latest RUN dnf -y install annobin-annocheck WORKDIR /work diff --git a/tool/annocheck/Dockerfile-copy b/tool/annocheck/Dockerfile-copy index e658d12ddc..d437f27387 100644 --- a/tool/annocheck/Dockerfile-copy +++ b/tool/annocheck/Dockerfile-copy @@ -1,7 +1,6 @@ -FROM docker.io/fedora:latest -ARG FILES +FROM ghcr.io/ruby/fedora:latest +ARG IN_DIR RUN dnf -y install annobin-annocheck -RUN mkdir /work -COPY ${FILES} /work +COPY ${IN_DIR} /work WORKDIR /work diff --git a/tool/auto-style.rb b/tool/auto-style.rb new file mode 100755 index 0000000000..3b93c8c317 --- /dev/null +++ b/tool/auto-style.rb @@ -0,0 +1,284 @@ +#!/usr/bin/env ruby +# Usage: +# auto-style.rb oldrev newrev [pushref] + +require 'shellwords' +require 'tmpdir' +ENV['LC_ALL'] = 'C' + +class Git + attr_reader :depth + + def initialize(oldrev, newrev, branch = nil) + @oldrev = oldrev + @newrev = !newrev || newrev.empty? ? 'HEAD' : newrev + @branch = branch + + return unless oldrev + + # GitHub may not fetch github.event.pull_request.base.sha at checkout + git('log', '--format=%H', '-1', @oldrev, out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=1', 'origin', @oldrev) + git('log', '--format=%H', '-1', "#@newrev~99", out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=100', 'origin', @newrev) + + with_clean_env do + @revs = {} + IO.popen(['git', 'log', '--format=%H %s', "#{@oldrev}..#{@newrev}"]) do |f| + f.each do |line| + line.chomp! + rev, subj = line.split(' ', 2) + @revs[rev] = subj + end + end + @depth = @revs.size + end + end + + # ["foo/bar.c", "baz.h", ...] + def updated_paths + with_clean_env do + IO.popen(['git', 'diff', '--name-only', @oldrev, @newrev], &:readlines).each(&:chomp!) + end + end + + # [0, 1, 4, ...] + def updated_lines(file) # NOTE: This doesn't work well on pull requests, so not used anymore + lines = [] + revs = @revs.map {|rev, subj| rev unless subj.start_with?("Revert ")}.compact + revs_pattern = /\A(?:#{revs.join('|')}) / + with_clean_env { IO.popen(['git', 'blame', '-l', '--', file], &:readlines) }.each_with_index do |line, index| + if revs_pattern =~ line + lines << index + end + end + lines + end + + def commit(log, *files) + git('add', *files) + git('commit', '-m', log) + end + + def push + git('push', 'origin', @branch) if @branch + end + + def diff + git('--no-pager', 'diff') + end + + private + + def git(*args, **opts) + cmd = ['git', *args] + puts "+ #{cmd.shelljoin}" + ret = with_clean_env { system(*cmd, **opts) } + unless ret or opts[:err] + abort "Failed to run: #{cmd}" + end + ret + end + + def with_clean_env + git_dir = ENV.delete('GIT_DIR') # this overcomes '-C' or pwd + yield + ensure + ENV['GIT_DIR'] = git_dir if git_dir + end +end + +DEFAULT_GEM_LIBS = %w[ + bundler + cmath + csv + e2mmap + fileutils + forwardable + ipaddr + irb + logger + matrix + mutex_m + ostruct + prime + rdoc + rexml + rss + scanf + shell + sync + thwait + tracer + webrick +] + +DEFAULT_GEM_EXTS = %w[ + bigdecimal + date + dbm + digest + etc + fcntl + fiddle + gdbm + io/console + io/nonblock + json + openssl + psych + racc + sdbm + stringio + strscan + zlib +] + +IGNORED_FILES = [ + # default gems whose master is GitHub + %r{\Abin/(?!erb)\w+\z}, + *(DEFAULT_GEM_LIBS + DEFAULT_GEM_EXTS).flat_map { |lib| + [ + %r{\Alib/#{lib}/}, + %r{\Alib/#{lib}\.gemspec\z}, + %r{\Alib/#{lib}\.rb\z}, + %r{\Atest/#{lib}/}, + ] + }, + *DEFAULT_GEM_EXTS.flat_map { |ext| + [ + %r{\Aext/#{ext}/}, + %r{\Atest/#{ext}/}, + ] + }, + + # vendoring (ccan) + %r{\Accan/}, + + # vendoring (io/) + %r{\Aext/io/}, + + # vendoring (nkf) + %r{\Aext/nkf/nkf-utf8/}, + + # vendoring (onigmo) + %r{\Aenc/}, + %r{\Ainclude/ruby/onigmo\.h\z}, + %r{\Areg.+\.(c|h)\z}, + + # explicit or implicit `c-file-style: "linux"` + %r{\Aaddr2line\.c\z}, + %r{\Amissing/}, + %r{\Astrftime\.c\z}, + %r{\Avsnprintf\.c\z}, + + # to respect the original statements of licenses + %r{\ALEGAL\z}, + + # trailing spaces could be intentional in TRICK code + %r{\Asample/trick[^/]*/}, +] + +DIFFERENT_STYLE_FILES = %w[ + addr2line.c io_buffer.c prism*.c scheduler.c +] + +def adjust_styles(files) + trailing = eofnewline = expandtab = indent = false + + edited_files = files.select do |f| + src = File.binread(f) rescue next + eofnewline = eofnewline0 = true if src.sub!(/(?<!\A|\n)\z/, "\n") + + trailing0 = false + expandtab0 = false + indent0 = false + + src.gsub!(/^.*$/).with_index do |line, lineno| + trailing = trailing0 = true if line.sub!(/[ \t]+$/, '') + line + end + + if f.end_with?('.c') || f.end_with?('.h') || f == 'insns.def' + # If and only if unedited lines did not have tab indentation, prevent introducing tab indentation to the file. + expandtab_allowed = src.each_line.with_index.all? do |line, lineno| + !line.start_with?("\t") + end + + if expandtab_allowed + src.gsub!(/^.*$/).with_index do |line, lineno| + if line.start_with?("\t") # last-committed line with hard tabs + expandtab = expandtab0 = true + line.sub(/\A\t+/) { |tabs| ' ' * (8 * tabs.length) } + else + line + end + end + end + end + + if File.fnmatch?("*.[chy]", f, File::FNM_PATHNAME) && + !DIFFERENT_STYLE_FILES.any? {|pat| File.fnmatch?(pat, f, File::FNM_PATHNAME)} + indent0 = true if src.gsub!(/^\w+\([^\n]*?\)\K[ \t]*(?=\{( *\\)?$)/, '\1' "\n") + indent0 = true if src.gsub!(/^([ \t]*)\}\K[ \t]*(?=else\b.*?( *\\)?$)/, '\2' "\n" '\1') + indent0 = true if src.gsub!(/^[ \t]*\}\n\K\n+(?=[ \t]*else\b)/, '') + indent ||= indent0 + end + + if trailing0 or eofnewline0 or expandtab0 or indent0 + File.binwrite(f, src) + true + end + end + if edited_files.empty? + return + else + msg = [('remove trailing spaces' if trailing), + ('append newline at EOF' if eofnewline), + ('expand tabs' if expandtab), + ('adjust indents' if indent), + ].compact + message = "* #{msg.join(', ')}. [ci skip]" + if expandtab + message += "\nPlease consider using misc/expand_tabs.rb as a pre-commit hook." + end + return message, edited_files + end +end + +oldrev, newrev, pushref = ARGV +if (dry_run = oldrev == '-n') or oldrev == '--' + _, *updated_files = ARGV + git = Git.new(nil, nil) +else + unless dry_run = pushref.nil? + branch = IO.popen(['git', 'rev-parse', '--symbolic', '--abbrev-ref', pushref], &:read).strip + end + git = Git.new(oldrev, newrev, branch) + + updated_files = git.updated_paths +end + +files = updated_files.select {|l| + /^\d/ !~ l and /\.bat\z/ !~ l and + (/\A(?:config|[Mm]akefile|GNUmakefile|README)/ =~ File.basename(l) or + /\A\z|\.(?:[chsy]|\d+|e?rb|tmpl|bas[eh]|z?sh|in|ma?k|def|src|trans|rdoc|ja|en|el|sed|awk|p[ly]|scm|mspec|html|rs)\z/ =~ File.extname(l)) +} +files.select! {|n| File.file?(n) } +files.reject! do |f| + IGNORED_FILES.any? { |re| f.match(re) } +end + +if files.empty? + puts "No files are an auto-style target:\n#{updated_files.join("\n")}" +elsif !(message, edited_files = adjust_styles(files)) + puts "All edited lines are formatted well:\n#{files.join("\n")}" +else + if dry_run + git.diff + abort message + else + git.commit(message, *edited_files) + git.push + end +end diff --git a/tool/auto_review_pr.rb b/tool/auto_review_pr.rb new file mode 100755 index 0000000000..07c98c7e0a --- /dev/null +++ b/tool/auto_review_pr.rb @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'uri' +require_relative './sync_default_gems' + +class GitHubAPIClient + def initialize(token) + @token = token + end + + def get(path) + response = Net::HTTP.get_response(URI("https://api.github.com#{path}"), { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end + + def post(path, body = {}) + body = JSON.dump(body) + response = Net::HTTP.post(URI("https://api.github.com#{path}"), body, { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + 'Content-Type' => 'application/json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end +end + +class AutoReviewPR + REPO = 'ruby/ruby' + + COMMENT_USER = 'github-actions[bot]' + COMMENT_PREFIX = 'The following files are maintained in the following upstream repositories:' + COMMENT_SUFFIX = 'Please file a pull request to the above instead. Thank you!' + + def initialize(client) + @client = client + end + + def review(pr_number) + # Fetch the list of files changed by the PR + changed_files = @client.get("/repos/#{REPO}/pulls/#{pr_number}/files").map { it.fetch(:filename) } + + # Build a Hash: { upstream_repo => files, ... } + upstream_repos = SyncDefaultGems::Repository.group(changed_files) + upstream_repos.delete(nil) # exclude no-upstream files + upstream_repos.delete('prism') if changed_files.include?('prism_compile.c') # allow prism changes in this case + if upstream_repos.empty? + puts "Skipped: The PR ##{pr_number} doesn't have upstream repositories." + return + end + + # Check if the PR is already reviewed + existing_comments = @client.get("/repos/#{REPO}/issues/#{pr_number}/comments") + existing_comments.map! { [it.fetch(:user).fetch(:login), it.fetch(:body)] } + if existing_comments.any? { |user, comment| user == COMMENT_USER && comment.start_with?(COMMENT_PREFIX) } + puts "Skipped: The PR ##{pr_number} already has an automated review comment." + return + end + + # Post a comment + comment = format_comment(upstream_repos) + result = @client.post("/repos/#{REPO}/issues/#{pr_number}/comments", { body: comment }) + puts "Success: #{JSON.pretty_generate(result)}" + end + + private + + # upstream_repos: { upstream_repo => files, ... } + def format_comment(upstream_repos) + comment = +'' + comment << "#{COMMENT_PREFIX}\n\n" + + upstream_repos.each do |upstream_repo, files| + comment << "* https://github.com/ruby/#{upstream_repo}\n" + files.each do |file| + comment << " * #{file}\n" + end + end + + comment << "\n#{COMMENT_SUFFIX}" + comment + end +end + +pr_number = ARGV[0] || abort("Usage: #{$0} <pr_number>") +client = GitHubAPIClient.new(ENV.fetch('GITHUB_TOKEN')) + +AutoReviewPR.new(client).review(pr_number) diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb index 5e5ba94ec2..91ac5628f1 100644 --- a/tool/bundler/dev_gems.rb +++ b/tool/bundler/dev_gems.rb @@ -3,18 +3,18 @@ source "https://rubygems.org" gem "test-unit", "~> 3.0" +gem "test-unit-ruby-core" gem "rake", "~> 13.1" gem "rb_sys" -gem "webrick", "~> 1.6" -gem "turbo_tests", "~> 2.1" -gem "parallel_tests", "< 3.9.0" +gem "turbo_tests", "~> 2.2.3" +gem "parallel_tests", "~> 4.10.1" gem "parallel", "~> 1.19" gem "rspec-core", "~> 3.12" gem "rspec-expectations", "~> 3.12" gem "rspec-mocks", "~> 3.12" -gem "uri", "~> 0.12.0" +gem "rubygems-generate_index", "~> 1.1" group :doc do - gem "nronn", "~> 0.11.1", platform: :ruby + gem "ronn-ng", "~> 0.10.1", platform: :ruby end diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock new file mode 100644 index 0000000000..832127fb4c --- /dev/null +++ b/tool/bundler/dev_gems.rb.lock @@ -0,0 +1,132 @@ +GEM + remote: https://rubygems.org/ + specs: + compact_index (0.15.0) + diff-lcs (1.6.2) + kramdown (2.5.1) + rexml (>= 3.3.9) + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + mini_portile2 (2.8.9) + mustache (1.1.1) + nokogiri (1.19.0) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.19.0-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-java) + racc (~> 1.4) + nokogiri (1.19.0-x64-mingw-ucrt) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-gnu) + racc (~> 1.4) + parallel (1.27.0) + parallel_tests (4.10.1) + parallel + power_assert (3.0.1) + racc (1.8.1) + racc (1.8.1-java) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rexml (3.4.4) + ronn-ng (0.10.1) + kramdown (~> 2, >= 2.1) + kramdown-parser-gfm (~> 1, >= 1.0.1) + mustache (~> 1) + nokogiri (~> 1, >= 1.14.3) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + test-unit (3.7.7) + power_assert + test-unit-ruby-core (1.0.14) + test-unit (>= 3.7.2) + turbo_tests (2.2.5) + parallel_tests (>= 3.3.0, < 5) + rspec (>= 3.10) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm-linux + arm64-darwin + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86-linux + x86_64-darwin + x86_64-linux + +DEPENDENCIES + parallel (~> 1.19) + parallel_tests (~> 4.10.1) + rake (~> 13.1) + rb_sys + ronn-ng (~> 0.10.1) + rspec-core (~> 3.12) + rspec-expectations (~> 3.12) + rspec-mocks (~> 3.12) + rubygems-generate_index (~> 1.1) + test-unit (~> 3.0) + test-unit-ruby-core + turbo_tests (~> 2.2.3) + +CHECKSUMS + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + kramdown (2.5.1) sha256=87bbb6abd9d3cebe4fc1f33e367c392b4500e6f8fa19dd61c0972cf4afe7368c + kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 + mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289 + mustache (1.1.1) sha256=90891fdd50b53919ca334c8c1031eada1215e78d226d5795e523d6123a2717d0 + nokogiri (1.19.0) sha256=e304d21865f62518e04f2bf59f93bd3a97ca7b07e7f03952946d8e1c05f45695 + nokogiri (1.19.0-aarch64-linux-gnu) sha256=11a97ecc3c0e7e5edcf395720b10860ef493b768f6aa80c539573530bc933767 + nokogiri (1.19.0-arm-linux-gnu) sha256=572a259026b2c8b7c161fdb6469fa2d0edd2b61cd599db4bbda93289abefbfe5 + nokogiri (1.19.0-arm64-darwin) sha256=0811dfd936d5f6dd3f6d32ef790568bf29b2b7bead9ba68866847b33c9cf5810 + nokogiri (1.19.0-java) sha256=5f3a70e252be641d8a4099f7fb4cc25c81c632cb594eec9b4b8f2ca8be4374f3 + nokogiri (1.19.0-x64-mingw-ucrt) sha256=05d7ed2d95731edc9bef2811522dc396df3e476ef0d9c76793a9fca81cab056b + nokogiri (1.19.0-x86_64-darwin) sha256=1dad56220b603a8edb9750cd95798bffa2b8dd9dd9aa47f664009ee5b43e3067 + nokogiri (1.19.0-x86_64-linux-gnu) sha256=f482b95c713d60031d48c44ce14562f8d2ce31e3a9e8dd0ccb131e9e5a68b58c + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parallel_tests (4.10.1) sha256=df05458c691462b210f7a41fc2651d4e4e8a881e8190e6d1e122c92c07735d70 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + ronn-ng (0.10.1) sha256=4eeb0185c0fbfa889efed923b5b50e949cd869e7d82ac74138acd0c9c7165ec0 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + test-unit-ruby-core (1.0.14) sha256=d2e997796c9c5c5e8e31ac014f83a473ff5c2523a67cfa491b08893e12d43d22 + turbo_tests (2.2.5) sha256=3fa31497d12976d11ccc298add29107b92bda94a90d8a0a5783f06f05102509f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb index 4d0b21060a..a9b6fda11b 100644 --- a/tool/bundler/rubocop_gems.rb +++ b/tool/bundler/rubocop_gems.rb @@ -4,7 +4,8 @@ source "https://rubygems.org" gem "rubocop", ">= 1.52.1", "< 2" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock new file mode 100644 index 0000000000..d0c3120e11 --- /dev/null +++ b/tool/bundler/rubocop_gems.rb.lock @@ -0,0 +1,159 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) + ast (~> 2.4.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) + rake + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.82.1) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.48.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + stringio (3.2.0) + test-unit (3.7.7) + power_assert + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + irb + minitest (~> 5.1) + rake + rake-compiler + rb_sys + rspec + rubocop (>= 1.52.1, < 2) + test-unit + +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.82.1) sha256=09f1a6a654a960eda767aebea33e47603080f8e9c9a3f019bf9b94c9cab5e273 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb index 20c1ecd827..f7bc34cf5e 100644 --- a/tool/bundler/standard_gems.rb +++ b/tool/bundler/standard_gems.rb @@ -4,7 +4,8 @@ source "https://rubygems.org" gem "standard", "~> 1.0" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock new file mode 100644 index 0000000000..f3792f8611 --- /dev/null +++ b/tool/bundler/standard_gems.rb.lock @@ -0,0 +1,179 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) + ast (~> 2.4.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) + rake + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.81.7) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + rubocop-performance (1.26.1) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (1.13.0) + standard (1.52.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.81.7) + standard-custom (~> 1.0.0) + standard-performance (~> 1.8) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.9.0) + lint_roller (~> 1.1) + rubocop-performance (~> 1.26.0) + stringio (3.2.0) + test-unit (3.7.7) + power_assert + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + irb + minitest (~> 5.1) + rake + rake-compiler + rb_sys + rspec + standard (~> 1.0) + test-unit + +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.81.7) sha256=6fb5cc298c731691e2a414fe0041a13eb1beed7bab23aec131da1bcc527af094 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834 + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + standard (1.52.0) sha256=ec050e63228e31fabe40da3ef96da7edda476f7acdf3e7c2ad47b6e153f6a076 + standard-custom (1.0.2) sha256=424adc84179a074f1a2a309bb9cf7cd6bfdb2b6541f20c6bf9436c0ba22a652b + standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb index aa1cfd09a5..ddc19e2939 100644 --- a/tool/bundler/test_gems.rb +++ b/tool/bundler/test_gems.rb @@ -2,12 +2,17 @@ source "https://rubygems.org" -gem "rack", "~> 2.0" -gem "webrick", "1.7.0" -gem "rack-test", "~> 1.1" +gem "rack", "~> 3.1" +gem "rack-test", "~> 2.1" gem "compact_index", "~> 0.15.0" -gem "sinatra", "~> 3.0" +gem "sinatra", "~> 4.1" gem "rake", "~> 13.1" gem "builder", "~> 3.2" gem "rb_sys" +gem "fiddle" gem "rubygems-generate_index", "~> 1.1" +gem "concurrent-ruby" +gem "psych" +gem "etc", platforms: [:ruby, :windows] +gem "open3" +gem "shellwords" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock new file mode 100644 index 0000000000..fdffc1f09d --- /dev/null +++ b/tool/bundler/test_gems.rb.lock @@ -0,0 +1,106 @@ +GEM + remote: https://rubygems.org/ + specs: + base64 (0.3.0) + builder (3.3.0) + compact_index (0.15.0) + concurrent-ruby (1.3.6) + date (3.5.1) + date (3.5.1-java) + etc (1.4.6) + fiddle (1.1.8) + jar-dependencies (0.5.5) + logger (1.7.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + open3 (0.2.1) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + rack (3.2.4) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + ruby2_keywords (0.0.5) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + shellwords (0.2.2) + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) + tilt (~> 2.0) + stringio (3.2.0) + tilt (2.6.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + builder (~> 3.2) + compact_index (~> 0.15.0) + concurrent-ruby + etc + fiddle + open3 + psych + rack (~> 3.1) + rack-test (~> 2.1) + rake (~> 13.1) + rb_sys + rubygems-generate_index (~> 1.1) + shellwords + sinatra (~> 4.1) + +CHECKSUMS + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + etc (1.4.6) sha256=0f7e9e7842ea5e3c3bd9bc81746ebb8c65ea29e4c42a93520a0d638129c7de01 + fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mustermann (3.0.4) sha256=85fadcb6b3c6493a8b511b42426f904b7f27b282835502233dd154daab13aa22 + open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + rack (3.2.4) sha256=5d74b6f75082a643f43c1e76b419c40f0e5527fcfee1e669ac1e6b73c0ccb6f6 + rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac + rack-session (2.1.1) sha256=0b6dc07dea7e4b583f58a48e8b806d4c9f1c6c9214ebc202ec94562cbea2e4e9 + rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94 + sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + tilt (2.6.1) sha256=35a99bba2adf7c1e362f5b48f9b581cce4edfba98117e34696dde6d308d84770 + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb index 2500e6c800..8d12c5adde 100644 --- a/tool/bundler/vendor_gems.rb +++ b/tool/bundler/vendor_gems.rb @@ -2,14 +2,16 @@ source "https://rubygems.org" -gem "fileutils", "1.7.2" -gem "molinillo", github: "cocoapods/molinillo" -gem "net-http", "0.4.0" -gem "net-http-persistent", "4.0.2" +gem "fileutils", "1.8.0" +gem "molinillo", github: "cocoapods/molinillo", ref: "1d62d7d5f448e79418716dc779a4909509ccda2a" +gem "net-http", "0.7.0" # net-http-0.8.0 is broken with JRuby +gem "net-http-persistent", "4.0.6" gem "net-protocol", "0.2.2" -gem "optparse", "0.4.0" -gem "pub_grub", github: "jhawthorn/pub_grub" -gem "resolv", "0.3.0" -gem "timeout", "0.4.1" -gem "thor", "1.3.0" +gem "optparse", "0.8.0" +gem "pub_grub", github: "jhawthorn/pub_grub", ref: "df6add45d1b4d122daff2f959c9bd1ca93d14261" +gem "resolv", "0.6.2" +gem "securerandom", "0.4.1" +gem "timeout", "0.4.4" +gem "thor", "1.4.0" gem "tsort", "0.2.0" +gem "uri", "1.1.1" diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock index ad3602c984..cc7886e60b 100644 --- a/tool/bundler/vendor_gems.rb.lock +++ b/tool/bundler/vendor_gems.rb.lock @@ -1,71 +1,75 @@ GIT remote: https://github.com/cocoapods/molinillo.git - revision: 6bc3d6045edadf800ba1b634fef15d3574369e60 + revision: 1d62d7d5f448e79418716dc779a4909509ccda2a + ref: 1d62d7d5f448e79418716dc779a4909509ccda2a specs: molinillo (0.8.0) GIT remote: https://github.com/jhawthorn/pub_grub.git - revision: 4250c533895080c356407d1f49619cb90fa2562d + revision: df6add45d1b4d122daff2f959c9bd1ca93d14261 + ref: df6add45d1b4d122daff2f959c9bd1ca93d14261 specs: pub_grub (0.5.0) GEM remote: https://rubygems.org/ specs: - connection_pool (2.4.1) - fileutils (1.7.2) - net-http (0.4.0) + connection_pool (2.5.4) + fileutils (1.8.0) + net-http (0.7.0) uri - net-http-persistent (4.0.2) - connection_pool (~> 2.2) + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) net-protocol (0.2.2) timeout - optparse (0.4.0) - resolv (0.3.0) - thor (1.3.0) - timeout (0.4.1) + optparse (0.8.0) + resolv (0.6.2) + securerandom (0.4.1) + thor (1.4.0) + timeout (0.4.4) tsort (0.2.0) - uri (0.13.0) + uri (1.1.1) PLATFORMS java ruby - universal-java-11 - universal-java-18 - universal-java-19 + universal-java x64-mingw-ucrt - x64-mingw32 - x86_64-darwin-20 + x64-mswin64-140 + x86_64-darwin x86_64-linux DEPENDENCIES - fileutils (= 1.7.2) + fileutils (= 1.8.0) molinillo! - net-http (= 0.4.0) - net-http-persistent (= 4.0.2) + net-http (= 0.7.0) + net-http-persistent (= 4.0.6) net-protocol (= 0.2.2) - optparse (= 0.4.0) + optparse (= 0.8.0) pub_grub! - resolv (= 0.3.0) - thor (= 1.3.0) - timeout (= 0.4.1) + resolv (= 0.6.2) + securerandom (= 0.4.1) + thor (= 1.4.0) + timeout (= 0.4.4) tsort (= 0.2.0) + uri (= 1.1.1) CHECKSUMS - connection_pool (2.4.1) sha256=0f40cf997091f1f04ff66da67eabd61a9fe0d4928b9a3645228532512fab62f4 - fileutils (1.7.2) sha256=36a0fb324218263e52b486ad7408e9a295378fe8edc9fd343709e523c0980631 + connection_pool (2.5.4) sha256=e9e1922327416091f3f6542f5f4446c2a20745276b9aa796dd0bb2fd0ea1e70a + fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02 molinillo (0.8.0) - net-http (0.4.0) sha256=d87a6163ce3c64008bc8764e210d5f4ec9b87ca558a9052eb390b2c2c277f157 - net-http-persistent (4.0.2) sha256=03f827a33857b1d56b4e796957ad19bf5b58367d853fd0a224eb70fba8d02a44 + net-http (0.7.0) sha256=4db7d9f558f8ffd4dcf832d0aefd02320c569c7d4f857def49e585069673a425 + net-http-persistent (4.0.6) sha256=2abb3a04438edf6cb9e0e7e505969605f709eda3e3c5211beadd621a2c84dd5d net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 - optparse (0.4.0) sha256=f584afc034f610ea7b28a9b1a68b0917d34e0da73c40c2b29cd7151c5eb0bade + optparse (0.8.0) sha256=ef6b7fbaf7ec331474f325bc08dd5622e6e1e651007a5341330ee4b08ce734f0 pub_grub (0.5.0) - resolv (0.3.0) sha256=14b917f1bb4f363c81601295b68097bf1ff8b3c4179972c2d174ffb7e997a406 - thor (1.3.0) sha256=1adc7f9e5b3655a68c71393fee8bd0ad088d14ee8e83a0b73726f23cbb3ca7c3 - timeout (0.4.1) sha256=6f1f4edd4bca28cffa59501733a94215407c6960bd2107331f0280d4abdebb9a + resolv (0.6.2) sha256=61efe545cedddeb1b14f77e51f85c85ca66af5098fdbf567fadf32c34590fb14 + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + thor (1.4.0) sha256=8763e822ccb0f1d7bee88cde131b19a65606657b847cc7b7b4b82e772bcd8a3d + timeout (0.4.4) sha256=f0f6f970104b82427cd990680f539b6bbb8b1e55efa913a55c6492935e4e0edb tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f - uri (0.13.0) sha256=26553c2a9399762e1e8bebd4444b4361c4b21298cf1c864b22eeabc9c4998f24 + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 BUNDLED WITH - 2.5.0.dev + 4.0.0.dev diff --git a/tool/commit-email.rb b/tool/commit-email.rb new file mode 100755 index 0000000000..c887f8783e --- /dev/null +++ b/tool/commit-email.rb @@ -0,0 +1,372 @@ +#!/usr/bin/env ruby + +require "optparse" +require "nkf" +require "shellwords" + +CommitEmailInfo = Struct.new( + :author, + :author_email, + :revision, + :entire_sha256, + :date, + :log, + :branch, + :diffs, + :added_files, :deleted_files, :updated_files, + :added_dirs, :deleted_dirs, :updated_dirs, +) + +class GitInfoBuilder + GitCommandFailure = Class.new(RuntimeError) + + def initialize(repo_path) + @repo_path = repo_path + end + + def build(oldrev, newrev, refname) + diffs = build_diffs(oldrev, newrev) + + info = CommitEmailInfo.new + info.author = git_show(newrev, format: '%an') + info.author_email = normalize_email(git_show(newrev, format: '%aE')) + info.revision = newrev[0...10] + info.entire_sha256 = newrev + info.date = Time.at(Integer(git_show(newrev, format: '%at'))) + info.log = git_show(newrev, format: '%B') + info.branch = git('rev-parse', '--symbolic', '--abbrev-ref', refname).strip + info.diffs = diffs + info.added_files = find_files(diffs, status: :added) + info.deleted_files = find_files(diffs, status: :deleted) + info.updated_files = find_files(diffs, status: :modified) + info.added_dirs = [] # git does not deal with directory + info.deleted_dirs = [] # git does not deal with directory + info.updated_dirs = [] # git does not deal with directory + info + end + + private + + # Force git-svn email address to @ruby-lang.org to avoid email bounce by invalid email address. + def normalize_email(email) + if email.match(/\A[^@]+@\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/) # git-svn + svn_user, _ = email.split('@', 2) + "#{svn_user}@ruby-lang.org" + else + email + end + end + + def find_files(diffs, status:) + files = [] + diffs.each do |path, values| + if values.keys.first == status + files << path + end + end + files + end + + # SVN version: + # { + # "filename" => { + # "[modified|added|deleted|copied|property_changed]" => { + # type: "[modified|added|deleted|copied|property_changed]", + # body: "diff body", # not implemented because not used + # added: Integer, + # deleted: Integer, + # } + # } + # } + def build_diffs(oldrev, newrev) + diffs = {} + + numstats = git('diff', '--numstat', oldrev, newrev).lines.map { |l| l.strip.split("\t", 3) } + git('diff', '--name-status', oldrev, newrev).each_line do |line| + status, path, _newpath = line.strip.split("\t", 3) + diff = build_diff(path, numstats) + + case status + when 'A' + diffs[path] = { added: { type: :added, **diff } } + when 'M' + diffs[path] = { modified: { type: :modified, **diff } } + when 'C' + diffs[path] = { copied: { type: :copied, **diff } } + when 'D' + diffs[path] = { deleted: { type: :deleted, **diff } } + when /\AR/ # R100 (which does not exist in git.ruby-lang.org's git 2.1.4) + # TODO: implement something + else + $stderr.puts "unexpected git diff status: #{status}" + end + end + + diffs + end + + def build_diff(path, numstats) + diff = { added: 0, deleted: 0 } # :body not implemented because not used + line = numstats.find { |(_added, _deleted, file, *)| file == path } + return diff if line.nil? + + added, deleted, _ = line + if added + diff[:added] = Integer(added) + end + if deleted + diff[:deleted] = Integer(deleted) + end + diff + end + + def git_show(revision, format:) + git('show', '--no-show-signature', "--pretty=#{format}", '--no-patch', revision).strip + end + + def git(*args) + command = ['git', '-C', @repo_path, *args] + output = with_gitenv { IO.popen(command, external_encoding: 'UTF-8', &:read) } + unless $?.success? + raise GitCommandFailure, "failed to execute '#{command.join(' ')}':\n#{output}" + end + output + end + + def with_gitenv + orig = ENV.to_h.dup + begin + ENV.delete('GIT_DIR') + yield + ensure + ENV.replace(orig) + end + end +end + +CommitEmailOptions = Struct.new(:error_to, :viewer_uri) + +CommitEmail = Module.new +class << CommitEmail + SENDMAIL = ENV.fetch('SENDMAIL', '/usr/sbin/sendmail') + private_constant :SENDMAIL + + def parse(args) + options = CommitEmailOptions.new + + opts = OptionParser.new do |opts| + opts.separator('') + + opts.on('-e', '--error-to [TO]', + 'Add [TO] to to address when error is occurred') do |to| + options.error_to = to + end + + opts.on('--viewer-uri [URI]', + 'Use [URI] as URI of revision viewer') do |uri| + options.viewer_uri = uri + end + + opts.on_tail('--help', 'Show this message') do + puts opts + exit + end + end + + return opts.parse(args), options + end + + def main(repo_path, to, rest) + args, options = parse(rest) + + infos = args.each_slice(3).flat_map do |oldrev, newrev, refname| + revisions = IO.popen(['git', 'log', '--no-show-signature', '--reverse', '--pretty=%H', "#{oldrev}^..#{newrev}"], &:read).lines.map(&:strip) + revisions[0..-2].zip(revisions[1..-1]).map do |old, new| + GitInfoBuilder.new(repo_path).build(old, new, refname) + end + end + + infos.each do |info| + next if info.branch.start_with?('notes/') + puts "#{info.branch}: #{info.revision} (#{info.author})" + + from = make_from(name: info.author, email: "noreply@ruby-lang.org") + sendmail(to, from, make_mail(to, from, info, viewer_uri: options.viewer_uri)) + end + end + + def sendmail(to, from, mail) + IO.popen([*SENDMAIL.shellsplit, to], 'w') do |f| + f.print(mail) + end + unless $?.success? + raise "Failed to run `#{SENDMAIL} #{to}` with: '#{mail}'" + end + end + + private + + def b_encode(str) + NKF.nkf('-WwM', str) + end + + def make_body(info, viewer_uri:) + body = +'' + body << "#{info.author}\t#{format_time(info.date)}\n" + body << "\n" + body << " New Revision: #{info.revision}\n" + body << "\n" + body << " #{viewer_uri}#{info.revision}\n" + body << "\n" + body << " Log:\n" + body << info.log.lstrip.gsub(/^\t*/, ' ').rstrip + body << "\n\n" + body << added_dirs(info) + body << added_files(info) + body << deleted_dirs(info) + body << deleted_files(info) + body << modified_dirs(info) + body << modified_files(info) + [body.rstrip].pack('M') + end + + def format_time(time) + time.strftime('%Y-%m-%d %X %z (%a, %d %b %Y)') + end + + def changed_items(title, type, items) + rv = '' + unless items.empty? + rv << " #{title} #{type}:\n" + rv << items.collect {|item| " #{item}\n"}.join('') + end + rv + end + + def changed_files(title, files) + changed_items(title, 'files', files) + end + + def added_files(info) + changed_files('Added', info.added_files) + end + + def deleted_files(info) + changed_files('Removed', info.deleted_files) + end + + def modified_files(info) + changed_files('Modified', info.updated_files) + end + + def changed_dirs(title, files) + changed_items(title, 'directories', files) + end + + def added_dirs(info) + changed_dirs('Added', info.added_dirs) + end + + def deleted_dirs(info) + changed_dirs('Removed', info.deleted_dirs) + end + + def modified_dirs(info) + changed_dirs('Modified', info.updated_dirs) + end + + def changed_dirs_info(info, uri) + (info.added_dirs.collect do |dir| + " Added: #{dir}\n" + end + info.deleted_dirs.collect do |dir| + " Deleted: #{dir}\n" + end + info.updated_dirs.collect do |dir| + " Modified: #{dir}\n" + end).join("\n") + end + + def diff_info(info, uri) + info.diffs.collect do |key, values| + [ + key, + values.collect do |type, value| + case type + when :added + rev = "?revision=#{info.revision}&view=markup" + when :modified, :property_changed + prev_revision = (info.revision.is_a?(Integer) ? info.revision - 1 : "#{info.revision}^") + rev = "?r1=#{info.revision}&r2=#{prev_revision}&diff_format=u" + when :deleted, :copied + rev = '' + else + raise "unknown diff type: #{value[:type]}" + end + + link = [uri, key.sub(/ .+/, '') || ''].join('/') + rev + + desc = '' + + [desc, link] + end + ] + end + end + + def make_header(to, from, info) + <<~EOS + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: #{from} + To: #{to} + Subject: #{make_subject(info)} + EOS + end + + def make_subject(info) + subject = +'' + subject << "#{info.revision}" + subject << " (#{info.branch})" + subject << ': ' + subject << info.log.lstrip.lines.first.to_s.strip + b_encode(subject) + end + + # https://tools.ietf.org/html/rfc822#section-4.1 + # https://tools.ietf.org/html/rfc822#section-6.1 + # https://tools.ietf.org/html/rfc822#appendix-D + # https://tools.ietf.org/html/rfc2047 + def make_from(name:, email:) + if name.ascii_only? + escaped_name = name.gsub(/["\\\n]/) { |c| "\\#{c}" } + %Q["#{escaped_name}" <#{email}>] + else + escaped_name = "=?UTF-8?B?#{NKF.nkf('-WwMB', name)}?=" + %Q[#{escaped_name} <#{email}>] + end + end + + def make_mail(to, from, info, viewer_uri:) + make_header(to, from, info) + make_body(info, viewer_uri: viewer_uri) + end +end + +repo_path, to, *rest = ARGV +begin + CommitEmail.main(repo_path, to, rest) +rescue StandardError => e + $stderr.puts "#{e.class}: #{e.message}" + $stderr.puts e.backtrace + + _, options = CommitEmail.parse(rest) + to = options.error_to + CommitEmail.sendmail(to, to, <<-MAIL) +From: #{to} +To: #{to} +Subject: Error + +#{$!.class}: #{$!.message} +#{$@.join("\n")} +MAIL + exit 1 +end diff --git a/tool/downloader.rb b/tool/downloader.rb index 2398fd7b04..39ebf44a83 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -1,39 +1,14 @@ # Used by configure and make to download or update mirrored Ruby and GCC -# files. This will use HTTPS if possible, falling back to HTTP. +# files. + +# -*- frozen-string-literal: true -*- require 'fileutils' require 'open-uri' require 'pathname' -begin - require 'net/https' -rescue LoadError - https = 'http' -else - https = 'https' - - # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old - # versions do not. so, patching OpenSSL::X509::Store#add_file instead. - class OpenSSL::X509::Store - alias orig_add_file add_file - def add_file(pems) - Array(pems).each do |pem| - if File.directory?(pem) - add_path pem - else - orig_add_file pem - end - end - end - end - # since open-uri internally checks ssl_ca_cert using File.directory?, - # allow to accept an array. - class <<File - alias orig_directory? directory? - def File.directory? files - files.is_a?(Array) ? false : orig_directory?(files) - end - end -end +verbose, $VERBOSE = $VERBOSE, nil +require 'net/https' +$VERBOSE = verbose class Downloader def self.find(dlname) @@ -42,44 +17,37 @@ class Downloader end end - def self.https=(https) - @@https = https - end - - def self.https? - @@https == 'https' - end - - def self.https - @@https - end - def self.get_option(argv, options) false end class GNU < self - def self.download(name, *rest) - if https? - begin - super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest) - rescue => e - m1, m2 = e.message.split("\n", 2) - STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" - super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) - end + Mirrors = %w[ + https://raw.githubusercontent.com/autotools-mirror/autoconf/refs/heads/master/build-aux/ + https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master + ] + + def self.download(name, *rest, **options) + Mirrors.each_with_index do |url, i| + super("#{url}/#{name}", name, *rest, **options) + rescue => e + raise if i + 1 == Mirrors.size # no more URLs + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" else - super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) + return end end end class RubyGems < self - def self.download(name, dir = nil, since = true, options = {}) + def self.download(name, dir = nil, since = true, **options) require 'rubygems' - options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) - super("https://rubygems.org/downloads/#{name}", name, dir, since, options) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true + end + super("https://rubygems.org/downloads/#{name}", name, dir, since, **options) end end @@ -104,16 +72,13 @@ class Downloader end end - def self.download(name, dir = nil, since = true, options = {}) - options = options.dup - unicode_beta = options.delete(:unicode_beta) + def self.download(name, dir = nil, since = true, unicode_beta: nil, **options) name_dir_part = name.sub(/[^\/]+$/, '') if unicode_beta == 'YES' if INDEX.size == 0 - index_options = options.dup - index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file + cache_save = false # TODO: make sure caching really doesn't work for index file index_data = File.read(under(dir, "index.html")) rescue nil - index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options) + index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, cache_save: cache_save, **options) INDEX[:index] = File.read(index_file) since = true unless INDEX[:index] == index_data end @@ -122,7 +87,7 @@ class Downloader beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] # make sure we always check for new versions of files, # because they can easily change in the beta period - super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options) + super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, **options) else index_file = Pathname.new(under(dir, name_dir_part+'index.html')) if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ @@ -130,7 +95,7 @@ class Downloader "Remove all files in this directory and in .downloaded-cache/ " + "because they may be leftovers from the beta period." end - super(UNICODE_PUBLIC+name, name, dir, since, options) + super(UNICODE_PUBLIC+name, name, dir, since, **options) end end end @@ -195,64 +160,53 @@ class Downloader # Example usage: # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', # 'UnicodeData.txt', 'enc/unicode/data' - def self.download(url, name, dir = nil, since = true, options = {}) - options = options.dup + def self.download(url, name, dir = nil, since = true, + cache_save: ENV["CACHE_SAVE"] != "no", cache_dir: nil, + ignore_http_client_errors: nil, + dryrun: nil, verbose: false, **options) url = URI(url) - dryrun = options.delete(:dryrun) - if name file = Pathname.new(under(dir, name)) else name = File.basename(url.path) end - cache_save = options.delete(:cache_save) { - ENV["CACHE_SAVE"] != "no" - } - cache = cache_file(url, name, options.delete(:cache_dir)) + cache = cache_file(url, name, cache_dir) file ||= cache if since.nil? and file.exist? - if $VERBOSE + if verbose $stdout.puts "#{file} already exists" $stdout.flush end - if cache_save - save_cache(cache, file, name) - end return file.to_path end if dryrun puts "Download #{url} into #{file}" return end - if link_cache(cache, file, name, $VERBOSE) + if link_cache(cache, file, name, verbose: verbose) return file.to_path end - if !https? and URI::HTTPS === url - warn "*** using http instead of https ***" - url.scheme = 'http' - url = URI(url.to_s) - end - if $VERBOSE + if verbose $stdout.print "downloading #{name} ... " $stdout.flush end mtime = nil options = options.merge(http_options(file, since.nil? ? true : since)) begin - data = with_retry(10) do - data = url.read(options) - if mtime = data.meta["last-modified"] - mtime = Time.httpdate(mtime) - end - data - end + data = with_retry(10) {url.read(options)} rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified - if $VERBOSE + case http_error.message + when /^304 / # 304 Not Modified + if verbose $stdout.puts "#{name} not modified" $stdout.flush end return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error @@ -267,6 +221,10 @@ class Downloader return file.to_path end raise + else + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end end dest = (cache_save && cache && !cache.exist? ? cache : file) dest.parent.mkpath @@ -278,7 +236,7 @@ class Downloader if mtime dest.utime(mtime, mtime) end - if $VERBOSE + if verbose $stdout.puts "done" $stdout.flush end @@ -296,20 +254,24 @@ class Downloader dir ? File.join(dir, File.basename(name)) : name end + def self.default_cache_dir + if cache_dir = ENV['CACHE_DIR'] + return cache_dir unless cache_dir.empty? + end + ".downloaded-cache" + end + def self.cache_file(url, name, cache_dir = nil) case cache_dir when false return nil when nil - cache_dir = ENV['CACHE_DIR'] - if !cache_dir or cache_dir.empty? - cache_dir = ".downloaded-cache" - end + cache_dir = default_cache_dir end Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) end - def self.link_cache(cache, file, name, verbose = false) + def self.link_cache(cache, file, name, verbose: false) return false unless cache and cache.exist? return true if cache.eql?(file) if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] @@ -381,8 +343,6 @@ class Downloader private_class_method :with_retry end -Downloader.https = https.freeze - if $0 == __FILE__ since = true options = {} @@ -407,26 +367,42 @@ if $0 == __FILE__ case ARGV[0] when '-d', '--destdir' + ## -d, --destdir DIRECTORY Download into the directory destdir = ARGV[1] ARGV.shift when '-p', '--prefix' - # strip directory names from the name to download, and add the - # prefix instead. + ## -p, --prefix Strip directory names from the name to download, + ## and add the prefix instead. prefix = ARGV[1] ARGV.shift when '-e', '--exist', '--non-existent-only' + ## -e, --exist, --non-existent-only Skip already existent files. since = nil when '-a', '--always' + ## -a, --always Download all files. since = false when '-u', '--update', '--if-modified' + ## -u, --update, --if-modified Download newer files only. since = true - when '-n', '--dryrun' + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Do not download actually. options[:dryrun] = true when '--cache-dir' + ## --cache-dir DIRECTORY Cache downloaded files in the directory. options[:cache_dir] = ARGV[1] ARGV.shift when /\A--cache-dir=(.*)/m options[:cache_dir] = $1 + when /\A--help\z/ + ## --help Print this message + puts "Usage: #$0 [options] relative-url..." + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ abort "#{$0}: unknown option #{ARGV[0]}" else @@ -434,7 +410,7 @@ if $0 == __FILE__ end ARGV.shift end - $VERBOSE = true + options[:verbose] = true if dl args.each do |name| dir = destdir @@ -453,10 +429,10 @@ if $0 == __FILE__ end name = "#{prefix}/#{name}" end - dl.download(name, dir, since, options) + dl.download(name, dir, since, **options) end else abort "usage: #{$0} url name" unless args.size == 2 - Downloader.download(args[0], args[1], destdir, since, options) + Downloader.download(args[0], args[1], destdir, since, **options) end end diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index 9d49f427bb..a89390ad8f 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -12,6 +12,9 @@ # You can get source file for gperf. After this, simply make ruby. # Or directly run: # tool/enc-unicode.rb --header data_dir emoji_data_dir > enc/unicode/<VERSION>/name2ctype.h +# +# There are Makefile rules that automate steps above: `make update-unicode` and +# `make enc/unicode/<VERSION>/name2ctype.h`. while arg = ARGV.shift case arg @@ -143,7 +146,8 @@ def define_posix_props(data) data['Space'] = data['White_Space'] data['Blank'] = data['Space_Separator'] + [0x0009] data['Cntrl'] = data['Cc'] - data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + + data['Connector_Punctuation'] + data['Join_Control'] data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - data['Surrogate'] - data['Unassigned'] data['Print'] = data['Graph'] + data['Space_Separator'] @@ -161,14 +165,24 @@ def parse_scripts(data, categories) names = {} files.each do |file| data_foreach(file[:fn]) do |line| + # Parse Unicode data files and store code points and properties. if /^# Total (?:code points|elements): / =~ line data[current] = cps categories[current] = file[:title] (names[file[:title]] ||= []) << current cps = [] - elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line - current = $3 + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w(?:[\w\s;]*\w)?)/ =~ line + # $1: The first hexadecimal code point or the start of a range. + # $2: The end code point of the range, if present. + # If there's no range (just a single code point), $2 is nil. + # $3: The property or other info. + # Example: + # line = "0915..0939 ; InCB; Consonant # Lo [37] DEVANAGARI LETTER KA..DEVANAGARI LETTER HA" + # $1 = "0915" + # $2 = "0939" + # $3 = "InCB; Consonant" $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + current = $3.gsub(/\W+/, '_') end end end @@ -486,7 +500,11 @@ end output.ifdef :USE_UNICODE_PROPERTIES do props.each do |name| i += 1 - name = normalize_propname(name) + name = if name.start_with?('InCB') + name.downcase.gsub(/_/, '=') + else + normalize_propname(name) + end name_to_index[name] = i puts "%-40s %3d" % [name + ',', i] end diff --git a/tool/extlibs.rb b/tool/extlibs.rb index 887cac61eb..cef6712833 100755 --- a/tool/extlibs.rb +++ b/tool/extlibs.rb @@ -43,7 +43,7 @@ class ExtLibs end def do_download(url, cache_dir) - Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) + Downloader.download(url, nil, nil, nil, cache_dir: cache_dir) end def do_checksum(cache, chksums) diff --git a/tool/fake.rb b/tool/fake.rb index 0366144531..2c458985d8 100644 --- a/tool/fake.rb +++ b/tool/fake.rb @@ -63,6 +63,8 @@ prehook = proc do |extmk| RbConfig.fire_update!("extout", $extout) RbConfig.fire_update!("rubyhdrdir", "$(top_srcdir)/include") RbConfig.fire_update!("rubyarchhdrdir", "$(extout)/include/$(arch)") + RbConfig.fire_update!("rubyarchdir", "$(extout)/$(arch)") + RbConfig.fire_update!("rubylibdir", "$(extout)/common") RbConfig.fire_update!("libdirname", "buildlibdir") trace_var(:$ruby, posthook) untrace_var(:$extmk, prehook) diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb index 595506a711..127ea236f3 100755 --- a/tool/fetch-bundled_gems.rb +++ b/tool/fetch-bundled_gems.rb @@ -1,6 +1,16 @@ -#!ruby -an +#!ruby -alnF\s+|#.* BEGIN { require 'fileutils' + require_relative 'lib/colorize' + + color = Colorize.new + + if ARGV.first.start_with?("BUNDLED_GEMS=") + bundled_gems = ARGV.shift[13..-1] + sep = bundled_gems.include?(",") ? "," : " " + bundled_gems = bundled_gems.split(sep) + bundled_gems = nil if bundled_gems.empty? + end dir = ARGV.shift ARGF.eof? @@ -11,24 +21,27 @@ BEGIN { n, v, u, r = $F next unless n -next if n =~ /^#/ +next if bundled_gems&.all? {|pat| !File.fnmatch?(pat, n)} -if File.directory?(n) - puts "updating #{n} ..." - system("git", "fetch", "--all", chdir: n) or abort -else - puts "retrieving #{n} ..." - system(*%W"git clone #{u} #{n}") or abort +unless File.exist?("#{n}/.git") + puts "retrieving #{color.notice(n)} ..." + system(*%W"git clone --depth=1 --no-tags #{u} #{n}") or abort end if r - puts "fetching #{r} ..." + puts "fetching #{color.notice(r)} ..." system("git", "fetch", "origin", r, chdir: n) or abort + c = r +else + c = ["v#{v}", v].find do |c| + puts "fetching #{color.notice(c)} ..." + system("git", "fetch", "origin", "refs/tags/#{c}:refs/tags/#{c}", chdir: n) + end or abort end -c = r || "v#{v}" checkout = %w"git -c advice.detachedHead=false checkout" -puts "checking out #{c} (v=#{v}, r=#{r}) ..." +info = %[, r=#{color.info(r)}] if r +puts "checking out #{color.notice(c)} (v=#{color.info(v)}#{info}) ..." unless system(*checkout, c, "--", chdir: n) abort if r or !system(*checkout, v, "--", chdir: n) end diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb index 6200e78a56..2de8379606 100755 --- a/tool/file2lastrev.rb +++ b/tool/file2lastrev.rb @@ -98,6 +98,7 @@ ok = true data.sub!(/(?<!\A|\n)\z/, "\n") @output.write(data, overwrite: true, create_only: create_only) rescue => e + next if @suppress_not_found and VCS::NotFoundError === e warn "#{File.basename(Program)}: #{e.message}" ok = false end diff --git a/tool/format-release b/tool/format-release index 737148e0ce..8bb6154243 100755 --- a/tool/format-release +++ b/tool/format-release @@ -1,8 +1,15 @@ #!/usr/bin/env ruby -# https://rubygems.org/gems/diffy -require "diffy" + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "diffy" +end + require "open-uri" require "yaml" +require_relative "./ruby-version" Diffy::Diff.default_options.merge!( include_diff_info: true, @@ -24,10 +31,9 @@ class Tarball def gz?; @url.end_with?('.gz'); end def zip?; @url.end_with?('.zip'); end - def bz2?; @url.end_with?('.bz2'); end def xz?; @url.end_with?('.xz'); end - def ext; @url[/(?:zip|tar\.(?:gz|bz2|xz))\z/]; end + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end def to_md <<eom @@ -49,25 +55,10 @@ eom unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" - info = YAML.load(URI(uri).read) + info = YAML.unsafe_load(URI(uri).read) if info.size != 1 raise "unexpected info.yml '#{uri}'" end @@ -82,10 +73,11 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" - stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) + stat = `git -C #{rubydir} diff -l0 --shortstat #{prev_tag}..#{tag}` files_changed, insertions, deletions = stat.scan(/\d+/) end @@ -178,7 +170,7 @@ eom if /\.0(?:-\w+)?\z/ =~ ver # preview, rc, or first release entry <<= <<eom - tag: v#{ver.tr('.-', '_')} + tag: #{RubyVersion.tag(ver)} stats: files_changed: #{files_changed} insertions: #{insertions} @@ -190,34 +182,25 @@ eom url: gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip - bz2: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.bz2 xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz size: gz: #{ary.find{|x|x.gz? }.size} zip: #{ary.find{|x|x.zip?}.size} - bz2: #{ary.find{|x|x.bz2?}&.size} xz: #{ary.find{|x|x.xz? }.size} sha1: gz: #{ary.find{|x|x.gz? }.sha1} zip: #{ary.find{|x|x.zip?}.sha1} - bz2: #{ary.find{|x|x.bz2?}&.sha1} xz: #{ary.find{|x|x.xz? }.sha1} sha256: gz: #{ary.find{|x|x.gz? }.sha256} zip: #{ary.find{|x|x.zip?}.sha256} - bz2: #{ary.find{|x|x.bz2?}&.sha256} xz: #{ary.find{|x|x.xz? }.sha256} sha512: gz: #{ary.find{|x|x.gz? }.sha512} zip: #{ary.find{|x|x.zip?}.sha512} - bz2: #{ary.find{|x|x.bz2?}&.sha512} xz: #{ary.find{|x|x.xz? }.sha512} eom - if ver.start_with?("3.") - entry = entry.gsub(/ bz2: .*\n/, "") - end - if data.include?("\n- version: #{ver}\n") # update existing entry data.sub!(/\n- version: #{ver}\n(^ .*\n)*\n/, "\n#{entry}\n") diff --git a/tool/generic_erb.rb b/tool/generic_erb.rb index 6607d5c256..b6cb51474e 100644 --- a/tool/generic_erb.rb +++ b/tool/generic_erb.rb @@ -12,7 +12,7 @@ source = false templates = [] ARGV.options do |o| - o.on('-i', '--input=PATH') {|v| template << v} + o.on('-i', '--input=PATH') {|v| templates << v} o.on('-x', '--source') {source = true} out.def_options(o) o.order!(ARGV) @@ -27,11 +27,7 @@ vpath = out.vpath output, vpath = output, vpath result = templates.map do |template| - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(File.read(template), trim_mode: '%-') - else - erb = ERB.new(File.read(template), nil, '%-') - end + erb = ERB.new(File.read(template), trim_mode: '%-') erb.filename = template source ? erb.src : proc{erb.result(binding)}.call end diff --git a/tool/gperf.sed b/tool/gperf.sed index 52915f80d5..9550a522b9 100644 --- a/tool/gperf.sed +++ b/tool/gperf.sed @@ -1,23 +1,4 @@ -/ANSI-C code/{ - h - s/.*/ANSI:offset:/ - x -} -/\/\*!ANSI{\*\//{ - G - s/\/\*!ANSI{\*\/\(.*\)\/\*}!ANSI\*\/\(.*\)\nANSI:.*/\/\*\1\*\/\2/ -} -s/(int)([a-z_]*)&((struct \([a-zA-Z_0-9][a-zA-Z_0-9]*\)_t *\*)0)->\1_str\([1-9][0-9]*\),/gperf_offsetof(\1, \2),/g -/^#line/{ - G - x - s/:offset:/:/ - x - s/\(.*\)\(\n\).*:offset:.*/#define gperf_offsetof(s, n) (short)offsetof(struct s##_t, s##_str##n)\2\1/ - s/\n[^#].*// -} /^[a-zA-Z_0-9]*hash/,/^}/{ s/ hval = / hval = (unsigned int)/ s/ return / return (unsigned int)/ } -s/^\(static\) \(unsigned char gperf_downcase\)/\1 const \2/ diff --git a/tool/ifchange b/tool/ifchange index 5af41e0156..9e4a89533c 100755 --- a/tool/ifchange +++ b/tool/ifchange @@ -18,7 +18,7 @@ HELP set -e timestamp= keepsuffix= -empty= +srcavail=f color=auto until [ $# -eq 0 ]; do case "$1" in @@ -39,7 +39,7 @@ until [ $# -eq 0 ]; do keepsuffix=`expr \( "$1" : '[^=]*=\(.*\)' \)` ;; --empty) - empty=yes + srcavail=s ;; --color) color=always @@ -97,7 +97,7 @@ fi targetdir= case "$target" in */*) targetdir=`dirname "$target"`;; esac -if [ -f "$target" -a ! -${empty:+f}${empty:-s} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then +if [ -f "$target" -a ! -${srcavail} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then echo "$target ${msg_unchanged}unchanged${msg_reset}" rm -f "$temp" else diff --git a/tool/leaked-globals b/tool/leaked-globals index 87089ebd81..6118cd56e8 100755 --- a/tool/leaked-globals +++ b/tool/leaked-globals @@ -1,24 +1,28 @@ #!/usr/bin/ruby require_relative 'lib/colorize' +require 'shellwords' until ARGV.empty? case ARGV[0] when /\A SYMBOL_PREFIX=(.*)/x SYMBOL_PREFIX = $1 when /\A NM=(.*)/x # may be multiple words - NM = $1 + NM = $1.shellsplit when /\A PLATFORM=(.+)?/x platform = $1 when /\A SOEXT=(.+)?/x soext = $1 when /\A SYMBOLS_IN_EMPTYLIB=(.*)/x SYMBOLS_IN_EMPTYLIB = $1.split(" ") + when /\A EXTSTATIC=(.+)?/x + EXTSTATIC = true else break end ARGV.shift end SYMBOLS_IN_EMPTYLIB ||= nil +EXTSTATIC ||= false config = ARGV.shift count = 0 @@ -44,7 +48,9 @@ if platform and !platform.empty? end missing = File.dirname(config) + "/missing/" ARGV.reject! do |n| - unless (src = Dir.glob(missing + File.basename(n, ".*") + ".[cS]")).empty? + base = File.basename(n, ".*") + next true if REPLACE.include?(base) + unless (src = Dir.glob(missing + base + ".[cS]")).empty? puts "Ignore #{col.skip(n)} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" true end @@ -56,9 +62,21 @@ REPLACE.push("rust_eh_personality") if RUBY_PLATFORM.include?("darwin") print "Checking leaked global symbols..." STDOUT.flush -soext = /\.#{soext}(?:$|\.)/ if soext -so = soext =~ ARGV.first if ARGV.size == 1 -IO.foreach("|#{NM} #{ARGV.join(' ')}") do |line| +if soext + soext = /\.#{soext}(?:$|\.)/ + if EXTSTATIC + ARGV.delete_if {|n| soext =~ n} + elsif ARGV.size == 1 + so = soext =~ ARGV.first + end +end + +Pipe = Struct.new(:command) do + def open(&block) IO.popen(command, &block) end + def each(&block) open {|f| f.each(&block)} end +end + +Pipe.new(NM + ARGV).each do |line| line.chomp! next so = nil if line.empty? if so.nil? and line.chomp!(":") @@ -70,6 +88,9 @@ IO.foreach("|#{NM} #{ARGV.join(' ')}") do |line| next unless n.sub!(/^#{SYMBOL_PREFIX}/o, "") next if n.include?(".") next if !so and n.start_with?("___asan_") + next if !so and n.start_with?("__odr_asan_") + next if !so and n.start_with?("__retguard_") + next if !so and n.start_with?("__dtrace") case n when /\A(?:Init_|InitVM_|pm_|[Oo]nig|dln_|coroutine_)/ next diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb new file mode 100644 index 0000000000..daa1a1f235 --- /dev/null +++ b/tool/lib/_tmpdir.rb @@ -0,0 +1,100 @@ +template = "rubytest." + +# This path is only for tests. +# Assume the directory by these environment variables are safe. +base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| + next unless tmp and tmp.size <= 50 and File.directory?(tmp) + # On macOS, the default TMPDIR is very long, in spite of UNIX socket + # path length being limited. + # + # Also Rubygems creates its own temporary directory per tests, and + # some tests copy the full path of gemhome there. In that case, the + # path contains both temporary names twice, and can exceed path name + # limit very easily. + tmp +end +begin + tmpdir = File.join(base, template + Random.new_seed.to_s(36)[-6..-1]) + Dir.mkdir(tmpdir, 0o700) +rescue Errno::EEXIST + retry +end +# warn "tmpdir(#{tmpdir.size}) = #{tmpdir}" + +pid = $$ +END { + if pid == $$ + begin + Dir.rmdir(tmpdir) + rescue Errno::ENOENT + rescue Errno::ENOTEMPTY + require_relative "colorize" + colorize = Colorize.new + ls = Struct.new(:colorize) do + def mode_inspect(m, s) + [ + (m & 0o4 == 0 ? ?- : ?r), + (m & 0o2 == 0 ? ?- : ?w), + (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)), + ] + end + def decorate_path(path, st) + case + when st.directory? + color = "bold;blue" + type = "/" + when st.symlink? + color = "bold;cyan" + # type = "@" + when st.executable? + color = "bold;green" + type = "*" + when path.end_with?(".gem") + color = "green" + end + colorize.decorate(path, color) + (type || "") + end + def list_tree(parent, indent = "", &block) + children = Dir.children(parent).map do |child| + [child, path = File.join(parent, child), File.lstat(path)] + end + nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size + size_width = children.map {|child, path, st| st.size}.max.to_s.size + + children.each do |child, path, st| + m = st.mode + m = [ + (st.file? ? ?- : st.ftype[0]), + mode_inspect(m >> 6, (?s unless m & 04000 == 0)), + mode_inspect(m >> 3, (?s unless m & 02000 == 0)), + mode_inspect(m, (?t unless m & 01000 == 0)), + ].join("") + warn sprintf("%s* %s %*d %*d %s % s%s", + indent, m, nlink_width, st.nlink, size_width, st.size, + st.mtime.to_s, decorate_path(child, st), + (" -> " + decorate_path(File.readlink(path), File.stat(path)) if + st.symlink?)) + if st.directory? + list_tree(File.join(parent, child), indent + " ", &block) + end + yield path, st if block + end + end + end.new(colorize) + warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":" + Dir.chdir(tmpdir) do + ls.list_tree(".") do |path, st| + if st.directory? + Dir.rmdir(path) + else + File.unlink(path) + end + end + end + require "fileutils" + FileUtils.rm_rf(tmpdir) + end + end +} + +ENV["TMPDIR"] = ENV["SPEC_TEMP_DIR"] = ENV["GEM_TEST_TMPDIR"] = tmpdir diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb new file mode 100644 index 0000000000..9ad5ea220b --- /dev/null +++ b/tool/lib/bundle_env.rb @@ -0,0 +1,4 @@ +ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_PATH__SYSTEM"] = "true" +ENV["BUNDLE_WITHOUT"] = "lint doc" diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index bfc0be1c81..d2ed61a508 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -6,11 +6,24 @@ require 'rubygems/package' # unpack bundled gem files. module BundledGem + DEFAULT_GEMS_DEPENDENCIES = [ + "net-protocol", # net-ftp + "time", # net-ftp + "singleton", # prime + "ipaddr", # rinda + "forwardable", # prime, rinda + "strscan", # rexml + "psych" # rdoc + ] + module_function def unpack(file, *rest) pkg = Gem::Package.new(file) - prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)} + prepare_test(pkg.spec, *rest) do |dir| + pkg.extract_files(dir) + FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)}) + end puts "Unpacked #{file}" rescue Gem::Package::FormatError, Errno::ENOENT puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." @@ -55,6 +68,9 @@ module BundledGem gem_dir = File.join(dir, "gems", target) yield gem_dir spec_dir = spec.extensions.empty? ? "specifications" : File.join("gems", target) + if spec.extensions.empty? + spec.dependencies.reject! {|dep| DEFAULT_GEMS_DEPENDENCIES.include?(dep.name)} + end File.binwrite(File.join(dir, spec_dir, "#{target}.gemspec"), spec.to_ruby) unless spec.extensions.empty? spec.dependencies.clear @@ -79,7 +95,10 @@ module BundledGem Dir.chdir(gemdir) do spec = Gem::Specification.new do |s| s.name = gemfile.chomp(".gemspec") - s.version = File.read("lib/#{s.name}.rb")[/VERSION = "(.+?)"/, 1] + s.version = + File.read("lib/#{s.name}.rb")[/VERSION = "(.+?)"/, 1] || + begin File.read("lib/#{s.name}/version.rb")[/VERSION = "(.+?)"/, 1]; rescue; nil; end || + raise("cannot find the version of #{ s.name } gem") s.authors = ["DUMMY"] s.email = ["dummy@ruby-lang.org"] s.files = Dir.glob("{lib,ext}/**/*").select {|f| File.file?(f)} diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb index 1131221586..0904312119 100644 --- a/tool/lib/colorize.rb +++ b/tool/lib/colorize.rb @@ -35,7 +35,8 @@ class Colorize "bright_blue"=>"94", "bright_magenta"=>"95", "bright_cyan"=>"96", "bright_white"=>"97", # abstract decorations - "pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold", "note"=>"bright_yellow", + "pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold", + "note"=>"bright_yellow", "notice"=>"bright_yellow", "info"=>"bright_magenta", } def coloring? diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 358e7d9551..e29a0e3c25 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -74,6 +74,20 @@ module Test module CoreAssertions require_relative 'envutil' require 'pp' + begin + require '-test-/sanitizers' + rescue LoadError + # in test-unit-ruby-core gem + def sanitizers + nil + end + else + def sanitizers + Test::Sanitizers + end + end + module_function :sanitizers + nil.pretty_inspect def mu_pp(obj) #:nodoc: @@ -92,11 +106,14 @@ module Test end def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, - success: nil, **opt) + success: nil, failed: nil, gems: false, **opt) args = Array(args).dup - args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') + unless gems.nil? + args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems") + end stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) - desc = FailDesc[status, message, stderr] + desc = failed[status, message, stderr] if failed + desc ||= FailDesc[status, message, stderr] if block_given? raise "test_stdout ignored, use block only or without block" if test_stdout != [] raise "test_stderr ignored, use block only or without block" if test_stderr != [] @@ -152,6 +169,9 @@ module Test pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? # For previous versions which implemented MJIT pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + # ASAN has the same problem - its shadow memory greatly increases memory usage + # (plus asan has better ways to detect memory leaks than this assertion) + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled? require_relative 'memory_status' raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) @@ -283,9 +303,34 @@ module Test def separated_runner(token, out = nil) include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + out = out ? IO.new(out, 'w') : STDOUT + + # avoid method redefinitions + out_write = out.method(:write) + integer_to_s = Integer.instance_method(:to_s) + array_pack = Array.instance_method(:pack) + marshal_dump = Marshal.method(:dump) + assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set) + assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get) + Test::Unit::Assertions.module_eval do + @_assertions = 0 + + undef _assertions= + define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)}) + + undef _assertions + define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)}) + end + # assume Method#call and UnboundMethod#bind_call need to work as the original + at_exit { - out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" + assertions = assertions_ivar_get.call(:@_assertions) + out_write.call <<~OUT + <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}> + #{array_pack.bind_call([marshal_dump.call($!)], 'm0')} + </error id="#{token}"> + OUT } if defined?(Test::Unit::Runner) Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) @@ -319,7 +364,16 @@ eom args = args.dup args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + # power_assert 3 requires ruby 3.1 or later + args << "-W:no-experimental" if RUBY_VERSION < "3.1." stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + + if sanitizers&.lsan_enabled? + # LSAN may output messages like the following line into stderr. We should ignore it. + # ==276855==Running thread 276851 was not suspended. False leaks are possible. + # See https://github.com/google/sanitizers/issues/1479 + stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "") + end ensure if res_c res_c.close @@ -330,15 +384,16 @@ eom end raise if $! abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + marshal_error = nil assert(!abort, FailDesc[status, nil, stderr]) - self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i - begin - res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m")) + res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do + self._assertions += $1.to_i + res = Marshal.load($2.unpack1("m")) or next rescue => marshal_error ignore_stderr = nil res = nil - end - if res and !(SystemExit === res) + else + next if SystemExit === res if bt = res.backtrace bt.each do |l| l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} @@ -350,7 +405,7 @@ eom raise res end - # really is it succeed? + # really did it succeed? unless ignore_stderr # the body of assert_separately must not output anything to detect error assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) @@ -361,9 +416,17 @@ eom # Run Ractor-related test without influencing the main test suite def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) - return unless defined?(Ractor) + omit unless defined?(Ractor) + + # https://bugs.ruby-lang.org/issues/21262 + shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value) + shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join) + + if require + require = [require] unless require.is_a?(Array) + require = require.map {|r| "require #{r.inspect}"}.join("\n") + end - require = "require #{require.inspect}" if require if require_relative dir = File.dirname(caller_locations[0,1][0].absolute_path) full_path = File.expand_path(require_relative, dir) @@ -371,6 +434,8 @@ eom end assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{shim_value} + #{shim_join} #{require} previous_verbose = $VERBOSE $VERBOSE = nil @@ -481,19 +546,15 @@ eom case expected when String assert = :assert_equal - when Regexp - assert = :assert_match else - raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" + assert_respond_to(expected, :===) + assert = :assert_match end - ex = m = nil - EnvUtil.with_default_internal(expected.encoding) do - ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do - yield - end - m = ex.message + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield end + m = ex.message msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} if assert == :assert_equal @@ -506,6 +567,43 @@ eom ex end + # :call-seq: + # assert_raise_kind_of(*args, &block) + # + #Tests if the given block raises one of the given exceptions or + #sub exceptions of the given exceptions. If the last argument + #is a String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise SystemCallErr do + # Dir.chdir(__FILE__) #Raises Errno::ENOTDIR, so assertion succeeds + # end + def assert_raise_kind_of(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + raise e unless exp.include? Test::Unit::PendedError + rescue *exp => e + pass + rescue Exception => e + flunk(message(msg) {"#{mu_pp(exp)} family exception expected, not #{mu_pp(e)}"}) + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} family expected but nothing was raised"}) + end + end + e + end + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: # :call-seq: @@ -625,7 +723,7 @@ eom def assert_warning(pat, msg = nil) result = nil - stderr = EnvUtil.with_default_internal(pat.encoding) { + stderr = EnvUtil.with_default_internal(of: pat) { EnvUtil.verbose_warning { result = yield } @@ -639,17 +737,15 @@ eom assert_warning(*args) {$VERBOSE = false; yield} end - def assert_deprecated_warning(mesg = /deprecated/) + def assert_deprecated_warning(mesg = /deprecated/, &block) assert_warning(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end - def assert_deprecated_warn(mesg = /deprecated/) + def assert_deprecated_warn(mesg = /deprecated/, &block) assert_warn(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end @@ -786,6 +882,9 @@ eom rescue # Constants may be defined but not implemented, e.g., mingw. else + unless Process.clock_getres(clk) < 1.0e-03 + next # needs msec precision + end PERFORMANCE_CLOCK = clk end end @@ -817,7 +916,9 @@ eom end times.compact! tmin, tmax = times.minmax - tbase = 10 ** Math.log10(tmax * ([(tmax / tmin), 2].max ** 2)).ceil + + # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) + tbase = 10 * tmax * [(tmax / tmin) ** 2 / 4, 1].max info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" seq.each do |i| @@ -851,6 +952,82 @@ eom token = "\e[7;1m#{$$.to_s}:#{Time.now.strftime('%s.%L')}:#{rand(0x10000).to_s(16)}:\e[m" return token.dump, Regexp.quote(token) end + + # Platform predicates + + def self.mswin? + defined?(@mswin) ? @mswin : @mswin = RUBY_PLATFORM.include?('mswin') + end + private def mswin? + CoreAssertions.mswin? + end + + def self.mingw? + defined?(@mingw) ? @mingw : @mingw = RUBY_PLATFORM.include?('mingw') + end + private def mingw? + CoreAssertions.mingw? + end + + module_function def windows? + mswin? or mingw? + end + + def self.version_compare(expected, actual) + expected.zip(actual).each {|e, a| z = (e <=> a); return z if z.nonzero?} + 0 + end + + def self.version_match?(expected, actual) + if !actual + false + elsif expected.empty? + true + elsif expected.size == 1 and Range === (range = expected.first) + b, e = range.begin, range.end + return false if b and (c = version_compare(Array(b), actual)) > 0 + return false if e and (c = version_compare(Array(e), actual)) < 0 + return false if e and range.exclude_end? and c == 0 + true + else + version_compare(expected, actual).zero? + end + end + + def self.linux?(*ver) + unless defined?(@linux) + @linux = RUBY_PLATFORM.include?('linux') && `uname -r`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @linux + end + private def linux?(*ver) + CoreAssertions.linux?(*ver) + end + + def self.glibc?(*ver) + unless defined?(@glibc) + libc = `/usr/bin/ldd /bin/sh`[/^\s*libc.*=> *\K\S*/] + if libc and /version (\d+)\.(\d+)\.$/ =~ IO.popen([libc], &:read)[] + @glibc = [$1.to_i, $2.to_i] + else + @glibc = false + end + end + version_match? ver, @glibc + end + private def glibc?(*ver) + CoreAssertions.glibc?(*ver) + end + + def self.macos?(*ver) + unless defined?(@macos) + @macos = RUBY_PLATFORM.include?('darwin') && `sw_vers -productVersion`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @macos + end + private def macos?(*ver) + CoreAssertions.macos?(*ver) + end end end end diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb new file mode 100644 index 0000000000..56b420a546 --- /dev/null +++ b/tool/lib/dump.gdb @@ -0,0 +1,17 @@ +set height 0 +set width 0 +set confirm off + +echo \n>>> Threads\n\n +info threads + +echo \n>>> Machine level backtrace\n\n +thread apply all info stack full + +echo \n>>> Dump Ruby level backtrace (if possible)\n\n +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +echo ">>> Finish\n" +detach +quit diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb new file mode 100644 index 0000000000..ed9cb89010 --- /dev/null +++ b/tool/lib/dump.lldb @@ -0,0 +1,13 @@ +script print("\n>>> Threads\n\n") +thread list + +script print("\n>>> Machine level backtrace\n\n") +thread backtrace all + +script print("\n>>> Dump Ruby level backtrace (if possible)\n\n") +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +script print(">>> Finish\n") +detach +quit diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index 309a6af40f..6089605056 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -52,7 +52,14 @@ module EnvUtil @original_internal_encoding = Encoding.default_internal @original_external_encoding = Encoding.default_external @original_verbose = $VERBOSE - @original_warning = defined?(Warning.[]) ? %i[deprecated experimental].to_h {|i| [i, Warning[i]]} : nil + @original_warning = + if defined?(Warning.categories) + Warning.categories.to_h {|i| [i, Warning[i]]} + elsif defined?(Warning.[]) # 2.7+ + %i[deprecated experimental performance].to_h do |i| + [i, begin Warning[i]; rescue ArgumentError; end] + end.compact + end end end @@ -72,6 +79,72 @@ module EnvUtil end module_function :timeout + class Debugger + @list = [] + + attr_accessor :name + + def self.register(name, &block) + @list << new(name, &block) + end + + def initialize(name, &block) + @name = name + instance_eval(&block) + end + + def usable?; false; end + + def start(pid, *args) end + + def dump(pid, timeout: 60, reprieve: timeout&.div(4)) + dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err) + rescue Errno::ENOENT + return + else + return unless dpid + [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig| + begin + return EnvUtil.timeout(t) {Process.wait(dpid)} + rescue Timeout::Error + Process.kill(sig, dpid) + end + end + true + end + + # sudo -n: --non-interactive + PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM) + + def spawn(*args, **opts) + super(*PRECOMMAND, *args, **opts) + end + + register("gdb") do + class << self + def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end + def start(pid, *args, **opts) + spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts) + end + def command_file(file) "--command=#{file}"; end + end + end + + register("lldb") do + class << self + def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end + def start(pid, *args, **opts) + spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts) + end + def command_file(file) ["--source", file]; end + end + end + + def self.search + @debugger ||= @list.find(&:usable?) + end + end + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) reprieve = apply_timeout_scale(reprieve) if reprieve @@ -87,17 +160,12 @@ module EnvUtil pgroup = pid end - lldb = true if /darwin/ =~ RUBY_PLATFORM - + dumped = false while signal = signals.shift - if lldb and [:ABRT, :KILL].include?(signal) - lldb = false - # sudo -n: --non-interactive - # lldb -p: attach - # -o: run command - system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit]) - true + if !dumped and [:ABRT, :KILL].include?(signal) + Debugger.search&.dump(pid) + dumped = true end begin @@ -158,6 +226,11 @@ module EnvUtil } args = [args] if args.kind_of?(String) + # use the same parser as current ruby + if (args.none? { |arg| arg.start_with?("--parser=") } and + /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read)) + args = ["--parser=#{current_parser}"] + args + end pid = spawn(child_env, *precommand, rubybin, *args, opt) in_c.close out_c&.close @@ -205,6 +278,12 @@ module EnvUtil end module_function :invoke_ruby + def current_parser + features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}] + features&.split&.include?("+PRISM") ? "prism" : "parse.y" + end + module_function :current_parser + def verbose_warning class << (stderr = "".dup) alias write concat @@ -221,6 +300,21 @@ module EnvUtil end module_function :verbose_warning + if defined?(Warning.[]=) + def deprecation_warning + previous_deprecated = Warning[:deprecated] + Warning[:deprecated] = true + yield + ensure + Warning[:deprecated] = previous_deprecated + end + else + def deprecation_warning + yield + end + end + module_function :deprecation_warning + def default_warning $VERBOSE = false yield @@ -246,11 +340,16 @@ module EnvUtil module_function :under_gc_stress def under_gc_compact_stress(val = :empty, &block) - auto_compact = GC.auto_compact - GC.auto_compact = val + raise "compaction doesn't work well on s390x. Omit the test in the caller." if RUBY_PLATFORM =~ /s390x/ # https://github.com/ruby/ruby/pull/5077 + + if GC.respond_to?(:auto_compact) + auto_compact = GC.auto_compact + GC.auto_compact = val + end + under_gc_stress(&block) ensure - GC.auto_compact = auto_compact + GC.auto_compact = auto_compact if GC.respond_to?(:auto_compact) end module_function :under_gc_compact_stress @@ -262,7 +361,8 @@ module EnvUtil end module_function :without_gc - def with_default_external(enc) + def with_default_external(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_external = enc } yield ensure @@ -270,7 +370,8 @@ module EnvUtil end module_function :with_default_external - def with_default_internal(enc) + def with_default_internal(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_internal = enc } yield ensure diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb new file mode 100644 index 0000000000..1893e07657 --- /dev/null +++ b/tool/lib/gem_env.rb @@ -0,0 +1 @@ +ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__) diff --git a/tool/lib/launchable.rb b/tool/lib/launchable.rb new file mode 100644 index 0000000000..38f4fe92b3 --- /dev/null +++ b/tool/lib/launchable.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +require 'json' +require 'uri' + +module Launchable + ## + # JsonStreamWriter writes a JSON file using a stream. + # By utilizing a stream, we can minimize memory usage, especially for large files. + class JsonStreamWriter + def initialize(path) + @file = File.open(path, "w") + @file.write("{") + @indent_level = 0 + @is_first_key_val = true + @is_first_obj = true + write_new_line + end + + def write_object obj + if @is_first_obj + @is_first_obj = false + else + write_comma + write_new_line + end + @indent_level += 1 + @file.write(to_json_str(obj)) + @indent_level -= 1 + @is_first_key_val = true + # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified. + # { + # "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds", + # "status": "TEST_PASSED", + # "duration": 2.7e-05, + # "createdAt": "2024-02-09 12:21:07 +0000", + # "stderr": null, + # "stdout": null + # }: null <- here + # }, + # To prevent this, IO#flush is called here. + @file.flush + end + + def write_array(key) + @indent_level += 1 + @file.write(to_json_str(key)) + write_colon + @file.write(" ", "[") + write_new_line + end + + def close + return if @file.closed? + close_array + @indent_level -= 1 + write_new_line + @file.write("}", "\n") + @file.flush + @file.close + end + + private + def to_json_str(obj) + json = JSON.pretty_generate(obj) + json.gsub(/^/, ' ' * (2 * @indent_level)) + end + + def write_indent + @file.write(" " * 2 * @indent_level) + end + + def write_new_line + @file.write("\n") + end + + def write_comma + @file.write(',') + end + + def write_colon + @file.write(":") + end + + def close_array + write_new_line + write_indent + @file.write("]") + @indent_level -= 1 + end + end +end diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb index 4cd28b9dd5..69df9a64b8 100644 --- a/tool/lib/leakchecker.rb +++ b/tool/lib/leakchecker.rb @@ -112,7 +112,7 @@ class LeakChecker } unless fd_leaked.empty? unless @@try_lsof == false - @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + @@try_lsof |= system(*%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) end end h.each {|fd, list| @@ -136,14 +136,14 @@ class LeakChecker attr_accessor :count end - def new(data) + def new(...) LeakChecker::TempfileCounter.count += 1 - super(data) + super end } LeakChecker.const_set(:TempfileCounter, m) - class << Tempfile::Remover + class << Tempfile prepend LeakChecker::TempfileCounter end end @@ -155,8 +155,8 @@ class LeakChecker if prev_count == count [prev_count, []] else - tempfiles = ObjectSpace.each_object(Tempfile).find_all {|t| - t.instance_variable_defined?(:@tmpfile) and t.path + tempfiles = ObjectSpace.each_object(Tempfile).reject {|t| + t.instance_variables.empty? || t.closed? } [count, tempfiles] end diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb index 60632523a8..429e5f6a1d 100644 --- a/tool/lib/memory_status.rb +++ b/tool/lib/memory_status.rb @@ -20,48 +20,68 @@ module Memory data.scan(pat) {|k, v| keys << k.downcase.intern} when /mswin|mingw/ =~ RUBY_PLATFORM - require 'fiddle/import' - require 'fiddle/types' - - module Win32 - extend Fiddle::Importer - dlload "kernel32.dll", "psapi.dll" - include Fiddle::Win32Types - typealias "SIZE_T", "size_t" - - PROCESS_MEMORY_COUNTERS = struct [ - "DWORD cb", - "DWORD PageFaultCount", - "SIZE_T PeakWorkingSetSize", - "SIZE_T WorkingSetSize", - "SIZE_T QuotaPeakPagedPoolUsage", - "SIZE_T QuotaPagedPoolUsage", - "SIZE_T QuotaPeakNonPagedPoolUsage", - "SIZE_T QuotaNonPagedPoolUsage", - "SIZE_T PagefileUsage", - "SIZE_T PeakPagefileUsage", - ] - - typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" - - extern "HANDLE GetCurrentProcess()", :stdcall - extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall - - module_function - def memory_info - size = PROCESS_MEMORY_COUNTERS.size - data = PROCESS_MEMORY_COUNTERS.malloc - data.cb = size - data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + keys.push(:size, :rss, :peak) + + begin + require 'fiddle/import' + require 'fiddle/types' + rescue LoadError + # Fallback to PowerShell command to get memory information for current process + def self.read_status + cmd = [ + "powershell.exe", "-NoProfile", "-Command", + "Get-Process -Id #{$$} | " \ + "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }" + ] + + IO.popen(cmd, "r", err: [:child, :out]) do |out| + if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read + yield :size, $1.to_i + yield :rss, $2.to_i + yield :peak, $3.to_i + end + end + end + else + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end end - end - keys.push(:size, :rss, :peak) - def self.read_status - if info = Win32.memory_info - yield :size, info.PagefileUsage - yield :rss, info.WorkingSetSize - yield :peak, info.PeakWorkingSetSize + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end end end when (require_relative 'find_executable' diff --git a/tool/lib/output.rb b/tool/lib/output.rb index 5c645daca6..8cb426ae4a 100644 --- a/tool/lib/output.rb +++ b/tool/lib/output.rb @@ -4,10 +4,15 @@ require_relative 'colorize' class Output attr_reader :path, :vpath - def initialize - @path = @timestamp = @ifchange = @color = nil - @overwrite = @create_only = false - @vpath = VPath.new + def initialize(path: nil, timestamp: nil, ifchange: nil, color: nil, + overwrite: false, create_only: false, vpath: VPath.new) + @path = path + @timestamp = timestamp + @ifchange = ifchange + @color = color + @overwrite = overwrite + @create_only = create_only + @vpath = vpath end COLOR_WHEN = { diff --git a/tool/lib/path.rb b/tool/lib/path.rb new file mode 100644 index 0000000000..f16a164338 --- /dev/null +++ b/tool/lib/path.rb @@ -0,0 +1,101 @@ +module Path + module_function + + def clean(path) + path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') + nil while path.sub!(/[^\/]+\/\.\.\//, '') + path + end + + def relative(path, base) + path = clean(path) + base = clean(base) + path, base = [path, base].map{|s|s.split("/")} + until path.empty? or base.empty? or path[0] != base[0] + path.shift + base.shift + end + path, base = [path, base].map{|s|s.join("/")} + if base.empty? + path + elsif base.start_with?("../") or File.absolute_path?(base) + File.expand_path(path) + else + base.gsub!(/[^\/]+/, '..') + File.join(base, path) + end + end + + def clean_link(src, dest) + begin + link = File.readlink(dest) + rescue + else + return if link == src + File.unlink(dest) + end + yield src, dest + end + + # Extensions to FileUtils + + module Mswin + def ln_safe(src, dest, real_src, *opt) + cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] + cmd[1, 0] = opt + return if system("cmd", "/c", *cmd) + # TODO: use RUNAS or something + puts cmd.join(" ") + end + + def ln_dir_safe(src, dest, real_src) + ln_safe(src, dest, "/d") + end + end + + module HardlinkExcutable + def ln_exe(relative_src, dest, src) + ln(src, dest, force: true) + end + end + + def ln_safe(src, dest, real_src) + ln_sf(src, dest) + rescue Errno::ENOENT + # Windows disallows to create broken symboic links, probably because + # it is a kind of reparse points. + raise if File.exist?(real_src) + end + + alias ln_dir_safe ln_safe + alias ln_exe ln_safe + + def ln_relative(src, dest, executable = false) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + if executable + return (ln_exe(relative(src, parent), dest, src) if File.exist?(src)) + end + clean_link(relative(src, parent), dest) {|s, d| ln_safe(s, d, src)} + end + + def ln_dir_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative(src, parent), dest) {|s, d| ln_dir_safe(s, d, src)} + end + + case (CROSS_COMPILING || RUBY_PLATFORM) + when /linux|darwin|solaris/ + prepend HardlinkExcutable + extend HardlinkExcutable + when /mingw|mswin/ + unless File.respond_to?(:symlink) + prepend Mswin + extend Mswin + end + else + end +end diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb new file mode 100644 index 0000000000..7b889163b0 --- /dev/null +++ b/tool/lib/test/jobserver.rb @@ -0,0 +1,47 @@ +module Test + module JobServer + end +end + +class << Test::JobServer + def connect(makeflags = ENV["MAKEFLAGS"]) + return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags + begin + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end + rescue + r&.close + nil + else + return r, w + end + end + + def acquire_possible(r, w, max) + return unless tokens = r.read_nonblock(max - 1, exception: false) + if (jobs = tokens.size) > 0 + jobserver, w = w, nil + at_exit do + jobserver.print(tokens) + jobserver.close + end + end + return jobs + 1 + rescue Errno::EBADF + ensure + r&.close + w&.close + end + + def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"]) + if max > 1 and (r, w = connect(makeflags)) + acquire_possible(r, w, max) + end + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 1c2d5fd924..2663b7b76a 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -19,6 +19,7 @@ require_relative '../envutil' require_relative '../colorize' require_relative '../leakchecker' require_relative '../test/unit/testcase' +require_relative '../test/jobserver' require 'optparse' # See Test::Unit @@ -37,6 +38,26 @@ module Test class PendedError < AssertionFailedError; end + class << self + ## + # Extract the location where the last assertion method was + # called. Returns "<empty>" if _e_ does not have backtrace, or + # an empty string if no assertion method location was found. + + def location e + last_before_assertion = nil + + return '<empty>' unless e&.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /:in \W(?:.*\#)?(?:assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + return "" unless last_before_assertion + /:in / =~ last_before_assertion ? $` : last_before_assertion + end + end + module Order class NoSort def initialize(seed) @@ -229,6 +250,8 @@ module Test end module Parallel # :nodoc: all + attr_accessor :prefix + def process_args(args = []) return @options if @options options = super @@ -240,29 +263,10 @@ module Test def non_options(files, options) @jobserver = nil - makeflags = ENV.delete("MAKEFLAGS") - if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags - begin - if fifo = $3 - fifo.gsub!(/\\(?=.)/, '') - r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) - w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) - else - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) - end - rescue - r.close if r - nil - else - r.close_on_exec = true - w.close_on_exec = true - @jobserver = [r, w] - options[:parallel] ||= 256 # number of tokens to acquire first - end + if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS")) + options[:parallel] ||= 256 # number of tokens to acquire first end - @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 180) + @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) super end @@ -278,7 +282,7 @@ module Test opts.separator "parallel test options:" - options[:retry] = true + options[:retry] = false opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| options[:testing] = true & t # For testing @@ -350,8 +354,12 @@ module Test @io.puts(*args) end - def run(task,type) - @file = File.basename(task, ".rb") + def run(task, type, base = nil) + if base + @file = task.delete_prefix(base).chomp(".rb") + else + @file = File.basename(task, ".rb") + end @real_file = task begin puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" @@ -386,16 +394,19 @@ module Test rescue IOError end - def quit + def quit(reason = :normal) return if @io.closed? @quit_called = true - @io.puts "quit" + @io.puts "quit #{reason}" rescue Errno::EPIPE => e warn "#{@pid}:#{@status.to_s.ljust(7)}:#{@file}: #{e.message}" end def kill - Process.kill(:KILL, @pid) + EnvUtil::Debugger.search&.dump(@pid) + signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV + Process.kill(signal, @pid) + warn "worker #{to_s} does not respond; #{signal} is sent" rescue Errno::ESRCH end @@ -513,15 +524,15 @@ module Test @workers.reject! do |worker| next unless cond&.call(worker) begin - Timeout.timeout(1) do - worker.quit + Timeout.timeout(5) do + worker.quit(cond ? :timeout : :normal) end rescue Errno::EPIPE rescue Timeout::Error end closed&.push worker begin - Timeout.timeout(0.2) do + Timeout.timeout(1) do worker.close end rescue Timeout::Error @@ -534,7 +545,7 @@ module Test return if (closed ||= @workers).empty? pids = closed.map(&:pid) begin - Timeout.timeout(0.2 * closed.size) do + Timeout.timeout(1 * closed.size) do Process.waitall end rescue Timeout::Error @@ -575,7 +586,7 @@ module Test worker.quit worker = launch_worker end - worker.run(task, type) + worker.run(task, type, (@prefix unless @options[:job_status] == :replace)) @test_count += 1 jobs_status(worker) @@ -716,7 +727,15 @@ module Test del_status_line or puts error, suites = suites.partition {|r| r[:error]} unless suites.empty? - puts "\n""Retrying..." + puts "\n" + @failed_output.puts "Failed tests:" + suites.each {|r| + r[:report].each {|c, m, e| + @failed_output.puts "#{c}##{m}: #{e&.class}: #{e&.message&.slice(/\A.*/)}" + } + } + @failed_output.puts "\n" + puts "Retrying..." @verbose = options[:verbose] suites.map! {|r| ::Object.const_get(r[:testcase])} _run_suites(suites, type) @@ -842,7 +861,7 @@ module Test end end - def record(suite, method, assertions, time, error) + def record(suite, method, assertions, time, error, source_location = nil) if @options.values_at(:longest, :most_asserted).any? @tops ||= {} rec = [suite.name, method, assertions, time, error] @@ -1262,10 +1281,15 @@ module Test parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| options[:repeat_count] = n end + options[:keep_repeating] = false + parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n| + options[:keep_repeating] = true + end end def _run_anything(type) @repeat_count = @options[:repeat_count] + @keep_repeating = @options[:keep_repeating] super end end @@ -1353,6 +1377,95 @@ module Test end end + module LaunchableOption + module Nothing + private + def setup_options(opts, options) + super + opts.define_tail 'Launchable options:' + # This is expected to be called by Test::Unit::Worker. + opts.on_tail '--launchable-test-reports=PATH', String, 'Do nothing' + end + end + + def record(suite, method, assertions, time, error, source_location = nil) + if writer = @options[:launchable_test_reports] + if loc = (source_location || suite.instance_method(method).source_location) + path, lineno = loc + # Launchable JSON schema is defined at + # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code. + e = case error + when nil + status = 'TEST_PASSED' + nil + when Test::Unit::PendedError + status = 'TEST_SKIPPED' + "Skipped:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Test::Unit::AssertionFailedError + status = 'TEST_FAILED' + "Failure:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Timeout::Error + status = 'TEST_FAILED' + "Timeout:\n#{suite.name}##{method}\n" + else + status = 'TEST_FAILED' + bt = Test::filter_backtrace(error.backtrace).join "\n " + "Error:\n#{suite.name}##{method}:\n#{error.class}: #{error.message.b}\n #{bt}\n" + end + repo_path = File.expand_path("#{__dir__}/../../../") + relative_path = path.delete_prefix("#{repo_path}/") + # The test path is a URL-encoded representation. + # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18 + test_path = {file: relative_path, class: suite.name, testcase: method}.map{|key, val| + "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}" + }.join('#') + end + end + super + ensure + if writer && test_path && status + # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified. + # In such cases, we proceed to execute the operation here. + writer.write_object( + { + testPath: test_path, + status: status, + duration: time, + createdAt: Time.now.to_s, + stderr: e, + stdout: nil, + data: { + lineNumber: lineno + } + } + ) + end + end + + private + def setup_options(opts, options) + super + opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path| + require_relative '../launchable' + options[:launchable_test_reports] = writer = Launchable::JsonStreamWriter.new(path) + writer.write_array('testCases') + main_pid = Process.pid + at_exit { + # This block is executed when the fork block in a test is completed. + # Therefore, we need to verify whether all tests have been completed. + stack = caller + if stack.size == 0 && main_pid == Process.pid && $!.is_a?(SystemExit) + writer.close + end + } + end + + def encode_test_path_component component + component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26') + end + end + end + class Runner # :nodoc: all attr_accessor :report, :failures, :errors, :skips # :nodoc: @@ -1498,7 +1611,7 @@ module Test [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, t, @test_count.fdiv(t), @assertion_count.fdiv(t)] end while @repeat_count && @@current_repeat_count < @repeat_count && - report.empty? && failures.zero? && errors.zero? + (@keep_repeating || report.empty? && failures.zero? && errors.zero?) output.sync = old_sync if sync @@ -1557,9 +1670,7 @@ module Test puts if @verbose $stdout.flush - unless defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? # compiler process is wrongly considered as leak - leakchecker.check("#{inst.class}\##{inst.__name__}") - end + leakchecker.check("#{inst.class}\##{inst.__name__}") _end_method(inst) @@ -1590,19 +1701,11 @@ module Test # failure or error in teardown, it will be sent again with the # error or failure. - def record suite, method, assertions, time, error + def record suite, method, assertions, time, error, source_location = nil end def location e # :nodoc: - last_before_assertion = "" - - return '<empty>' unless e.backtrace # SystemStackError can return nil. - - e.backtrace.reverse_each do |s| - break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ - last_before_assertion = s - end - last_before_assertion.sub(/:in .*$/, '') + Test::Unit.location e end ## @@ -1681,6 +1784,7 @@ module Test prepend Test::Unit::ExcludesOption prepend Test::Unit::TimeoutOption prepend Test::Unit::RunCount + prepend Test::Unit::LaunchableOption::Nothing ## # Begins the full test run. Delegates to +runner+'s #_run method. @@ -1737,6 +1841,7 @@ module Test class AutoRunner # :nodoc: all class Runner < Test::Unit::Runner include Test::Unit::RequireFiles + include Test::Unit::LaunchableOption end attr_accessor :to_run, :options @@ -1745,6 +1850,7 @@ module Test @force_standalone = force_standalone @runner = Runner.new do |files, options| base = options[:base_directory] ||= default_dir + @runner.prefix = base ? (base + "/") : nil files << default_dir if files.empty? and default_dir @to_run = files yield self if block_given? diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index b4f1dbc176..19581fc3ab 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -522,7 +522,7 @@ module Test # Skips the current test. Gets listed at the end of the run but # doesn't cause a failure exit code. - def pend msg = nil, bt = caller + def pend msg = nil, bt = caller, &_ msg ||= "Skipped, no message given" @skip = true raise Test::Unit::PendedError, msg, bt @@ -768,7 +768,14 @@ EOT e = assert_raise(SyntaxError, mesg) do syntax_check(src, fname, line) end - assert_match(error, e.message, mesg) + + # Prism adds ANSI escape sequences to syntax error messages to + # colorize and format them. We strip them out here to make them easier + # to match against in tests. + message = e.message + message.gsub!(/\e\[.*?m/, "") + + assert_match(error, message, mesg) e end end @@ -791,33 +798,37 @@ EOT MIN_MEASURABLE = 1.0 / MIN_HZ def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) - require 'benchmark' - wait = EnvUtil.apply_timeout_scale(wait) if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" end - tms = Benchmark.measure(msg || '') do - if stop - th = Thread.start {sleep wait; stop.call} - yield - th.join - else - begin - Timeout.timeout(wait) {yield} - rescue Timeout::Error - end + + t0, r0 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error end end - max = pct * tms.real + t1, r1 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + total = t1.utime - t0.utime + t1.stime - t0.stime + t1.cutime - t0.cutime + t1.cstime - t0.cstime + real = r1 - r0 + + max = pct * real min_measurable = MIN_MEASURABLE min_measurable *= 1.30 # add a little (30%) to account for misc. overheads if max < min_measurable max = min_measurable end - assert_operator tms.total, :<=, max, msg + assert_operator total, :<=, max, msg end def assert_is_minus_zero(f) diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb index f2244ec20a..188a0d1a19 100644 --- a/tool/lib/test/unit/parallel.rb +++ b/tool/lib/test/unit/parallel.rb @@ -127,7 +127,18 @@ module Test else _report "ready" end - when /^quit$/ + when /^quit (.+?)$/, "quit" + if $1 == "timeout" + err = ["", "!!! worker #{$$} killed due to timeout:"] + Thread.list.each do |th| + err << "#{ th.inspect }:" + th.backtrace.each do |s| + err << " #{ s }" + end + end + err << "" + STDERR.puts err.join("\n") + end _report "bye" exit end @@ -180,7 +191,7 @@ module Test else error = ProxyError.new(error) end - _report "record", Marshal.dump([suite.name, method, assertions, time, error]) + _report "record", Marshal.dump([suite.name, method, assertions, time, error, suite.instance_method(method).source_location]) super end end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb index 7ed6c677e3..51ffff37eb 100644 --- a/tool/lib/test/unit/testcase.rb +++ b/tool/lib/test/unit/testcase.rb @@ -137,6 +137,9 @@ module Test attr_reader :__name__ # :nodoc: + # Method name of this test. + alias method_name __name__ + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, Interrupt, SystemExit] # :nodoc: diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb index 900ea59017..26c9763c13 100644 --- a/tool/lib/vcs.rb +++ b/tool/lib/vcs.rb @@ -51,23 +51,9 @@ module DebugPOpen end using DebugPOpen module DebugSystem - def system(*args) + def system(*args, exception: true, **opts) VCS.dump(args, "args: ") if $DEBUG - exception = false - opts = Hash.try_convert(args[-1]) - if RUBY_VERSION >= "2.6" - unless opts - opts = {} - args << opts - end - exception = opts.fetch(:exception) {opts[:exception] = true} - elsif opts - exception = opts.delete(:exception) {true} - args.pop if opts.empty? - end - ret = super(*args) - raise "Command failed with status (#$?): #{args[0]}" if exception and !ret - ret + super(*args, exception: exception, **opts) end end @@ -226,6 +212,7 @@ class VCS def after_export(dir) FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + FileUtils.rm_rf(Dir.glob("#{dir}/.mailmap")) end def revision_handler(rev) @@ -270,155 +257,6 @@ class VCS code end - class SVN < self - register(".svn") - COMMAND = ENV['SVN'] || 'svn' - - def self.revision_name(rev) - "r#{rev}" - end - - def _get_revisions(path, srcdir = nil) - if srcdir and self.class.local_path?(path) - path = File.join(srcdir, path) - end - if srcdir - info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") - info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s - end - info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") - _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) - modified = info_xml[/<date>([^<>]*)/, 1] - branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] - [Integer(last), Integer(changed), modified, branch] - end - - def self.search_root(path) - return unless local_path?(path) - parent = File.realpath(path) - begin - parent = File.dirname(wkdir = parent) - return wkdir if File.directory?(wkdir + "/.svn") - end until parent == wkdir - end - - def get_info - @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") - end - - def url - @url ||= begin - url = get_info[/<root>(.*)<\/root>/, 1] - @url = URI.parse(url+"/") if url - end - end - - def wcroot - @wcroot ||= begin - info = get_info - @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] - @wcroot ||= self.class.search_root(@srcdir) - end - end - - def branch(name) - return trunk if name == "trunk" - url + "branches/#{name}" - end - - def tag(name) - url + "tags/#{name}" - end - - def trunk - url + "trunk" - end - alias master trunk - - def branch_list(pat) - IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| - f.each do |line| - line.chomp! - line.chomp!('/') - yield(line) if File.fnmatch?(pat, line) - end - end - end - - def grep(pat, tag, *files, &block) - cmd = %W"#{COMMAND} cat" - files.map! {|n| File.join(tag, n)} if tag - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files]) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end - - def export(revision, url, dir, keep_temp = false) - if @srcdir and (rootdir = wcroot) - srcdir = File.realpath(@srcdir) - rootdir << "/" - if srcdir.start_with?(rootdir) - subdir = srcdir[rootdir.size..-1] - subdir = nil if subdir.empty? - FileUtils.mkdir_p(svndir = dir+"/.svn") - FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) - system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false - FileUtils.rm_rf(svndir) unless keep_temp - if subdir - tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") - File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") - FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) - begin - Dir.rmdir("#{dir}/#{subdir}") - end until (subdir = File.dirname(subdir)) == '.' - FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) - Dir.rmdir(tmpdir) - end - return self - end - end - IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| - pipe.each {|line| /^A/ =~ line or yield line} - end - self if $?.success? - end - - def after_export(dir) - super - FileUtils.rm_rf(dir+"/.svn") - end - - def branch_beginning(url) - # `--limit` of svn-log is useless in this case, because it is - # applied before `--search`. - rev = IO.pread(%W[ #{COMMAND} log --xml - --search=matz --search-and=has\ started - -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] - rev.to_i if rev - end - - def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path) - range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') - IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -r#{range} #{url}") do |r| - IO.copy_stream(r, path) - end - end - - def commit - args = %W"#{COMMAND} commit" - if dryrun? - VCS.dump(args, "commit: ") - return true - end - system(*args) - end - end - class GIT < self register(".git") do |path, dir| SAFE_DIRECTORIES ||= @@ -478,7 +316,7 @@ class VCS last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref, err: w]]).rstrip w.close unless r.eof? - raise "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" + raise VCS::NotFoundError, "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" end end log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) @@ -532,15 +370,6 @@ class VCS rev[0, 10] end - def revision_handler(rev) - case rev - when Integer - SVN - else - super - end - end - def without_gitconfig envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| h[v] = ENV.delete(v) @@ -615,32 +444,26 @@ class VCS def export(revision, url, dir, keep_temp = false) system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return - (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir)) + GIT.new(File.expand_path(dir)) end def branch_beginning(url) - cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H + year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i + cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25 --author=matz --committer=matz --grep=started\\.$ - #{url.to_str} -- version.h include/ruby/version.h]) + #{url} -- version.h include/ruby/version.h])[/.*/] end - def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil) - svn = nil + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true) from, to = [from, to].map do |rev| rev or next - if Integer === rev - svn = true - rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -n1 --format=format:%H" << - "--grep=^ *git-svn-id: .*@#{rev} ") - end rev unless rev.empty? end - unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from)) + unless from&.match?(/./) or (from = branch_beginning(url))&.match?(/./) warn "no starting commit found", uplevel: 1 from = nil end - if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", chdir: @srcdir, exception: false) system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", chdir: @srcdir, exception: false) @@ -653,22 +476,18 @@ class VCS else arg = ["--since=25 Dec 00:00:00", to] end - writer = - if svn - format_changelog_as_svn(path, arg) - else - if base_url == true - remote, = upstream - if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) - remote.chomp! - # hack to redirect git.r-l.o to github - remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') - remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') - end - base_url = remote - end - format_changelog(path, arg, base_url) + if base_url == true + env = CHANGELOG_ENV + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') end + base_url = remote + end + writer = changelog_formatter(path, arg, base_url) if !path or path == '-' writer[$stdout] else @@ -677,9 +496,10 @@ class VCS end LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' + CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} - def format_changelog(path, arg, base_url = nil) - env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + def changelog_formatter(path, arg, base_url = nil) + env = CHANGELOG_ENV cmd = %W[#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] @@ -691,17 +511,32 @@ class VCS cmd << date cmd.concat(arg) proc do |w| - w.print "-*- coding: utf-8 -*-\n\n" - w.print "base-url = #{base_url}\n\n" if base_url + w.print "-*- coding: utf-8 -*-\n" + w.print "\n""base-url = #{base_url}\n" if base_url + + begin + ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true) + .grep_v(/^ *(?:#|$)/) + .to_h {|v| [v, true]} + ignore_revs = nil if ignore_revs.empty? + rescue Errno::ENOENT + end + cmd_pipe(env, cmd, chdir: @srcdir) do |r| - while s = r.gets("\ncommit ") + r.gets(sep = "commit ") + sep = "\n" + sep + while s = r.gets(sep, chomp: true) h, s = s.split(/^$/, 2) + if ignore_revs&.key?(h[/\A\h{40}/]) + next + end next if /^Author: *dependabot\[bot\]/ =~ h h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') fix = $1 + next if /\A *skip\Z/ =~ fix s = s.lines fix.each_line do |x| next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') @@ -738,7 +573,7 @@ class VCS next end end - message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] + message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] from = [1, n-2].max to = [s.size-1, n+2].min s.each_with_index do |e, i| @@ -760,6 +595,10 @@ class VCS s = s.join('') end + s.gsub!(%r[(?!<\w)([-\w]+/[-\w]+)(?:@(\h{8,40})|#(\d{5,}))\b]) do + path = defined?($2) ? "commit/#{$2}" : "pull/#{$3}" + "[#$&](https://github.com/#{$1}/#{path})" + end if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s issue = "#{$1}pull/" s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} @@ -767,28 +606,7 @@ class VCS s.gsub!(/ +\n/, "\n") s.sub!(/^Notes:/, ' \&') - w.print h, s - end - end - end - end - - def format_changelog_as_svn(path, arg) - cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B" - cmd.concat(arg) - proc do |w| - sep = "-"*72 + "\n" - w.print sep - cmd_pipe(cmd) do |r| - while s = r.gets("\0") - s.chomp!("\0") - author, time, s = s.split("\n", 3) - s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '') - rev = $1 - time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)") - lines = s.count("\n") + 1 - lines = "#{lines} line#{lines == 1 ? '' : 's'}" - w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep + w.print sep, h, s end end end @@ -825,46 +643,6 @@ class VCS end end - class GITSVN < GIT - def self.revision_name(rev) - SVN.revision_name(rev) - end - - def last_changed_revision - rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] - com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp - return rev, com - end - - def commit(opts = {}) - rev, com = last_changed_revision - head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp - - commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") - commits.each_with_index do |l, i| - r, a, c = l.split(' ') - dcommit = [COMMAND, "svn", "dcommit"] - dcommit.insert(-2, "-n") if dryrun? - dcommit << "--add-author-from" unless a == c - dcommit << r - system(*dcommit) or return false - system(COMMAND, "checkout", head) or return false - system(COMMAND, "rebase") or return false - end - - if rev - old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] - old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] - 3.times do - sleep 2 - system(*%W"#{COMMAND} pull --no-edit --rebase") - break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) - end - end - true - end - end - class Null < self def get_revisions(path, srcdir = nil) @modified ||= Time.now - 10 diff --git a/tool/lib/webrick.rb b/tool/lib/webrick.rb deleted file mode 100644 index b854b68db4..0000000000 --- a/tool/lib/webrick.rb +++ /dev/null @@ -1,232 +0,0 @@ -# frozen_string_literal: false -## -# = WEB server toolkit. -# -# WEBrick is an HTTP server toolkit that can be configured as an HTTPS server, -# a proxy server, and a virtual-host server. WEBrick features complete -# logging of both server operations and HTTP access. WEBrick supports both -# basic and digest authentication in addition to algorithms not in RFC 2617. -# -# A WEBrick server can be composed of multiple WEBrick servers or servlets to -# provide differing behavior on a per-host or per-path basis. WEBrick -# includes servlets for handling CGI scripts, ERB pages, Ruby blocks and -# directory listings. -# -# WEBrick also includes tools for daemonizing a process and starting a process -# at a higher privilege level and dropping permissions. -# -# == Security -# -# *Warning:* WEBrick is not recommended for production. It only implements -# basic security checks. -# -# == Starting an HTTP server -# -# To create a new WEBrick::HTTPServer that will listen to connections on port -# 8000 and serve documents from the current user's public_html folder: -# -# require 'webrick' -# -# root = File.expand_path '~/public_html' -# server = WEBrick::HTTPServer.new :Port => 8000, :DocumentRoot => root -# -# To run the server you will need to provide a suitable shutdown hook as -# starting the server blocks the current thread: -# -# trap 'INT' do server.shutdown end -# -# server.start -# -# == Custom Behavior -# -# The easiest way to have a server perform custom operations is through -# WEBrick::HTTPServer#mount_proc. The block given will be called with a -# WEBrick::HTTPRequest with request info and a WEBrick::HTTPResponse which -# must be filled in appropriately: -# -# server.mount_proc '/' do |req, res| -# res.body = 'Hello, world!' -# end -# -# Remember that +server.mount_proc+ must precede +server.start+. -# -# == Servlets -# -# Advanced custom behavior can be obtained through mounting a subclass of -# WEBrick::HTTPServlet::AbstractServlet. Servlets provide more modularity -# when writing an HTTP server than mount_proc allows. Here is a simple -# servlet: -# -# class Simple < WEBrick::HTTPServlet::AbstractServlet -# def do_GET request, response -# status, content_type, body = do_stuff_with request -# -# response.status = 200 -# response['Content-Type'] = 'text/plain' -# response.body = 'Hello, World!' -# end -# end -# -# To initialize the servlet you mount it on the server: -# -# server.mount '/simple', Simple -# -# See WEBrick::HTTPServlet::AbstractServlet for more details. -# -# == Virtual Hosts -# -# A server can act as a virtual host for multiple host names. After creating -# the listening host, additional hosts that do not listen can be created and -# attached as virtual hosts: -# -# server = WEBrick::HTTPServer.new # ... -# -# vhost = WEBrick::HTTPServer.new :ServerName => 'vhost.example', -# :DoNotListen => true, # ... -# vhost.mount '/', ... -# -# server.virtual_host vhost -# -# If no +:DocumentRoot+ is provided and no servlets or procs are mounted on the -# main server it will return 404 for all URLs. -# -# == HTTPS -# -# To create an HTTPS server you only need to enable SSL and provide an SSL -# certificate name: -# -# require 'webrick' -# require 'webrick/https' -# -# cert_name = [ -# %w[CN localhost], -# ] -# -# server = WEBrick::HTTPServer.new(:Port => 8000, -# :SSLEnable => true, -# :SSLCertName => cert_name) -# -# This will start the server with a self-generated self-signed certificate. -# The certificate will be changed every time the server is restarted. -# -# To create a server with a pre-determined key and certificate you can provide -# them: -# -# require 'webrick' -# require 'webrick/https' -# require 'openssl' -# -# cert = OpenSSL::X509::Certificate.new File.read '/path/to/cert.pem' -# pkey = OpenSSL::PKey::RSA.new File.read '/path/to/pkey.pem' -# -# server = WEBrick::HTTPServer.new(:Port => 8000, -# :SSLEnable => true, -# :SSLCertificate => cert, -# :SSLPrivateKey => pkey) -# -# == Proxy Server -# -# WEBrick can act as a proxy server: -# -# require 'webrick' -# require 'webrick/httpproxy' -# -# proxy = WEBrick::HTTPProxyServer.new :Port => 8000 -# -# trap 'INT' do proxy.shutdown end -# -# See WEBrick::HTTPProxy for further details including modifying proxied -# responses. -# -# == Basic and Digest authentication -# -# WEBrick provides both Basic and Digest authentication for regular and proxy -# servers. See WEBrick::HTTPAuth, WEBrick::HTTPAuth::BasicAuth and -# WEBrick::HTTPAuth::DigestAuth. -# -# == WEBrick as a daemonized Web Server -# -# WEBrick can be run as a daemonized server for small loads. -# -# === Daemonizing -# -# To start a WEBrick server as a daemon simple run WEBrick::Daemon.start -# before starting the server. -# -# === Dropping Permissions -# -# WEBrick can be started as one user to gain permission to bind to port 80 or -# 443 for serving HTTP or HTTPS traffic then can drop these permissions for -# regular operation. To listen on all interfaces for HTTP traffic: -# -# sockets = WEBrick::Utils.create_listeners nil, 80 -# -# Then drop privileges: -# -# WEBrick::Utils.su 'www' -# -# Then create a server that does not listen by default: -# -# server = WEBrick::HTTPServer.new :DoNotListen => true, # ... -# -# Then overwrite the listening sockets with the port 80 sockets: -# -# server.listeners.replace sockets -# -# === Logging -# -# WEBrick can separately log server operations and end-user access. For -# server operations: -# -# log_file = File.open '/var/log/webrick.log', 'a+' -# log = WEBrick::Log.new log_file -# -# For user access logging: -# -# access_log = [ -# [log_file, WEBrick::AccessLog::COMBINED_LOG_FORMAT], -# ] -# -# server = WEBrick::HTTPServer.new :Logger => log, :AccessLog => access_log -# -# See WEBrick::AccessLog for further log formats. -# -# === Log Rotation -# -# To rotate logs in WEBrick on a HUP signal (like syslogd can send), open the -# log file in 'a+' mode (as above) and trap 'HUP' to reopen the log file: -# -# trap 'HUP' do log_file.reopen '/path/to/webrick.log', 'a+' -# -# == Copyright -# -# Author: IPR -- Internet Programming with Ruby -- writers -# -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -#-- -# $IPR: webrick.rb,v 1.12 2002/10/01 17:16:31 gotoyuzo Exp $ - -module WEBrick -end - -require 'webrick/compat.rb' - -require 'webrick/version.rb' -require 'webrick/config.rb' -require 'webrick/log.rb' -require 'webrick/server.rb' -require_relative 'webrick/utils.rb' -require 'webrick/accesslog' - -require 'webrick/htmlutils.rb' -require 'webrick/httputils.rb' -require 'webrick/cookie.rb' -require 'webrick/httpversion.rb' -require 'webrick/httpstatus.rb' -require 'webrick/httprequest.rb' -require 'webrick/httpresponse.rb' -require 'webrick/httpserver.rb' -require 'webrick/httpservlet.rb' -require 'webrick/httpauth.rb' diff --git a/tool/lib/webrick/.document b/tool/lib/webrick/.document deleted file mode 100644 index c62f89083b..0000000000 --- a/tool/lib/webrick/.document +++ /dev/null @@ -1,6 +0,0 @@ -# Add files to this as they become documented - -*.rb - -httpauth -httpservlet diff --git a/tool/lib/webrick/accesslog.rb b/tool/lib/webrick/accesslog.rb deleted file mode 100644 index e4849637f3..0000000000 --- a/tool/lib/webrick/accesslog.rb +++ /dev/null @@ -1,157 +0,0 @@ -# frozen_string_literal: false -#-- -# accesslog.rb -- Access log handling utilities -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 keita yamaguchi -# Copyright (c) 2002 Internet Programming with Ruby writers -# -# $IPR: accesslog.rb,v 1.1 2002/10/01 17:16:32 gotoyuzo Exp $ - -module WEBrick - - ## - # AccessLog provides logging to various files in various formats. - # - # Multiple logs may be written to at the same time: - # - # access_log = [ - # [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT], - # [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT], - # ] - # - # server = WEBrick::HTTPServer.new :AccessLog => access_log - # - # Custom log formats may be defined. WEBrick::AccessLog provides a subset - # of the formatting from Apache's mod_log_config - # http://httpd.apache.org/docs/mod/mod_log_config.html#formats. See - # AccessLog::setup_params for a list of supported options - - module AccessLog - - ## - # Raised if a parameter such as %e, %i, %o or %n is used without fetching - # a specific field. - - class AccessLogError < StandardError; end - - ## - # The Common Log Format's time format - - CLF_TIME_FORMAT = "[%d/%b/%Y:%H:%M:%S %Z]" - - ## - # Common Log Format - - COMMON_LOG_FORMAT = "%h %l %u %t \"%r\" %s %b" - - ## - # Short alias for Common Log Format - - CLF = COMMON_LOG_FORMAT - - ## - # Referer Log Format - - REFERER_LOG_FORMAT = "%{Referer}i -> %U" - - ## - # User-Agent Log Format - - AGENT_LOG_FORMAT = "%{User-Agent}i" - - ## - # Combined Log Format - - COMBINED_LOG_FORMAT = "#{CLF} \"%{Referer}i\" \"%{User-agent}i\"" - - module_function - - # This format specification is a subset of mod_log_config of Apache: - # - # %a:: Remote IP address - # %b:: Total response size - # %e{variable}:: Given variable in ENV - # %f:: Response filename - # %h:: Remote host name - # %{header}i:: Given request header - # %l:: Remote logname, always "-" - # %m:: Request method - # %{attr}n:: Given request attribute from <tt>req.attributes</tt> - # %{header}o:: Given response header - # %p:: Server's request port - # %{format}p:: The canonical port of the server serving the request or the - # actual port or the client's actual port. Valid formats are - # canonical, local or remote. - # %q:: Request query string - # %r:: First line of the request - # %s:: Request status - # %t:: Time the request was received - # %T:: Time taken to process the request - # %u:: Remote user from auth - # %U:: Unparsed URI - # %%:: Literal % - - def setup_params(config, req, res) - params = Hash.new("") - params["a"] = req.peeraddr[3] - params["b"] = res.sent_size - params["e"] = ENV - params["f"] = res.filename || "" - params["h"] = req.peeraddr[2] - params["i"] = req - params["l"] = "-" - params["m"] = req.request_method - params["n"] = req.attributes - params["o"] = res - params["p"] = req.port - params["q"] = req.query_string - params["r"] = req.request_line.sub(/\x0d?\x0a\z/o, '') - params["s"] = res.status # won't support "%>s" - params["t"] = req.request_time - params["T"] = Time.now - req.request_time - params["u"] = req.user || "-" - params["U"] = req.unparsed_uri - params["v"] = config[:ServerName] - params - end - - ## - # Formats +params+ according to +format_string+ which is described in - # setup_params. - - def format(format_string, params) - format_string.gsub(/\%(?:\{(.*?)\})?>?([a-zA-Z%])/){ - param, spec = $1, $2 - case spec[0] - when ?e, ?i, ?n, ?o - raise AccessLogError, - "parameter is required for \"#{spec}\"" unless param - (param = params[spec][param]) ? escape(param) : "-" - when ?t - params[spec].strftime(param || CLF_TIME_FORMAT) - when ?p - case param - when 'remote' - escape(params["i"].peeraddr[1].to_s) - else - escape(params["p"].to_s) - end - when ?% - "%" - else - escape(params[spec].to_s) - end - } - end - - ## - # Escapes control characters in +data+ - - def escape(data) - data = data.gsub(/[[:cntrl:]\\]+/) {$&.dump[1...-1]} - data.untaint if RUBY_VERSION < '2.7' - data - end - end -end diff --git a/tool/lib/webrick/cgi.rb b/tool/lib/webrick/cgi.rb deleted file mode 100644 index bb0ae2fc84..0000000000 --- a/tool/lib/webrick/cgi.rb +++ /dev/null @@ -1,313 +0,0 @@ -# frozen_string_literal: false -# -# cgi.rb -- Yet another CGI library -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $Id$ - -require_relative "httprequest" -require_relative "httpresponse" -require_relative "config" -require "stringio" - -module WEBrick - - # A CGI library using WEBrick requests and responses. - # - # Example: - # - # class MyCGI < WEBrick::CGI - # def do_GET req, res - # res.body = 'it worked!' - # res.status = 200 - # end - # end - # - # MyCGI.new.start - - class CGI - - # The CGI error exception class - - CGIError = Class.new(StandardError) - - ## - # The CGI configuration. This is based on WEBrick::Config::HTTP - - attr_reader :config - - ## - # The CGI logger - - attr_reader :logger - - ## - # Creates a new CGI interface. - # - # The first argument in +args+ is a configuration hash which would update - # WEBrick::Config::HTTP. - # - # Any remaining arguments are stored in the <code>@options</code> instance - # variable for use by a subclass. - - def initialize(*args) - if defined?(MOD_RUBY) - unless ENV.has_key?("GATEWAY_INTERFACE") - Apache.request.setup_cgi_env - end - end - if %r{HTTP/(\d+\.\d+)} =~ ENV["SERVER_PROTOCOL"] - httpv = $1 - end - @config = WEBrick::Config::HTTP.dup.update( - :ServerSoftware => ENV["SERVER_SOFTWARE"] || "null", - :HTTPVersion => HTTPVersion.new(httpv || "1.0"), - :RunOnCGI => true, # to detect if it runs on CGI. - :NPH => false # set true to run as NPH script. - ) - if config = args.shift - @config.update(config) - end - @config[:Logger] ||= WEBrick::BasicLog.new($stderr) - @logger = @config[:Logger] - @options = args - end - - ## - # Reads +key+ from the configuration - - def [](key) - @config[key] - end - - ## - # Starts the CGI process with the given environment +env+ and standard - # input and output +stdin+ and +stdout+. - - def start(env=ENV, stdin=$stdin, stdout=$stdout) - sock = WEBrick::CGI::Socket.new(@config, env, stdin, stdout) - req = HTTPRequest.new(@config) - res = HTTPResponse.new(@config) - unless @config[:NPH] or defined?(MOD_RUBY) - def res.setup_header - unless @header["status"] - phrase = HTTPStatus::reason_phrase(@status) - @header["status"] = "#{@status} #{phrase}" - end - super - end - def res.status_line - "" - end - end - - begin - req.parse(sock) - req.script_name = (env["SCRIPT_NAME"] || File.expand_path($0)).dup - req.path_info = (env["PATH_INFO"] || "").dup - req.query_string = env["QUERY_STRING"] - req.user = env["REMOTE_USER"] - res.request_method = req.request_method - res.request_uri = req.request_uri - res.request_http_version = req.http_version - res.keep_alive = req.keep_alive? - self.service(req, res) - rescue HTTPStatus::Error => ex - res.set_error(ex) - rescue HTTPStatus::Status => ex - res.status = ex.code - rescue Exception => ex - @logger.error(ex) - res.set_error(ex, true) - ensure - req.fixup - if defined?(MOD_RUBY) - res.setup_header - Apache.request.status_line = "#{res.status} #{res.reason_phrase}" - Apache.request.status = res.status - table = Apache.request.headers_out - res.header.each{|key, val| - case key - when /^content-encoding$/i - Apache::request.content_encoding = val - when /^content-type$/i - Apache::request.content_type = val - else - table[key] = val.to_s - end - } - res.cookies.each{|cookie| - table.add("Set-Cookie", cookie.to_s) - } - Apache.request.send_http_header - res.send_body(sock) - else - res.send_response(sock) - end - end - end - - ## - # Services the request +req+ which will fill in the response +res+. See - # WEBrick::HTTPServlet::AbstractServlet#service for details. - - def service(req, res) - method_name = "do_" + req.request_method.gsub(/-/, "_") - if respond_to?(method_name) - __send__(method_name, req, res) - else - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - end - end - - ## - # Provides HTTP socket emulation from the CGI environment - - class Socket # :nodoc: - include Enumerable - - private - - def initialize(config, env, stdin, stdout) - @config = config - @env = env - @header_part = StringIO.new - @body_part = stdin - @out_port = stdout - @out_port.binmode - - @server_addr = @env["SERVER_ADDR"] || "0.0.0.0" - @server_name = @env["SERVER_NAME"] - @server_port = @env["SERVER_PORT"] - @remote_addr = @env["REMOTE_ADDR"] - @remote_host = @env["REMOTE_HOST"] || @remote_addr - @remote_port = @env["REMOTE_PORT"] || 0 - - begin - @header_part << request_line << CRLF - setup_header - @header_part << CRLF - @header_part.rewind - rescue Exception - raise CGIError, "invalid CGI environment" - end - end - - def request_line - meth = @env["REQUEST_METHOD"] || "GET" - unless url = @env["REQUEST_URI"] - url = (@env["SCRIPT_NAME"] || File.expand_path($0)).dup - url << @env["PATH_INFO"].to_s - url = WEBrick::HTTPUtils.escape_path(url) - if query_string = @env["QUERY_STRING"] - unless query_string.empty? - url << "?" << query_string - end - end - end - # we cannot get real HTTP version of client ;) - httpv = @config[:HTTPVersion] - return "#{meth} #{url} HTTP/#{httpv}" - end - - def setup_header - @env.each{|key, value| - case key - when "CONTENT_TYPE", "CONTENT_LENGTH" - add_header(key.gsub(/_/, "-"), value) - when /^HTTP_(.*)/ - add_header($1.gsub(/_/, "-"), value) - end - } - end - - def add_header(hdrname, value) - unless value.empty? - @header_part << hdrname << ": " << value << CRLF - end - end - - def input - @header_part.eof? ? @body_part : @header_part - end - - public - - def peeraddr - [nil, @remote_port, @remote_host, @remote_addr] - end - - def addr - [nil, @server_port, @server_name, @server_addr] - end - - def gets(eol=LF, size=nil) - input.gets(eol, size) - end - - def read(size=nil) - input.read(size) - end - - def each - input.each{|line| yield(line) } - end - - def eof? - input.eof? - end - - def <<(data) - @out_port << data - end - - def write(data) - @out_port.write(data) - end - - def cert - return nil unless defined?(OpenSSL) - if pem = @env["SSL_SERVER_CERT"] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - - def peer_cert - return nil unless defined?(OpenSSL) - if pem = @env["SSL_CLIENT_CERT"] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - - def peer_cert_chain - return nil unless defined?(OpenSSL) - if @env["SSL_CLIENT_CERT_CHAIN_0"] - keys = @env.keys - certs = keys.sort.collect{|k| - if /^SSL_CLIENT_CERT_CHAIN_\d+$/ =~ k - if pem = @env[k] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - } - certs.compact - end - end - - def cipher - return nil unless defined?(OpenSSL) - if cipher = @env["SSL_CIPHER"] - ret = [ cipher ] - ret << @env["SSL_PROTOCOL"] - ret << @env["SSL_CIPHER_USEKEYSIZE"] - ret << @env["SSL_CIPHER_ALGKEYSIZE"] - ret - end - end - end - end -end diff --git a/tool/lib/webrick/compat.rb b/tool/lib/webrick/compat.rb deleted file mode 100644 index c497a1933c..0000000000 --- a/tool/lib/webrick/compat.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: false -# -# compat.rb -- cross platform compatibility -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: compat.rb,v 1.6 2002/10/01 17:16:32 gotoyuzo Exp $ - -## -# System call error module used by webrick for cross platform compatibility. -# -# EPROTO:: protocol error -# ECONNRESET:: remote host reset the connection request -# ECONNABORTED:: Client sent TCP reset (RST) before server has accepted the -# connection requested by client. -# -module Errno - ## - # Protocol error. - - class EPROTO < SystemCallError; end - - ## - # Remote host reset the connection request. - - class ECONNRESET < SystemCallError; end - - ## - # Client sent TCP reset (RST) before server has accepted the connection - # requested by client. - - class ECONNABORTED < SystemCallError; end -end diff --git a/tool/lib/webrick/config.rb b/tool/lib/webrick/config.rb deleted file mode 100644 index 9f2ab44f49..0000000000 --- a/tool/lib/webrick/config.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: false -# -# config.rb -- Default configurations. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: config.rb,v 1.52 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'version' -require_relative 'httpversion' -require_relative 'httputils' -require_relative 'utils' -require_relative 'log' - -module WEBrick - module Config - LIBDIR = File::dirname(__FILE__) # :nodoc: - - # for GenericServer - General = Hash.new { |hash, key| - case key - when :ServerName - hash[key] = Utils.getservername - else - nil - end - }.update( - :BindAddress => nil, # "0.0.0.0" or "::" or nil - :Port => nil, # users MUST specify this!! - :MaxClients => 100, # maximum number of the concurrent connections - :ServerType => nil, # default: WEBrick::SimpleServer - :Logger => nil, # default: WEBrick::Log.new - :ServerSoftware => "WEBrick/#{WEBrick::VERSION} " + - "(Ruby/#{RUBY_VERSION}/#{RUBY_RELEASE_DATE})", - :TempDir => ENV['TMPDIR']||ENV['TMP']||ENV['TEMP']||'/tmp', - :DoNotListen => false, - :StartCallback => nil, - :StopCallback => nil, - :AcceptCallback => nil, - :DoNotReverseLookup => true, - :ShutdownSocketWithoutClose => false, - ) - - # for HTTPServer, HTTPRequest, HTTPResponse ... - HTTP = General.dup.update( - :Port => 80, - :RequestTimeout => 30, - :HTTPVersion => HTTPVersion.new("1.1"), - :AccessLog => nil, - :MimeTypes => HTTPUtils::DefaultMimeTypes, - :DirectoryIndex => ["index.html","index.htm","index.cgi","index.rhtml"], - :DocumentRoot => nil, - :DocumentRootOptions => { :FancyIndexing => true }, - :RequestCallback => nil, - :ServerAlias => nil, - :InputBufferSize => 65536, # input buffer size in reading request body - :OutputBufferSize => 65536, # output buffer size in sending File or IO - - # for HTTPProxyServer - :ProxyAuthProc => nil, - :ProxyContentHandler => nil, - :ProxyVia => true, - :ProxyTimeout => true, - :ProxyURI => nil, - - :CGIInterpreter => nil, - :CGIPathEnv => nil, - - # workaround: if Request-URIs contain 8bit chars, - # they should be escaped before calling of URI::parse(). - :Escape8bitURI => false - ) - - ## - # Default configuration for WEBrick::HTTPServlet::FileHandler - # - # :AcceptableLanguages:: - # Array of languages allowed for accept-language. There is no default - # :DirectoryCallback:: - # Allows preprocessing of directory requests. There is no default - # callback. - # :FancyIndexing:: - # If true, show an index for directories. The default is true. - # :FileCallback:: - # Allows preprocessing of file requests. There is no default callback. - # :HandlerCallback:: - # Allows preprocessing of requests. There is no default callback. - # :HandlerTable:: - # Maps file suffixes to file handlers. DefaultFileHandler is used by - # default but any servlet can be used. - # :NondisclosureName:: - # Do not show files matching this array of globs. .ht* and *~ are - # excluded by default. - # :UserDir:: - # Directory inside ~user to serve content from for /~user requests. - # Only works if mounted on /. Disabled by default. - - FileHandler = { - :NondisclosureName => [".ht*", "*~"], - :FancyIndexing => false, - :HandlerTable => {}, - :HandlerCallback => nil, - :DirectoryCallback => nil, - :FileCallback => nil, - :UserDir => nil, # e.g. "public_html" - :AcceptableLanguages => [] # ["en", "ja", ... ] - } - - ## - # Default configuration for WEBrick::HTTPAuth::BasicAuth - # - # :AutoReloadUserDB:: Reload the user database provided by :UserDB - # automatically? - - BasicAuth = { - :AutoReloadUserDB => true, - } - - ## - # Default configuration for WEBrick::HTTPAuth::DigestAuth. - # - # :Algorithm:: MD5, MD5-sess (default), SHA1, SHA1-sess - # :Domain:: An Array of URIs that define the protected space - # :Qop:: 'auth' for authentication, 'auth-int' for integrity protection or - # both - # :UseOpaque:: Should the server send opaque values to the client? This - # helps prevent replay attacks. - # :CheckNc:: Should the server check the nonce count? This helps the - # server detect replay attacks. - # :UseAuthenticationInfoHeader:: Should the server send an - # AuthenticationInfo header? - # :AutoReloadUserDB:: Reload the user database provided by :UserDB - # automatically? - # :NonceExpirePeriod:: How long should we store used nonces? Default is - # 30 minutes. - # :NonceExpireDelta:: How long is a nonce valid? Default is 1 minute - # :InternetExplorerHack:: Hack which allows Internet Explorer to work. - # :OperaHack:: Hack which allows Opera to work. - - DigestAuth = { - :Algorithm => 'MD5-sess', # or 'MD5' - :Domain => nil, # an array includes domain names. - :Qop => [ 'auth' ], # 'auth' or 'auth-int' or both. - :UseOpaque => true, - :UseNextNonce => false, - :CheckNc => false, - :UseAuthenticationInfoHeader => true, - :AutoReloadUserDB => true, - :NonceExpirePeriod => 30*60, - :NonceExpireDelta => 60, - :InternetExplorerHack => true, - :OperaHack => true, - } - end -end diff --git a/tool/lib/webrick/cookie.rb b/tool/lib/webrick/cookie.rb deleted file mode 100644 index 5fd3bfb228..0000000000 --- a/tool/lib/webrick/cookie.rb +++ /dev/null @@ -1,172 +0,0 @@ -# frozen_string_literal: false -# -# cookie.rb -- Cookie class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cookie.rb,v 1.16 2002/09/21 12:23:35 gotoyuzo Exp $ - -require 'time' -require_relative 'httputils' - -module WEBrick - - ## - # Processes HTTP cookies - - class Cookie - - ## - # The cookie name - - attr_reader :name - - ## - # The cookie value - - attr_accessor :value - - ## - # The cookie version - - attr_accessor :version - - ## - # The cookie domain - attr_accessor :domain - - ## - # The cookie path - - attr_accessor :path - - ## - # Is this a secure cookie? - - attr_accessor :secure - - ## - # The cookie comment - - attr_accessor :comment - - ## - # The maximum age of the cookie - - attr_accessor :max_age - - #attr_accessor :comment_url, :discard, :port - - ## - # Creates a new cookie with the given +name+ and +value+ - - def initialize(name, value) - @name = name - @value = value - @version = 0 # Netscape Cookie - - @domain = @path = @secure = @comment = @max_age = - @expires = @comment_url = @discard = @port = nil - end - - ## - # Sets the cookie expiration to the time +t+. The expiration time may be - # a false value to disable expiration or a Time or HTTP format time string - # to set the expiration date. - - def expires=(t) - @expires = t && (t.is_a?(Time) ? t.httpdate : t.to_s) - end - - ## - # Retrieves the expiration time as a Time - - def expires - @expires && Time.parse(@expires) - end - - ## - # The cookie string suitable for use in an HTTP header - - def to_s - ret = "" - ret << @name << "=" << @value - ret << "; " << "Version=" << @version.to_s if @version > 0 - ret << "; " << "Domain=" << @domain if @domain - ret << "; " << "Expires=" << @expires if @expires - ret << "; " << "Max-Age=" << @max_age.to_s if @max_age - ret << "; " << "Comment=" << @comment if @comment - ret << "; " << "Path=" << @path if @path - ret << "; " << "Secure" if @secure - ret - end - - ## - # Parses a Cookie field sent from the user-agent. Returns an array of - # cookies. - - def self.parse(str) - if str - ret = [] - cookie = nil - ver = 0 - str.split(/;\s+/).each{|x| - key, val = x.split(/=/,2) - val = val ? HTTPUtils::dequote(val) : "" - case key - when "$Version"; ver = val.to_i - when "$Path"; cookie.path = val - when "$Domain"; cookie.domain = val - when "$Port"; cookie.port = val - else - ret << cookie if cookie - cookie = self.new(key, val) - cookie.version = ver - end - } - ret << cookie if cookie - ret - end - end - - ## - # Parses the cookie in +str+ - - def self.parse_set_cookie(str) - cookie_elem = str.split(/;/) - first_elem = cookie_elem.shift - first_elem.strip! - key, value = first_elem.split(/=/, 2) - cookie = new(key, HTTPUtils.dequote(value)) - cookie_elem.each{|pair| - pair.strip! - key, value = pair.split(/=/, 2) - if value - value = HTTPUtils.dequote(value.strip) - end - case key.downcase - when "domain" then cookie.domain = value - when "path" then cookie.path = value - when "expires" then cookie.expires = value - when "max-age" then cookie.max_age = Integer(value) - when "comment" then cookie.comment = value - when "version" then cookie.version = Integer(value) - when "secure" then cookie.secure = true - end - } - return cookie - end - - ## - # Parses the cookies in +str+ - - def self.parse_set_cookies(str) - return str.split(/,(?=[^;,]*=)|,$/).collect{|c| - parse_set_cookie(c) - } - end - end -end diff --git a/tool/lib/webrick/htmlutils.rb b/tool/lib/webrick/htmlutils.rb deleted file mode 100644 index ed9f4ac0d3..0000000000 --- a/tool/lib/webrick/htmlutils.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: false -#-- -# htmlutils.rb -- HTMLUtils Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htmlutils.rb,v 1.7 2002/09/21 12:23:35 gotoyuzo Exp $ - -module WEBrick - module HTMLUtils - - ## - # Escapes &, ", > and < in +string+ - - def escape(string) - return "" unless string - str = string.b - str.gsub!(/&/n, '&') - str.gsub!(/\"/n, '"') - str.gsub!(/>/n, '>') - str.gsub!(/</n, '<') - str.force_encoding(string.encoding) - end - module_function :escape - - end -end diff --git a/tool/lib/webrick/httpauth.rb b/tool/lib/webrick/httpauth.rb deleted file mode 100644 index f8bf09a6f1..0000000000 --- a/tool/lib/webrick/httpauth.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: false -# -# httpauth.rb -- HTTP access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpauth.rb,v 1.14 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'httpauth/basicauth' -require_relative 'httpauth/digestauth' -require_relative 'httpauth/htpasswd' -require_relative 'httpauth/htdigest' -require_relative 'httpauth/htgroup' - -module WEBrick - - ## - # HTTPAuth provides both basic and digest authentication. - # - # To enable authentication for requests in WEBrick you will need a user - # database and an authenticator. To start, here's an Htpasswd database for - # use with a DigestAuth authenticator: - # - # config = { :Realm => 'DigestAuth example realm' } - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' - # htpasswd.auth_type = WEBrick::HTTPAuth::DigestAuth - # htpasswd.set_passwd config[:Realm], 'username', 'password' - # htpasswd.flush - # - # The +:Realm+ is used to provide different access to different groups - # across several resources on a server. Typically you'll need only one - # realm for a server. - # - # This database can be used to create an authenticator: - # - # config[:UserDB] = htpasswd - # - # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config - # - # To authenticate a request call #authenticate with a request and response - # object in a servlet: - # - # def do_GET req, res - # @authenticator.authenticate req, res - # end - # - # For digest authentication the authenticator must not be created every - # request, it must be passed in as an option via WEBrick::HTTPServer#mount. - - module HTTPAuth - module_function - - def _basic_auth(req, res, realm, req_field, res_field, err_type, - block) # :nodoc: - user = pass = nil - if /^Basic\s+(.*)/o =~ req[req_field] - userpass = $1 - user, pass = userpass.unpack("m*")[0].split(":", 2) - end - if block.call(user, pass) - req.user = user - return - end - res[res_field] = "Basic realm=\"#{realm}\"" - raise err_type - end - - ## - # Simple wrapper for providing basic authentication for a request. When - # called with a request +req+, response +res+, authentication +realm+ and - # +block+ the block will be called with a +username+ and +password+. If - # the block returns true the request is allowed to continue, otherwise an - # HTTPStatus::Unauthorized error is raised. - - def basic_auth(req, res, realm, &block) # :yield: username, password - _basic_auth(req, res, realm, "Authorization", "WWW-Authenticate", - HTTPStatus::Unauthorized, block) - end - - ## - # Simple wrapper for providing basic authentication for a proxied request. - # When called with a request +req+, response +res+, authentication +realm+ - # and +block+ the block will be called with a +username+ and +password+. - # If the block returns true the request is allowed to continue, otherwise - # an HTTPStatus::ProxyAuthenticationRequired error is raised. - - def proxy_basic_auth(req, res, realm, &block) # :yield: username, password - _basic_auth(req, res, realm, "Proxy-Authorization", "Proxy-Authenticate", - HTTPStatus::ProxyAuthenticationRequired, block) - end - end -end diff --git a/tool/lib/webrick/httpauth/authenticator.rb b/tool/lib/webrick/httpauth/authenticator.rb deleted file mode 100644 index 8f0eaa3aca..0000000000 --- a/tool/lib/webrick/httpauth/authenticator.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: false -#-- -# httpauth/authenticator.rb -- Authenticator mix-in module. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: authenticator.rb,v 1.3 2003/02/20 07:15:47 gotoyuzo Exp $ - -module WEBrick - module HTTPAuth - - ## - # Module providing generic support for both Digest and Basic - # authentication schemes. - - module Authenticator - - RequestField = "Authorization" # :nodoc: - ResponseField = "WWW-Authenticate" # :nodoc: - ResponseInfoField = "Authentication-Info" # :nodoc: - AuthException = HTTPStatus::Unauthorized # :nodoc: - - ## - # Method of authentication, must be overridden by the including class - - AuthScheme = nil - - ## - # The realm this authenticator covers - - attr_reader :realm - - ## - # The user database for this authenticator - - attr_reader :userdb - - ## - # The logger for this authenticator - - attr_reader :logger - - private - - # :stopdoc: - - ## - # Initializes the authenticator from +config+ - - def check_init(config) - [:UserDB, :Realm].each{|sym| - unless config[sym] - raise ArgumentError, "Argument #{sym.inspect} missing." - end - } - @realm = config[:Realm] - @userdb = config[:UserDB] - @logger = config[:Logger] || Log::new($stderr) - @reload_db = config[:AutoReloadUserDB] - @request_field = self::class::RequestField - @response_field = self::class::ResponseField - @resp_info_field = self::class::ResponseInfoField - @auth_exception = self::class::AuthException - @auth_scheme = self::class::AuthScheme - end - - ## - # Ensures +req+ has credentials that can be authenticated. - - def check_scheme(req) - unless credentials = req[@request_field] - error("no credentials in the request.") - return nil - end - unless match = /^#{@auth_scheme}\s+/i.match(credentials) - error("invalid scheme in %s.", credentials) - info("%s: %s", @request_field, credentials) if $DEBUG - return nil - end - return match.post_match - end - - def log(meth, fmt, *args) - msg = format("%s %s: ", @auth_scheme, @realm) - msg << fmt % args - @logger.__send__(meth, msg) - end - - def error(fmt, *args) - if @logger.error? - log(:error, fmt, *args) - end - end - - def info(fmt, *args) - if @logger.info? - log(:info, fmt, *args) - end - end - - # :startdoc: - end - - ## - # Module providing generic support for both Digest and Basic - # authentication schemes for proxies. - - module ProxyAuthenticator - RequestField = "Proxy-Authorization" # :nodoc: - ResponseField = "Proxy-Authenticate" # :nodoc: - InfoField = "Proxy-Authentication-Info" # :nodoc: - AuthException = HTTPStatus::ProxyAuthenticationRequired # :nodoc: - end - end -end diff --git a/tool/lib/webrick/httpauth/basicauth.rb b/tool/lib/webrick/httpauth/basicauth.rb deleted file mode 100644 index 7d0a9cfc8f..0000000000 --- a/tool/lib/webrick/httpauth/basicauth.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/basicauth.rb -- HTTP basic access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: basicauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ - -require_relative '../config' -require_relative '../httpstatus' -require_relative 'authenticator' - -module WEBrick - module HTTPAuth - - ## - # Basic Authentication for WEBrick - # - # Use this class to add basic authentication to a WEBrick servlet. - # - # Here is an example of how to set up a BasicAuth: - # - # config = { :Realm => 'BasicAuth example realm' } - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file', password_hash: :bcrypt - # htpasswd.set_passwd config[:Realm], 'username', 'password' - # htpasswd.flush - # - # config[:UserDB] = htpasswd - # - # basic_auth = WEBrick::HTTPAuth::BasicAuth.new config - - class BasicAuth - include Authenticator - - AuthScheme = "Basic" # :nodoc: - - ## - # Used by UserDB to create a basic password entry - - def self.make_passwd(realm, user, pass) - pass ||= "" - pass.crypt(Utils::random_string(2)) - end - - attr_reader :realm, :userdb, :logger - - ## - # Creates a new BasicAuth instance. - # - # See WEBrick::Config::BasicAuth for default configuration entries - # - # You must supply the following configuration entries: - # - # :Realm:: The name of the realm being protected. - # :UserDB:: A database of usernames and passwords. - # A WEBrick::HTTPAuth::Htpasswd instance should be used. - - def initialize(config, default=Config::BasicAuth) - check_init(config) - @config = default.dup.update(config) - end - - ## - # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if - # the authentication was not correct. - - def authenticate(req, res) - unless basic_credentials = check_scheme(req) - challenge(req, res) - end - userid, password = basic_credentials.unpack("m*")[0].split(":", 2) - password ||= "" - if userid.empty? - error("user id was not given.") - challenge(req, res) - end - unless encpass = @userdb.get_passwd(@realm, userid, @reload_db) - error("%s: the user is not allowed.", userid) - challenge(req, res) - end - - case encpass - when /\A\$2[aby]\$/ - password_matches = BCrypt::Password.new(encpass.sub(/\A\$2[aby]\$/, '$2a$')) == password - else - password_matches = password.crypt(encpass) == encpass - end - - unless password_matches - error("%s: password unmatch.", userid) - challenge(req, res) - end - info("%s: authentication succeeded.", userid) - req.user = userid - end - - ## - # Returns a challenge response which asks for authentication information - - def challenge(req, res) - res[@response_field] = "#{@auth_scheme} realm=\"#{@realm}\"" - raise @auth_exception - end - end - - ## - # Basic authentication for proxy servers. See BasicAuth for details. - - class ProxyBasicAuth < BasicAuth - include ProxyAuthenticator - end - end -end diff --git a/tool/lib/webrick/httpauth/digestauth.rb b/tool/lib/webrick/httpauth/digestauth.rb deleted file mode 100644 index 3cf12899d2..0000000000 --- a/tool/lib/webrick/httpauth/digestauth.rb +++ /dev/null @@ -1,395 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/digestauth.rb -- HTTP digest access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. -# Copyright (c) 2003 H.M. -# -# The original implementation is provided by H.M. -# URL: http://rwiki.jin.gr.jp/cgi-bin/rw-cgi.rb?cmd=view;name= -# %C7%A7%BE%DA%B5%A1%C7%BD%A4%F2%B2%FE%C2%A4%A4%B7%A4%C6%A4%DF%A4%EB -# -# $IPR: digestauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ - -require_relative '../config' -require_relative '../httpstatus' -require_relative 'authenticator' -require 'digest/md5' -require 'digest/sha1' - -module WEBrick - module HTTPAuth - - ## - # RFC 2617 Digest Access Authentication for WEBrick - # - # Use this class to add digest authentication to a WEBrick servlet. - # - # Here is an example of how to set up DigestAuth: - # - # config = { :Realm => 'DigestAuth example realm' } - # - # htdigest = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' - # htdigest.set_passwd config[:Realm], 'username', 'password' - # htdigest.flush - # - # config[:UserDB] = htdigest - # - # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config - # - # When using this as with a servlet be sure not to create a new DigestAuth - # object in the servlet's #initialize. By default WEBrick creates a new - # servlet instance for every request and the DigestAuth object must be - # used across requests. - - class DigestAuth - include Authenticator - - AuthScheme = "Digest" # :nodoc: - - ## - # Struct containing the opaque portion of the digest authentication - - OpaqueInfo = Struct.new(:time, :nonce, :nc) # :nodoc: - - ## - # Digest authentication algorithm - - attr_reader :algorithm - - ## - # Quality of protection. RFC 2617 defines "auth" and "auth-int" - - attr_reader :qop - - ## - # Used by UserDB to create a digest password entry - - def self.make_passwd(realm, user, pass) - pass ||= "" - Digest::MD5::hexdigest([user, realm, pass].join(":")) - end - - ## - # Creates a new DigestAuth instance. Be sure to use the same DigestAuth - # instance for multiple requests as it saves state between requests in - # order to perform authentication. - # - # See WEBrick::Config::DigestAuth for default configuration entries - # - # You must supply the following configuration entries: - # - # :Realm:: The name of the realm being protected. - # :UserDB:: A database of usernames and passwords. - # A WEBrick::HTTPAuth::Htdigest instance should be used. - - def initialize(config, default=Config::DigestAuth) - check_init(config) - @config = default.dup.update(config) - @algorithm = @config[:Algorithm] - @domain = @config[:Domain] - @qop = @config[:Qop] - @use_opaque = @config[:UseOpaque] - @use_next_nonce = @config[:UseNextNonce] - @check_nc = @config[:CheckNc] - @use_auth_info_header = @config[:UseAuthenticationInfoHeader] - @nonce_expire_period = @config[:NonceExpirePeriod] - @nonce_expire_delta = @config[:NonceExpireDelta] - @internet_explorer_hack = @config[:InternetExplorerHack] - - case @algorithm - when 'MD5','MD5-sess' - @h = Digest::MD5 - when 'SHA1','SHA1-sess' # it is a bonus feature :-) - @h = Digest::SHA1 - else - msg = format('Algorithm "%s" is not supported.', @algorithm) - raise ArgumentError.new(msg) - end - - @instance_key = hexdigest(self.__id__, Time.now.to_i, Process.pid) - @opaques = {} - @last_nonce_expire = Time.now - @mutex = Thread::Mutex.new - end - - ## - # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if - # the authentication was not correct. - - def authenticate(req, res) - unless result = @mutex.synchronize{ _authenticate(req, res) } - challenge(req, res) - end - if result == :nonce_is_stale - challenge(req, res, true) - end - return true - end - - ## - # Returns a challenge response which asks for authentication information - - def challenge(req, res, stale=false) - nonce = generate_next_nonce(req) - if @use_opaque - opaque = generate_opaque(req) - @opaques[opaque].nonce = nonce - end - - param = Hash.new - param["realm"] = HTTPUtils::quote(@realm) - param["domain"] = HTTPUtils::quote(@domain.to_a.join(" ")) if @domain - param["nonce"] = HTTPUtils::quote(nonce) - param["opaque"] = HTTPUtils::quote(opaque) if opaque - param["stale"] = stale.to_s - param["algorithm"] = @algorithm - param["qop"] = HTTPUtils::quote(@qop.to_a.join(",")) if @qop - - res[@response_field] = - "#{@auth_scheme} " + param.map{|k,v| "#{k}=#{v}" }.join(", ") - info("%s: %s", @response_field, res[@response_field]) if $DEBUG - raise @auth_exception - end - - private - - # :stopdoc: - - MustParams = ['username','realm','nonce','uri','response'] - MustParamsAuth = ['cnonce','nc'] - - def _authenticate(req, res) - unless digest_credentials = check_scheme(req) - return false - end - - auth_req = split_param_value(digest_credentials) - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - req_params = MustParams + MustParamsAuth - else - req_params = MustParams - end - req_params.each{|key| - unless auth_req.has_key?(key) - error('%s: parameter missing. "%s"', auth_req['username'], key) - raise HTTPStatus::BadRequest - end - } - - if !check_uri(req, auth_req) - raise HTTPStatus::BadRequest - end - - if auth_req['realm'] != @realm - error('%s: realm unmatch. "%s" for "%s"', - auth_req['username'], auth_req['realm'], @realm) - return false - end - - auth_req['algorithm'] ||= 'MD5' - if auth_req['algorithm'].upcase != @algorithm.upcase - error('%s: algorithm unmatch. "%s" for "%s"', - auth_req['username'], auth_req['algorithm'], @algorithm) - return false - end - - if (@qop.nil? && auth_req.has_key?('qop')) || - (@qop && (! @qop.member?(auth_req['qop']))) - error('%s: the qop is not allowed. "%s"', - auth_req['username'], auth_req['qop']) - return false - end - - password = @userdb.get_passwd(@realm, auth_req['username'], @reload_db) - unless password - error('%s: the user is not allowed.', auth_req['username']) - return false - end - - nonce_is_invalid = false - if @use_opaque - info("@opaque = %s", @opaque.inspect) if $DEBUG - if !(opaque = auth_req['opaque']) - error('%s: opaque is not given.', auth_req['username']) - nonce_is_invalid = true - elsif !(opaque_struct = @opaques[opaque]) - error('%s: invalid opaque is given.', auth_req['username']) - nonce_is_invalid = true - elsif !check_opaque(opaque_struct, req, auth_req) - @opaques.delete(auth_req['opaque']) - nonce_is_invalid = true - end - elsif !check_nonce(req, auth_req) - nonce_is_invalid = true - end - - if /-sess$/i =~ auth_req['algorithm'] - ha1 = hexdigest(password, auth_req['nonce'], auth_req['cnonce']) - else - ha1 = password - end - - if auth_req['qop'] == "auth" || auth_req['qop'] == nil - ha2 = hexdigest(req.request_method, auth_req['uri']) - ha2_res = hexdigest("", auth_req['uri']) - elsif auth_req['qop'] == "auth-int" - body_digest = @h.new - req.body { |chunk| body_digest.update(chunk) } - body_digest = body_digest.hexdigest - ha2 = hexdigest(req.request_method, auth_req['uri'], body_digest) - ha2_res = hexdigest("", auth_req['uri'], body_digest) - end - - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - param2 = ['nonce', 'nc', 'cnonce', 'qop'].map{|key| - auth_req[key] - }.join(':') - digest = hexdigest(ha1, param2, ha2) - digest_res = hexdigest(ha1, param2, ha2_res) - else - digest = hexdigest(ha1, auth_req['nonce'], ha2) - digest_res = hexdigest(ha1, auth_req['nonce'], ha2_res) - end - - if digest != auth_req['response'] - error("%s: digest unmatch.", auth_req['username']) - return false - elsif nonce_is_invalid - error('%s: digest is valid, but nonce is not valid.', - auth_req['username']) - return :nonce_is_stale - elsif @use_auth_info_header - auth_info = { - 'nextnonce' => generate_next_nonce(req), - 'rspauth' => digest_res - } - if @use_opaque - opaque_struct.time = req.request_time - opaque_struct.nonce = auth_info['nextnonce'] - opaque_struct.nc = "%08x" % (auth_req['nc'].hex + 1) - end - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - ['qop','cnonce','nc'].each{|key| - auth_info[key] = auth_req[key] - } - end - res[@resp_info_field] = auth_info.keys.map{|key| - if key == 'nc' - key + '=' + auth_info[key] - else - key + "=" + HTTPUtils::quote(auth_info[key]) - end - }.join(', ') - end - info('%s: authentication succeeded.', auth_req['username']) - req.user = auth_req['username'] - return true - end - - def split_param_value(string) - ret = {} - string.scan(/\G\s*([\w\-.*%!]+)=\s*(?:\"((?>\\.|[^\"])*)\"|([^,\"]*))\s*,?/) do - ret[$1] = $3 || $2.gsub(/\\(.)/, "\\1") - end - ret - end - - def generate_next_nonce(req) - now = "%012d" % req.request_time.to_i - pk = hexdigest(now, @instance_key)[0,32] - nonce = [now + ":" + pk].pack("m0") # it has 60 length of chars. - nonce - end - - def check_nonce(req, auth_req) - username = auth_req['username'] - nonce = auth_req['nonce'] - - pub_time, pk = nonce.unpack("m*")[0].split(":", 2) - if (!pub_time || !pk) - error("%s: empty nonce is given", username) - return false - elsif (hexdigest(pub_time, @instance_key)[0,32] != pk) - error("%s: invalid private-key: %s for %s", - username, hexdigest(pub_time, @instance_key)[0,32], pk) - return false - end - - diff_time = req.request_time.to_i - pub_time.to_i - if (diff_time < 0) - error("%s: difference of time-stamp is negative.", username) - return false - elsif diff_time > @nonce_expire_period - error("%s: nonce is expired.", username) - return false - end - - return true - end - - def generate_opaque(req) - @mutex.synchronize{ - now = req.request_time - if now - @last_nonce_expire > @nonce_expire_delta - @opaques.delete_if{|key,val| - (now - val.time) > @nonce_expire_period - } - @last_nonce_expire = now - end - begin - opaque = Utils::random_string(16) - end while @opaques[opaque] - @opaques[opaque] = OpaqueInfo.new(now, nil, '00000001') - opaque - } - end - - def check_opaque(opaque_struct, req, auth_req) - if (@use_next_nonce && auth_req['nonce'] != opaque_struct.nonce) - error('%s: nonce unmatched. "%s" for "%s"', - auth_req['username'], auth_req['nonce'], opaque_struct.nonce) - return false - elsif !check_nonce(req, auth_req) - return false - end - if (@check_nc && auth_req['nc'] != opaque_struct.nc) - error('%s: nc unmatched."%s" for "%s"', - auth_req['username'], auth_req['nc'], opaque_struct.nc) - return false - end - true - end - - def check_uri(req, auth_req) - uri = auth_req['uri'] - if uri != req.request_uri.to_s && uri != req.unparsed_uri && - (@internet_explorer_hack && uri != req.path) - error('%s: uri unmatch. "%s" for "%s"', auth_req['username'], - auth_req['uri'], req.request_uri.to_s) - return false - end - true - end - - def hexdigest(*args) - @h.hexdigest(args.join(":")) - end - - # :startdoc: - end - - ## - # Digest authentication for proxy servers. See DigestAuth for details. - - class ProxyDigestAuth < DigestAuth - include ProxyAuthenticator - - private - def check_uri(req, auth_req) # :nodoc: - return true - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htdigest.rb b/tool/lib/webrick/httpauth/htdigest.rb deleted file mode 100644 index 93b18e2c75..0000000000 --- a/tool/lib/webrick/httpauth/htdigest.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htdigest.rb -- Apache compatible htdigest file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htdigest.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ - -require_relative 'userdb' -require_relative 'digestauth' -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htdigest accesses apache-compatible digest password files. Passwords are - # matched to a realm where they are valid. For security, the path for a - # digest password database should be stored outside of the paths available - # to the HTTP server. - # - # Htdigest is intended for use with WEBrick::HTTPAuth::DigestAuth and - # stores passwords using cryptographic hashes. - # - # htpasswd = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' - # htpasswd.set_passwd 'my realm', 'username', 'password' - # htpasswd.flush - - class Htdigest - include UserDB - - ## - # Open a digest password database at +path+ - - def initialize(path) - @path = path - @mtime = Time.at(0) - @digest = Hash.new - @mutex = Thread::Mutex::new - @auth_type = DigestAuth - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reloads passwords from the database - - def reload - mtime = File::mtime(@path) - if mtime > @mtime - @digest.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - user, realm, pass = line.split(/:/, 3) - unless @digest[realm] - @digest[realm] = Hash.new - end - @digest[realm][user] = pass - end - } - @mtime = mtime - end - end - - ## - # Flush the password database. If +output+ is given the database will - # be written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htpasswd", File::dirname(output)) - renamed = false - begin - each{|item| tmp.puts(item.join(":")) } - tmp.close - File::rename(tmp.path, output) - renamed = true - ensure - tmp.close - File.unlink(tmp.path) if !renamed - end - end - - ## - # Retrieves a password from the database for +user+ in +realm+. If - # +reload_db+ is true the database will be reloaded first. - - def get_passwd(realm, user, reload_db) - reload() if reload_db - if hash = @digest[realm] - hash[user] - end - end - - ## - # Sets a password in the database for +user+ in +realm+ to +pass+. - - def set_passwd(realm, user, pass) - @mutex.synchronize{ - unless @digest[realm] - @digest[realm] = Hash.new - end - @digest[realm][user] = make_passwd(realm, user, pass) - } - end - - ## - # Removes a password from the database for +user+ in +realm+. - - def delete_passwd(realm, user) - if hash = @digest[realm] - hash.delete(user) - end - end - - ## - # Iterate passwords in the database. - - def each # :yields: [user, realm, password_hash] - @digest.keys.sort.each{|realm| - hash = @digest[realm] - hash.keys.sort.each{|user| - yield([user, realm, hash[user]]) - } - } - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htgroup.rb b/tool/lib/webrick/httpauth/htgroup.rb deleted file mode 100644 index e06c441b18..0000000000 --- a/tool/lib/webrick/httpauth/htgroup.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htgroup.rb -- Apache compatible htgroup file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htgroup.rb,v 1.1 2003/02/16 22:22:56 gotoyuzo Exp $ - -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htgroup accesses apache-compatible group files. Htgroup can be used to - # provide group-based authentication for users. Currently Htgroup is not - # directly integrated with any authenticators in WEBrick. For security, - # the path for a digest password database should be stored outside of the - # paths available to the HTTP server. - # - # Example: - # - # htgroup = WEBrick::HTTPAuth::Htgroup.new 'my_group_file' - # htgroup.add 'superheroes', %w[spiderman batman] - # - # htgroup.members('superheroes').include? 'magneto' # => false - - class Htgroup - - ## - # Open a group database at +path+ - - def initialize(path) - @path = path - @mtime = Time.at(0) - @group = Hash.new - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reload groups from the database - - def reload - if (mtime = File::mtime(@path)) > @mtime - @group.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - group, members = line.split(/:\s*/) - @group[group] = members.split(/\s+/) - end - } - @mtime = mtime - end - end - - ## - # Flush the group database. If +output+ is given the database will be - # written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htgroup", File::dirname(output)) - begin - @group.keys.sort.each{|group| - tmp.puts(format("%s: %s", group, self.members(group).join(" "))) - } - ensure - tmp.close - if $! - File.unlink(tmp.path) - else - return File.rename(tmp.path, output) - end - end - end - - ## - # Retrieve the list of members from +group+ - - def members(group) - reload - @group[group] || [] - end - - ## - # Add an Array of +members+ to +group+ - - def add(group, members) - @group[group] = members(group) | members - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htpasswd.rb b/tool/lib/webrick/httpauth/htpasswd.rb deleted file mode 100644 index abca30532e..0000000000 --- a/tool/lib/webrick/httpauth/htpasswd.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htpasswd -- Apache compatible htpasswd file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htpasswd.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ - -require_relative 'userdb' -require_relative 'basicauth' -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htpasswd accesses apache-compatible password files. Passwords are - # matched to a realm where they are valid. For security, the path for a - # password database should be stored outside of the paths available to the - # HTTP server. - # - # Htpasswd is intended for use with WEBrick::HTTPAuth::BasicAuth. - # - # To create an Htpasswd database with a single user: - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' - # htpasswd.set_passwd 'my realm', 'username', 'password' - # htpasswd.flush - - class Htpasswd - include UserDB - - ## - # Open a password database at +path+ - - def initialize(path, password_hash: nil) - @path = path - @mtime = Time.at(0) - @passwd = Hash.new - @auth_type = BasicAuth - @password_hash = password_hash - - case @password_hash - when nil - # begin - # require "string/crypt" - # rescue LoadError - # warn("Unable to load string/crypt, proceeding with deprecated use of String#crypt, consider using password_hash: :bcrypt") - # end - @password_hash = :crypt - when :crypt - # require "string/crypt" - when :bcrypt - require "bcrypt" - else - raise ArgumentError, "only :crypt and :bcrypt are supported for password_hash keyword argument" - end - - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reload passwords from the database - - def reload - mtime = File::mtime(@path) - if mtime > @mtime - @passwd.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - case line - when %r!\A[^:]+:[a-zA-Z0-9./]{13}\z! - if @password_hash == :bcrypt - raise StandardError, ".htpasswd file contains crypt password, only bcrypt passwords supported" - end - user, pass = line.split(":") - when %r!\A[^:]+:\$2[aby]\$\d{2}\$.{53}\z! - if @password_hash == :crypt - raise StandardError, ".htpasswd file contains bcrypt password, only crypt passwords supported" - end - user, pass = line.split(":") - when /:\$/, /:{SHA}/ - raise NotImplementedError, - 'MD5, SHA1 .htpasswd file not supported' - else - raise StandardError, 'bad .htpasswd file' - end - @passwd[user] = pass - end - } - @mtime = mtime - end - end - - ## - # Flush the password database. If +output+ is given the database will - # be written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htpasswd", File::dirname(output)) - renamed = false - begin - each{|item| tmp.puts(item.join(":")) } - tmp.close - File::rename(tmp.path, output) - renamed = true - ensure - tmp.close - File.unlink(tmp.path) if !renamed - end - end - - ## - # Retrieves a password from the database for +user+ in +realm+. If - # +reload_db+ is true the database will be reloaded first. - - def get_passwd(realm, user, reload_db) - reload() if reload_db - @passwd[user] - end - - ## - # Sets a password in the database for +user+ in +realm+ to +pass+. - - def set_passwd(realm, user, pass) - if @password_hash == :bcrypt - # Cost of 5 to match Apache default, and because the - # bcrypt default of 10 will introduce significant delays - # for every request. - @passwd[user] = BCrypt::Password.create(pass, :cost=>5) - else - @passwd[user] = make_passwd(realm, user, pass) - end - end - - ## - # Removes a password from the database for +user+ in +realm+. - - def delete_passwd(realm, user) - @passwd.delete(user) - end - - ## - # Iterate passwords in the database. - - def each # :yields: [user, password] - @passwd.keys.sort.each{|user| - yield([user, @passwd[user]]) - } - end - end - end -end diff --git a/tool/lib/webrick/httpauth/userdb.rb b/tool/lib/webrick/httpauth/userdb.rb deleted file mode 100644 index 7a17715cdf..0000000000 --- a/tool/lib/webrick/httpauth/userdb.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: false -#-- -# httpauth/userdb.rb -- UserDB mix-in module. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: userdb.rb,v 1.2 2003/02/20 07:15:48 gotoyuzo Exp $ - -module WEBrick - module HTTPAuth - - ## - # User database mixin for HTTPAuth. This mixin dispatches user record - # access to the underlying auth_type for this database. - - module UserDB - - ## - # The authentication type. - # - # WEBrick::HTTPAuth::BasicAuth or WEBrick::HTTPAuth::DigestAuth are - # built-in. - - attr_accessor :auth_type - - ## - # Creates an obscured password in +realm+ with +user+ and +password+ - # using the auth_type of this database. - - def make_passwd(realm, user, pass) - @auth_type::make_passwd(realm, user, pass) - end - - ## - # Sets a password in +realm+ with +user+ and +password+ for the - # auth_type of this database. - - def set_passwd(realm, user, pass) - self[user] = pass - end - - ## - # Retrieves a password in +realm+ for +user+ for the auth_type of this - # database. +reload_db+ is a dummy value. - - def get_passwd(realm, user, reload_db=false) - make_passwd(realm, user, self[user]) - end - end - end -end diff --git a/tool/lib/webrick/httpproxy.rb b/tool/lib/webrick/httpproxy.rb deleted file mode 100644 index 7607c3df88..0000000000 --- a/tool/lib/webrick/httpproxy.rb +++ /dev/null @@ -1,354 +0,0 @@ -# frozen_string_literal: false -# -# httpproxy.rb -- HTTPProxy Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 GOTO Kentaro -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpproxy.rb,v 1.18 2003/03/08 18:58:10 gotoyuzo Exp $ -# $kNotwork: straw.rb,v 1.3 2002/02/12 15:13:07 gotoken Exp $ - -require_relative "httpserver" -require "net/http" - -module WEBrick - - NullReader = Object.new # :nodoc: - class << NullReader # :nodoc: - def read(*args) - nil - end - alias gets read - end - - FakeProxyURI = Object.new # :nodoc: - class << FakeProxyURI # :nodoc: - def method_missing(meth, *args) - if %w(scheme host port path query userinfo).member?(meth.to_s) - return nil - end - super - end - end - - # :startdoc: - - ## - # An HTTP Proxy server which proxies GET, HEAD and POST requests. - # - # To create a simple proxy server: - # - # require 'webrick' - # require 'webrick/httpproxy' - # - # proxy = WEBrick::HTTPProxyServer.new Port: 8000 - # - # trap 'INT' do proxy.shutdown end - # trap 'TERM' do proxy.shutdown end - # - # proxy.start - # - # See ::new for proxy-specific configuration items. - # - # == Modifying proxied responses - # - # To modify content the proxy server returns use the +:ProxyContentHandler+ - # option: - # - # handler = proc do |req, res| - # if res['content-type'] == 'text/plain' then - # res.body << "\nThis content was proxied!\n" - # end - # end - # - # proxy = - # WEBrick::HTTPProxyServer.new Port: 8000, ProxyContentHandler: handler - - class HTTPProxyServer < HTTPServer - - ## - # Proxy server configurations. The proxy server handles the following - # configuration items in addition to those supported by HTTPServer: - # - # :ProxyAuthProc:: Called with a request and response to authorize a - # request - # :ProxyVia:: Appended to the via header - # :ProxyURI:: The proxy server's URI - # :ProxyContentHandler:: Called with a request and response and allows - # modification of the response - # :ProxyTimeout:: Sets the proxy timeouts to 30 seconds for open and 60 - # seconds for read operations - - def initialize(config={}, default=Config::HTTP) - super(config, default) - c = @config - @via = "#{c[:HTTPVersion]} #{c[:ServerName]}:#{c[:Port]}" - end - - # :stopdoc: - def service(req, res) - if req.request_method == "CONNECT" - do_CONNECT(req, res) - elsif req.unparsed_uri =~ %r!^http://! - proxy_service(req, res) - else - super(req, res) - end - end - - def proxy_auth(req, res) - if proc = @config[:ProxyAuthProc] - proc.call(req, res) - end - req.header.delete("proxy-authorization") - end - - def proxy_uri(req, res) - # should return upstream proxy server's URI - return @config[:ProxyURI] - end - - def proxy_service(req, res) - # Proxy Authentication - proxy_auth(req, res) - - begin - public_send("do_#{req.request_method}", req, res) - rescue NoMethodError - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - rescue => err - logger.debug("#{err.class}: #{err.message}") - raise HTTPStatus::ServiceUnavailable, err.message - end - - # Process contents - if handler = @config[:ProxyContentHandler] - handler.call(req, res) - end - end - - def do_CONNECT(req, res) - # Proxy Authentication - proxy_auth(req, res) - - ua = Thread.current[:WEBrickSocket] # User-Agent - raise HTTPStatus::InternalServerError, - "[BUG] cannot get socket" unless ua - - host, port = req.unparsed_uri.split(":", 2) - # Proxy authentication for upstream proxy server - if proxy = proxy_uri(req, res) - proxy_request_line = "CONNECT #{host}:#{port} HTTP/1.0" - if proxy.userinfo - credentials = "Basic " + [proxy.userinfo].pack("m0") - end - host, port = proxy.host, proxy.port - end - - begin - @logger.debug("CONNECT: upstream proxy is `#{host}:#{port}'.") - os = TCPSocket.new(host, port) # origin server - - if proxy - @logger.debug("CONNECT: sending a Request-Line") - os << proxy_request_line << CRLF - @logger.debug("CONNECT: > #{proxy_request_line}") - if credentials - @logger.debug("CONNECT: sending credentials") - os << "Proxy-Authorization: " << credentials << CRLF - end - os << CRLF - proxy_status_line = os.gets(LF) - @logger.debug("CONNECT: read Status-Line from the upstream server") - @logger.debug("CONNECT: < #{proxy_status_line}") - if %r{^HTTP/\d+\.\d+\s+200\s*} =~ proxy_status_line - while line = os.gets(LF) - break if /\A(#{CRLF}|#{LF})\z/om =~ line - end - else - raise HTTPStatus::BadGateway - end - end - @logger.debug("CONNECT #{host}:#{port}: succeeded") - res.status = HTTPStatus::RC_OK - rescue => ex - @logger.debug("CONNECT #{host}:#{port}: failed `#{ex.message}'") - res.set_error(ex) - raise HTTPStatus::EOFError - ensure - if handler = @config[:ProxyContentHandler] - handler.call(req, res) - end - res.send_response(ua) - access_log(@config, req, res) - - # Should clear request-line not to send the response twice. - # see: HTTPServer#run - req.parse(NullReader) rescue nil - end - - begin - while fds = IO::select([ua, os]) - if fds[0].member?(ua) - buf = ua.readpartial(1024); - @logger.debug("CONNECT: #{buf.bytesize} byte from User-Agent") - os.write(buf) - elsif fds[0].member?(os) - buf = os.readpartial(1024); - @logger.debug("CONNECT: #{buf.bytesize} byte from #{host}:#{port}") - ua.write(buf) - end - end - rescue - os.close - @logger.debug("CONNECT #{host}:#{port}: closed") - end - - raise HTTPStatus::EOFError - end - - def do_GET(req, res) - perform_proxy_request(req, res, Net::HTTP::Get) - end - - def do_HEAD(req, res) - perform_proxy_request(req, res, Net::HTTP::Head) - end - - def do_POST(req, res) - perform_proxy_request(req, res, Net::HTTP::Post, req.body_reader) - end - - def do_OPTIONS(req, res) - res['allow'] = "GET,HEAD,POST,OPTIONS,CONNECT" - end - - private - - # Some header fields should not be transferred. - HopByHop = %w( connection keep-alive proxy-authenticate upgrade - proxy-authorization te trailers transfer-encoding ) - ShouldNotTransfer = %w( set-cookie proxy-connection ) - def split_field(f) f ? f.split(/,\s+/).collect{|i| i.downcase } : [] end - - def choose_header(src, dst) - connections = split_field(src['connection']) - src.each{|key, value| - key = key.downcase - if HopByHop.member?(key) || # RFC2616: 13.5.1 - connections.member?(key) || # RFC2616: 14.10 - ShouldNotTransfer.member?(key) # pragmatics - @logger.debug("choose_header: `#{key}: #{value}'") - next - end - dst[key] = value - } - end - - # Net::HTTP is stupid about the multiple header fields. - # Here is workaround: - def set_cookie(src, dst) - if str = src['set-cookie'] - cookies = [] - str.split(/,\s*/).each{|token| - if /^[^=]+;/o =~ token - cookies[-1] << ", " << token - elsif /=/o =~ token - cookies << token - else - cookies[-1] << ", " << token - end - } - dst.cookies.replace(cookies) - end - end - - def set_via(h) - if @config[:ProxyVia] - if h['via'] - h['via'] << ", " << @via - else - h['via'] = @via - end - end - end - - def setup_proxy_header(req, res) - # Choose header fields to transfer - header = Hash.new - choose_header(req, header) - set_via(header) - return header - end - - def setup_upstream_proxy_authentication(req, res, header) - if upstream = proxy_uri(req, res) - if upstream.userinfo - header['proxy-authorization'] = - "Basic " + [upstream.userinfo].pack("m0") - end - return upstream - end - return FakeProxyURI - end - - def create_net_http(uri, upstream) - Net::HTTP.new(uri.host, uri.port, upstream.host, upstream.port) - end - - def perform_proxy_request(req, res, req_class, body_stream = nil) - uri = req.request_uri - path = uri.path.dup - path << "?" << uri.query if uri.query - header = setup_proxy_header(req, res) - upstream = setup_upstream_proxy_authentication(req, res, header) - - body_tmp = [] - http = create_net_http(uri, upstream) - req_fib = Fiber.new do - http.start do - if @config[:ProxyTimeout] - ################################## these issues are - http.open_timeout = 30 # secs # necessary (maybe because - http.read_timeout = 60 # secs # Ruby's bug, but why?) - ################################## - end - if body_stream && req['transfer-encoding'] =~ /\bchunked\b/i - header['Transfer-Encoding'] = 'chunked' - end - http_req = req_class.new(path, header) - http_req.body_stream = body_stream if body_stream - http.request(http_req) do |response| - # Persistent connection requirements are mysterious for me. - # So I will close the connection in every response. - res['proxy-connection'] = "close" - res['connection'] = "close" - - # stream Net::HTTP::HTTPResponse to WEBrick::HTTPResponse - res.status = response.code.to_i - res.chunked = response.chunked? - choose_header(response, res) - set_cookie(response, res) - set_via(res) - response.read_body do |buf| - body_tmp << buf - Fiber.yield # wait for res.body Proc#call - end - end # http.request - end - end - req_fib.resume # read HTTP response headers and first chunk of the body - res.body = ->(socket) do - while buf = body_tmp.shift - socket.write(buf) - buf.clear - req_fib.resume # continue response.read_body - end - end - end - # :stopdoc: - end -end diff --git a/tool/lib/webrick/httprequest.rb b/tool/lib/webrick/httprequest.rb deleted file mode 100644 index d34eac7ecf..0000000000 --- a/tool/lib/webrick/httprequest.rb +++ /dev/null @@ -1,636 +0,0 @@ -# frozen_string_literal: false -# -# httprequest.rb -- HTTPRequest Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httprequest.rb,v 1.64 2003/07/13 17:18:22 gotoyuzo Exp $ - -require 'fiber' -require 'uri' -require_relative 'httpversion' -require_relative 'httpstatus' -require_relative 'httputils' -require_relative 'cookie' - -module WEBrick - - ## - # An HTTP request. This is consumed by service and do_* methods in - # WEBrick servlets - - class HTTPRequest - - BODY_CONTAINABLE_METHODS = [ "POST", "PUT" ] # :nodoc: - - # :section: Request line - - ## - # The complete request line such as: - # - # GET / HTTP/1.1 - - attr_reader :request_line - - ## - # The request method, GET, POST, PUT, etc. - - attr_reader :request_method - - ## - # The unparsed URI of the request - - attr_reader :unparsed_uri - - ## - # The HTTP version of the request - - attr_reader :http_version - - # :section: Request-URI - - ## - # The parsed URI of the request - - attr_reader :request_uri - - ## - # The request path - - attr_reader :path - - ## - # The script name (CGI variable) - - attr_accessor :script_name - - ## - # The path info (CGI variable) - - attr_accessor :path_info - - ## - # The query from the URI of the request - - attr_accessor :query_string - - # :section: Header and entity body - - ## - # The raw header of the request - - attr_reader :raw_header - - ## - # The parsed header of the request - - attr_reader :header - - ## - # The parsed request cookies - - attr_reader :cookies - - ## - # The Accept header value - - attr_reader :accept - - ## - # The Accept-Charset header value - - attr_reader :accept_charset - - ## - # The Accept-Encoding header value - - attr_reader :accept_encoding - - ## - # The Accept-Language header value - - attr_reader :accept_language - - # :section: - - ## - # The remote user (CGI variable) - - attr_accessor :user - - ## - # The socket address of the server - - attr_reader :addr - - ## - # The socket address of the client - - attr_reader :peeraddr - - ## - # Hash of request attributes - - attr_reader :attributes - - ## - # Is this a keep-alive connection? - - attr_reader :keep_alive - - ## - # The local time this request was received - - attr_reader :request_time - - ## - # Creates a new HTTP request. WEBrick::Config::HTTP is the default - # configuration. - - def initialize(config) - @config = config - @buffer_size = @config[:InputBufferSize] - @logger = config[:Logger] - - @request_line = @request_method = - @unparsed_uri = @http_version = nil - - @request_uri = @host = @port = @path = nil - @script_name = @path_info = nil - @query_string = nil - @query = nil - @form_data = nil - - @raw_header = Array.new - @header = nil - @cookies = [] - @accept = [] - @accept_charset = [] - @accept_encoding = [] - @accept_language = [] - @body = "" - - @addr = @peeraddr = nil - @attributes = {} - @user = nil - @keep_alive = false - @request_time = nil - - @remaining_size = nil - @socket = nil - - @forwarded_proto = @forwarded_host = @forwarded_port = - @forwarded_server = @forwarded_for = nil - end - - ## - # Parses a request from +socket+. This is called internally by - # WEBrick::HTTPServer. - - def parse(socket=nil) - @socket = socket - begin - @peeraddr = socket.respond_to?(:peeraddr) ? socket.peeraddr : [] - @addr = socket.respond_to?(:addr) ? socket.addr : [] - rescue Errno::ENOTCONN - raise HTTPStatus::EOFError - end - - read_request_line(socket) - if @http_version.major > 0 - read_header(socket) - @header['cookie'].each{|cookie| - @cookies += Cookie::parse(cookie) - } - @accept = HTTPUtils.parse_qvalues(self['accept']) - @accept_charset = HTTPUtils.parse_qvalues(self['accept-charset']) - @accept_encoding = HTTPUtils.parse_qvalues(self['accept-encoding']) - @accept_language = HTTPUtils.parse_qvalues(self['accept-language']) - end - return if @request_method == "CONNECT" - return if @unparsed_uri == "*" - - begin - setup_forwarded_info - @request_uri = parse_uri(@unparsed_uri) - @path = HTTPUtils::unescape(@request_uri.path) - @path = HTTPUtils::normalize_path(@path) - @host = @request_uri.host - @port = @request_uri.port - @query_string = @request_uri.query - @script_name = "" - @path_info = @path.dup - rescue - raise HTTPStatus::BadRequest, "bad URI `#{@unparsed_uri}'." - end - - if /\Aclose\z/io =~ self["connection"] - @keep_alive = false - elsif /\Akeep-alive\z/io =~ self["connection"] - @keep_alive = true - elsif @http_version < "1.1" - @keep_alive = false - else - @keep_alive = true - end - end - - ## - # Generate HTTP/1.1 100 continue response if the client expects it, - # otherwise does nothing. - - def continue # :nodoc: - if self['expect'] == '100-continue' && @config[:HTTPVersion] >= "1.1" - @socket << "HTTP/#{@config[:HTTPVersion]} 100 continue#{CRLF}#{CRLF}" - @header.delete('expect') - end - end - - ## - # Returns the request body. - - def body(&block) # :yields: body_chunk - block ||= Proc.new{|chunk| @body << chunk } - read_body(@socket, block) - @body.empty? ? nil : @body - end - - ## - # Prepares the HTTPRequest object for use as the - # source for IO.copy_stream - - def body_reader - @body_tmp = [] - @body_rd = Fiber.new do - body do |buf| - @body_tmp << buf - Fiber.yield - end - end - @body_rd.resume # grab the first chunk and yield - self - end - - # for IO.copy_stream. - def readpartial(size, buf = ''.b) # :nodoc - res = @body_tmp.shift or raise EOFError, 'end of file reached' - if res.length > size - @body_tmp.unshift(res[size..-1]) - res = res[0..size - 1] - end - buf.replace(res) - res.clear - # get more chunks - check alive? because we can take a partial chunk - @body_rd.resume if @body_rd.alive? - buf - end - - ## - # Request query as a Hash - - def query - unless @query - parse_query() - end - @query - end - - ## - # The content-length header - - def content_length - return Integer(self['content-length']) - end - - ## - # The content-type header - - def content_type - return self['content-type'] - end - - ## - # Retrieves +header_name+ - - def [](header_name) - if @header - value = @header[header_name.downcase] - value.empty? ? nil : value.join(", ") - end - end - - ## - # Iterates over the request headers - - def each - if @header - @header.each{|k, v| - value = @header[k] - yield(k, value.empty? ? nil : value.join(", ")) - } - end - end - - ## - # The host this request is for - - def host - return @forwarded_host || @host - end - - ## - # The port this request is for - - def port - return @forwarded_port || @port - end - - ## - # The server name this request is for - - def server_name - return @forwarded_server || @config[:ServerName] - end - - ## - # The client's IP address - - def remote_ip - return self["client-ip"] || @forwarded_for || @peeraddr[3] - end - - ## - # Is this an SSL request? - - def ssl? - return @request_uri.scheme == "https" - end - - ## - # Should the connection this request was made on be kept alive? - - def keep_alive? - @keep_alive - end - - def to_s # :nodoc: - ret = @request_line.dup - @raw_header.each{|line| ret << line } - ret << CRLF - ret << body if body - ret - end - - ## - # Consumes any remaining body and updates keep-alive status - - def fixup() # :nodoc: - begin - body{|chunk| } # read remaining body - rescue HTTPStatus::Error => ex - @logger.error("HTTPRequest#fixup: #{ex.class} occurred.") - @keep_alive = false - rescue => ex - @logger.error(ex) - @keep_alive = false - end - end - - # This method provides the metavariables defined by the revision 3 - # of "The WWW Common Gateway Interface Version 1.1" - # To browse the current document of CGI Version 1.1, see below: - # http://tools.ietf.org/html/rfc3875 - - def meta_vars - meta = Hash.new - - cl = self["Content-Length"] - ct = self["Content-Type"] - meta["CONTENT_LENGTH"] = cl if cl.to_i > 0 - meta["CONTENT_TYPE"] = ct.dup if ct - meta["GATEWAY_INTERFACE"] = "CGI/1.1" - meta["PATH_INFO"] = @path_info ? @path_info.dup : "" - #meta["PATH_TRANSLATED"] = nil # no plan to be provided - meta["QUERY_STRING"] = @query_string ? @query_string.dup : "" - meta["REMOTE_ADDR"] = @peeraddr[3] - meta["REMOTE_HOST"] = @peeraddr[2] - #meta["REMOTE_IDENT"] = nil # no plan to be provided - meta["REMOTE_USER"] = @user - meta["REQUEST_METHOD"] = @request_method.dup - meta["REQUEST_URI"] = @request_uri.to_s - meta["SCRIPT_NAME"] = @script_name.dup - meta["SERVER_NAME"] = @host - meta["SERVER_PORT"] = @port.to_s - meta["SERVER_PROTOCOL"] = "HTTP/" + @config[:HTTPVersion].to_s - meta["SERVER_SOFTWARE"] = @config[:ServerSoftware].dup - - self.each{|key, val| - next if /^content-type$/i =~ key - next if /^content-length$/i =~ key - name = "HTTP_" + key - name.gsub!(/-/o, "_") - name.upcase! - meta[name] = val - } - - meta - end - - private - - # :stopdoc: - - MAX_URI_LENGTH = 2083 # :nodoc: - - # same as Mongrel, Thin and Puma - MAX_HEADER_LENGTH = (112 * 1024) # :nodoc: - - def read_request_line(socket) - @request_line = read_line(socket, MAX_URI_LENGTH) if socket - raise HTTPStatus::EOFError unless @request_line - - @request_bytes = @request_line.bytesize - if @request_bytes >= MAX_URI_LENGTH and @request_line[-1, 1] != LF - raise HTTPStatus::RequestURITooLarge - end - - @request_time = Time.now - if /^(\S+)\s+(\S++)(?:\s+HTTP\/(\d+\.\d+))?\r?\n/mo =~ @request_line - @request_method = $1 - @unparsed_uri = $2 - @http_version = HTTPVersion.new($3 ? $3 : "0.9") - else - rl = @request_line.sub(/\x0d?\x0a\z/o, '') - raise HTTPStatus::BadRequest, "bad Request-Line `#{rl}'." - end - end - - def read_header(socket) - if socket - while line = read_line(socket) - break if /\A(#{CRLF}|#{LF})\z/om =~ line - if (@request_bytes += line.bytesize) > MAX_HEADER_LENGTH - raise HTTPStatus::RequestEntityTooLarge, 'headers too large' - end - @raw_header << line - end - end - @header = HTTPUtils::parse_header(@raw_header.join) - end - - def parse_uri(str, scheme="http") - if @config[:Escape8bitURI] - str = HTTPUtils::escape8bit(str) - end - str.sub!(%r{\A/+}o, '/') - uri = URI::parse(str) - return uri if uri.absolute? - if @forwarded_host - host, port = @forwarded_host, @forwarded_port - elsif self["host"] - pattern = /\A(#{URI::REGEXP::PATTERN::HOST})(?::(\d+))?\z/n - host, port = *self['host'].scan(pattern)[0] - elsif @addr.size > 0 - host, port = @addr[2], @addr[1] - else - host, port = @config[:ServerName], @config[:Port] - end - uri.scheme = @forwarded_proto || scheme - uri.host = host - uri.port = port ? port.to_i : nil - return URI::parse(uri.to_s) - end - - def read_body(socket, block) - return unless socket - if tc = self['transfer-encoding'] - case tc - when /\Achunked\z/io then read_chunked(socket, block) - else raise HTTPStatus::NotImplemented, "Transfer-Encoding: #{tc}." - end - elsif self['content-length'] || @remaining_size - @remaining_size ||= self['content-length'].to_i - while @remaining_size > 0 - sz = [@buffer_size, @remaining_size].min - break unless buf = read_data(socket, sz) - @remaining_size -= buf.bytesize - block.call(buf) - end - if @remaining_size > 0 && @socket.eof? - raise HTTPStatus::BadRequest, "invalid body size." - end - elsif BODY_CONTAINABLE_METHODS.member?(@request_method) && !@socket.eof - raise HTTPStatus::LengthRequired - end - return @body - end - - def read_chunk_size(socket) - line = read_line(socket) - if /^([0-9a-fA-F]+)(?:;(\S+))?/ =~ line - chunk_size = $1.hex - chunk_ext = $2 - [ chunk_size, chunk_ext ] - else - raise HTTPStatus::BadRequest, "bad chunk `#{line}'." - end - end - - def read_chunked(socket, block) - chunk_size, = read_chunk_size(socket) - while chunk_size > 0 - begin - sz = [ chunk_size, @buffer_size ].min - data = read_data(socket, sz) # read chunk-data - if data.nil? || data.bytesize != sz - raise HTTPStatus::BadRequest, "bad chunk data size." - end - block.call(data) - end while (chunk_size -= sz) > 0 - - read_line(socket) # skip CRLF - chunk_size, = read_chunk_size(socket) - end - read_header(socket) # trailer + CRLF - @header.delete("transfer-encoding") - @remaining_size = 0 - end - - def _read_data(io, method, *arg) - begin - WEBrick::Utils.timeout(@config[:RequestTimeout]){ - return io.__send__(method, *arg) - } - rescue Errno::ECONNRESET - return nil - rescue Timeout::Error - raise HTTPStatus::RequestTimeout - end - end - - def read_line(io, size=4096) - _read_data(io, :gets, LF, size) - end - - def read_data(io, size) - _read_data(io, :read, size) - end - - def parse_query() - begin - if @request_method == "GET" || @request_method == "HEAD" - @query = HTTPUtils::parse_query(@query_string) - elsif self['content-type'] =~ /^application\/x-www-form-urlencoded/ - @query = HTTPUtils::parse_query(body) - elsif self['content-type'] =~ /^multipart\/form-data; boundary=(.+)/ - boundary = HTTPUtils::dequote($1) - @query = HTTPUtils::parse_form_data(body, boundary) - else - @query = Hash.new - end - rescue => ex - raise HTTPStatus::BadRequest, ex.message - end - end - - PrivateNetworkRegexp = / - ^unknown$| - ^((::ffff:)?127.0.0.1|::1)$| - ^(::ffff:)?(10|172\.(1[6-9]|2[0-9]|3[01])|192\.168)\. - /ixo - - # It's said that all X-Forwarded-* headers will contain more than one - # (comma-separated) value if the original request already contained one of - # these headers. Since we could use these values as Host header, we choose - # the initial(first) value. (apr_table_mergen() adds new value after the - # existing value with ", " prefix) - def setup_forwarded_info - if @forwarded_server = self["x-forwarded-server"] - @forwarded_server = @forwarded_server.split(",", 2).first - end - if @forwarded_proto = self["x-forwarded-proto"] - @forwarded_proto = @forwarded_proto.split(",", 2).first - end - if host_port = self["x-forwarded-host"] - host_port = host_port.split(",", 2).first - if host_port =~ /\A(\[[0-9a-fA-F:]+\])(?::(\d+))?\z/ - @forwarded_host = $1 - tmp = $2 - else - @forwarded_host, tmp = host_port.split(":", 2) - end - @forwarded_port = (tmp || (@forwarded_proto == "https" ? 443 : 80)).to_i - end - if addrs = self["x-forwarded-for"] - addrs = addrs.split(",").collect(&:strip) - addrs.reject!{|ip| PrivateNetworkRegexp =~ ip } - @forwarded_for = addrs.first - end - end - - # :startdoc: - end -end diff --git a/tool/lib/webrick/httpresponse.rb b/tool/lib/webrick/httpresponse.rb deleted file mode 100644 index ba4494ab74..0000000000 --- a/tool/lib/webrick/httpresponse.rb +++ /dev/null @@ -1,564 +0,0 @@ -# frozen_string_literal: false -# -# httpresponse.rb -- HTTPResponse Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpresponse.rb,v 1.45 2003/07/11 11:02:25 gotoyuzo Exp $ - -require 'time' -require 'uri' -require_relative 'httpversion' -require_relative 'htmlutils' -require_relative 'httputils' -require_relative 'httpstatus' - -module WEBrick - ## - # An HTTP response. This is filled in by the service or do_* methods of a - # WEBrick HTTP Servlet. - - class HTTPResponse - class InvalidHeader < StandardError - end - - ## - # HTTP Response version - - attr_reader :http_version - - ## - # Response status code (200) - - attr_reader :status - - ## - # Response header - - attr_reader :header - - ## - # Response cookies - - attr_reader :cookies - - ## - # Response reason phrase ("OK") - - attr_accessor :reason_phrase - - ## - # Body may be: - # * a String; - # * an IO-like object that responds to +#read+ and +#readpartial+; - # * a Proc-like object that responds to +#call+. - # - # In the latter case, either #chunked= should be set to +true+, - # or <code>header['content-length']</code> explicitly provided. - # Example: - # - # server.mount_proc '/' do |req, res| - # res.chunked = true - # # or - # # res.header['content-length'] = 10 - # res.body = proc { |out| out.write(Time.now.to_s) } - # end - - attr_accessor :body - - ## - # Request method for this response - - attr_accessor :request_method - - ## - # Request URI for this response - - attr_accessor :request_uri - - ## - # Request HTTP version for this response - - attr_accessor :request_http_version - - ## - # Filename of the static file in this response. Only used by the - # FileHandler servlet. - - attr_accessor :filename - - ## - # Is this a keep-alive response? - - attr_accessor :keep_alive - - ## - # Configuration for this response - - attr_reader :config - - ## - # Bytes sent in this response - - attr_reader :sent_size - - ## - # Creates a new HTTP response object. WEBrick::Config::HTTP is the - # default configuration. - - def initialize(config) - @config = config - @buffer_size = config[:OutputBufferSize] - @logger = config[:Logger] - @header = Hash.new - @status = HTTPStatus::RC_OK - @reason_phrase = nil - @http_version = HTTPVersion::convert(@config[:HTTPVersion]) - @body = '' - @keep_alive = true - @cookies = [] - @request_method = nil - @request_uri = nil - @request_http_version = @http_version # temporary - @chunked = false - @filename = nil - @sent_size = 0 - @bodytempfile = nil - end - - ## - # The response's HTTP status line - - def status_line - "HTTP/#@http_version #@status #@reason_phrase".rstrip << CRLF - end - - ## - # Sets the response's status to the +status+ code - - def status=(status) - @status = status - @reason_phrase = HTTPStatus::reason_phrase(status) - end - - ## - # Retrieves the response header +field+ - - def [](field) - @header[field.downcase] - end - - ## - # Sets the response header +field+ to +value+ - - def []=(field, value) - @chunked = value.to_s.downcase == 'chunked' if field.downcase == 'transfer-encoding' - @header[field.downcase] = value.to_s - end - - ## - # The content-length header - - def content_length - if len = self['content-length'] - return Integer(len) - end - end - - ## - # Sets the content-length header to +len+ - - def content_length=(len) - self['content-length'] = len.to_s - end - - ## - # The content-type header - - def content_type - self['content-type'] - end - - ## - # Sets the content-type header to +type+ - - def content_type=(type) - self['content-type'] = type - end - - ## - # Iterates over each header in the response - - def each - @header.each{|field, value| yield(field, value) } - end - - ## - # Will this response body be returned using chunked transfer-encoding? - - def chunked? - @chunked - end - - ## - # Enables chunked transfer encoding. - - def chunked=(val) - @chunked = val ? true : false - end - - ## - # Will this response's connection be kept alive? - - def keep_alive? - @keep_alive - end - - ## - # Sends the response on +socket+ - - def send_response(socket) # :nodoc: - begin - setup_header() - send_header(socket) - send_body(socket) - rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ENOTCONN => ex - @logger.debug(ex) - @keep_alive = false - rescue Exception => ex - @logger.error(ex) - @keep_alive = false - end - end - - ## - # Sets up the headers for sending - - def setup_header() # :nodoc: - @reason_phrase ||= HTTPStatus::reason_phrase(@status) - @header['server'] ||= @config[:ServerSoftware] - @header['date'] ||= Time.now.httpdate - - # HTTP/0.9 features - if @request_http_version < "1.0" - @http_version = HTTPVersion.new("0.9") - @keep_alive = false - end - - # HTTP/1.0 features - if @request_http_version < "1.1" - if chunked? - @chunked = false - ver = @request_http_version.to_s - msg = "chunked is set for an HTTP/#{ver} request. (ignored)" - @logger.warn(msg) - end - end - - # Determine the message length (RFC2616 -- 4.4 Message Length) - if @status == 304 || @status == 204 || HTTPStatus::info?(@status) - @header.delete('content-length') - @body = "" - elsif chunked? - @header["transfer-encoding"] = "chunked" - @header.delete('content-length') - elsif %r{^multipart/byteranges} =~ @header['content-type'] - @header.delete('content-length') - elsif @header['content-length'].nil? - if @body.respond_to? :readpartial - elsif @body.respond_to? :call - make_body_tempfile - else - @header['content-length'] = (@body ? @body.bytesize : 0).to_s - end - end - - # Keep-Alive connection. - if @header['connection'] == "close" - @keep_alive = false - elsif keep_alive? - if chunked? || @header['content-length'] || @status == 304 || @status == 204 || HTTPStatus.info?(@status) - @header['connection'] = "Keep-Alive" - else - msg = "Could not determine content-length of response body. Set content-length of the response or set Response#chunked = true" - @logger.warn(msg) - @header['connection'] = "close" - @keep_alive = false - end - else - @header['connection'] = "close" - end - - # Location is a single absoluteURI. - if location = @header['location'] - if @request_uri - @header['location'] = @request_uri.merge(location).to_s - end - end - end - - def make_body_tempfile # :nodoc: - return if @bodytempfile - bodytempfile = Tempfile.create("webrick") - if @body.nil? - # nothing - elsif @body.respond_to? :readpartial - IO.copy_stream(@body, bodytempfile) - @body.close - elsif @body.respond_to? :call - @body.call(bodytempfile) - else - bodytempfile.write @body - end - bodytempfile.rewind - @body = @bodytempfile = bodytempfile - @header['content-length'] = bodytempfile.stat.size.to_s - end - - def remove_body_tempfile # :nodoc: - if @bodytempfile - @bodytempfile.close - File.unlink @bodytempfile.path - @bodytempfile = nil - end - end - - - ## - # Sends the headers on +socket+ - - def send_header(socket) # :nodoc: - if @http_version.major > 0 - data = status_line() - @header.each{|key, value| - tmp = key.gsub(/\bwww|^te$|\b\w/){ $&.upcase } - data << "#{tmp}: #{check_header(value)}" << CRLF - } - @cookies.each{|cookie| - data << "Set-Cookie: " << check_header(cookie.to_s) << CRLF - } - data << CRLF - socket.write(data) - end - rescue InvalidHeader => e - @header.clear - @cookies.clear - set_error e - retry - end - - ## - # Sends the body on +socket+ - - def send_body(socket) # :nodoc: - if @body.respond_to? :readpartial then - send_body_io(socket) - elsif @body.respond_to?(:call) then - send_body_proc(socket) - else - send_body_string(socket) - end - end - - ## - # Redirects to +url+ with a WEBrick::HTTPStatus::Redirect +status+. - # - # Example: - # - # res.set_redirect WEBrick::HTTPStatus::TemporaryRedirect - - def set_redirect(status, url) - url = URI(url).to_s - @body = "<HTML><A HREF=\"#{url}\">#{url}</A>.</HTML>\n" - @header['location'] = url - raise status - end - - ## - # Creates an error page for exception +ex+ with an optional +backtrace+ - - def set_error(ex, backtrace=false) - case ex - when HTTPStatus::Status - @keep_alive = false if HTTPStatus::error?(ex.code) - self.status = ex.code - else - @keep_alive = false - self.status = HTTPStatus::RC_INTERNAL_SERVER_ERROR - end - @header['content-type'] = "text/html; charset=ISO-8859-1" - - if respond_to?(:create_error_page) - create_error_page() - return - end - - if @request_uri - host, port = @request_uri.host, @request_uri.port - else - host, port = @config[:ServerName], @config[:Port] - end - - error_body(backtrace, ex, host, port) - end - - private - - def check_header(header_value) - header_value = header_value.to_s - if /[\r\n]/ =~ header_value - raise InvalidHeader - else - header_value - end - end - - # :stopdoc: - - def error_body(backtrace, ex, host, port) - @body = '' - @body << <<-_end_of_html_ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN"> -<HTML> - <HEAD><TITLE>#{HTMLUtils::escape(@reason_phrase)}</TITLE></HEAD> - <BODY> - <H1>#{HTMLUtils::escape(@reason_phrase)}</H1> - #{HTMLUtils::escape(ex.message)} - <HR> - _end_of_html_ - - if backtrace && $DEBUG - @body << "backtrace of `#{HTMLUtils::escape(ex.class.to_s)}' " - @body << "#{HTMLUtils::escape(ex.message)}" - @body << "<PRE>" - ex.backtrace.each{|line| @body << "\t#{line}\n"} - @body << "</PRE><HR>" - end - - @body << <<-_end_of_html_ - <ADDRESS> - #{HTMLUtils::escape(@config[:ServerSoftware])} at - #{host}:#{port} - </ADDRESS> - </BODY> -</HTML> - _end_of_html_ - end - - def send_body_io(socket) - begin - if @request_method == "HEAD" - # do nothing - elsif chunked? - buf = '' - begin - @body.readpartial(@buffer_size, buf) - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - socket.write(data) - data.clear - @sent_size += size - rescue EOFError - break - end while true - buf.clear - socket.write("0#{CRLF}#{CRLF}") - else - if %r{\Abytes (\d+)-(\d+)/\d+\z} =~ @header['content-range'] - offset = $1.to_i - size = $2.to_i - offset + 1 - else - offset = nil - size = @header['content-length'] - size = size.to_i if size - end - begin - @sent_size = IO.copy_stream(@body, socket, size, offset) - rescue NotImplementedError - @body.seek(offset, IO::SEEK_SET) - @sent_size = IO.copy_stream(@body, socket, size) - end - end - ensure - @body.close - end - remove_body_tempfile - end - - def send_body_string(socket) - if @request_method == "HEAD" - # do nothing - elsif chunked? - body ? @body.bytesize : 0 - while buf = @body[@sent_size, @buffer_size] - break if buf.empty? - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - buf.clear - socket.write(data) - @sent_size += size - end - socket.write("0#{CRLF}#{CRLF}") - else - if @body && @body.bytesize > 0 - socket.write(@body) - @sent_size = @body.bytesize - end - end - end - - def send_body_proc(socket) - if @request_method == "HEAD" - # do nothing - elsif chunked? - @body.call(ChunkedWrapper.new(socket, self)) - socket.write("0#{CRLF}#{CRLF}") - else - size = @header['content-length'].to_i - if @bodytempfile - @bodytempfile.rewind - IO.copy_stream(@bodytempfile, socket) - else - @body.call(socket) - end - @sent_size = size - end - end - - class ChunkedWrapper - def initialize(socket, resp) - @socket = socket - @resp = resp - end - - def write(buf) - return 0 if buf.empty? - socket = @socket - @resp.instance_eval { - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - socket.write(data) - data.clear - @sent_size += size - size - } - end - - def <<(*buf) - write(buf) - self - end - end - - # preserved for compatibility with some 3rd-party handlers - def _write_data(socket, data) - socket << data - end - - # :startdoc: - end - -end diff --git a/tool/lib/webrick/https.rb b/tool/lib/webrick/https.rb deleted file mode 100644 index b0a49bc40b..0000000000 --- a/tool/lib/webrick/https.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: false -# -# https.rb -- SSL/TLS enhancement for HTTPServer -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: https.rb,v 1.15 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'ssl' -require_relative 'httpserver' - -module WEBrick - module Config - HTTP.update(SSL) - end - - ## - #-- - # Adds SSL functionality to WEBrick::HTTPRequest - - class HTTPRequest - - ## - # HTTP request SSL cipher - - attr_reader :cipher - - ## - # HTTP request server certificate - - attr_reader :server_cert - - ## - # HTTP request client certificate - - attr_reader :client_cert - - # :stopdoc: - - alias orig_parse parse - - def parse(socket=nil) - if socket.respond_to?(:cert) - @server_cert = socket.cert || @config[:SSLCertificate] - @client_cert = socket.peer_cert - @client_cert_chain = socket.peer_cert_chain - @cipher = socket.cipher - end - orig_parse(socket) - end - - alias orig_parse_uri parse_uri - - def parse_uri(str, scheme="https") - if server_cert - return orig_parse_uri(str, scheme) - end - return orig_parse_uri(str) - end - private :parse_uri - - alias orig_meta_vars meta_vars - - def meta_vars - meta = orig_meta_vars - if server_cert - meta["HTTPS"] = "on" - meta["SSL_SERVER_CERT"] = @server_cert.to_pem - meta["SSL_CLIENT_CERT"] = @client_cert ? @client_cert.to_pem : "" - if @client_cert_chain - @client_cert_chain.each_with_index{|cert, i| - meta["SSL_CLIENT_CERT_CHAIN_#{i}"] = cert.to_pem - } - end - meta["SSL_CIPHER"] = @cipher[0] - meta["SSL_PROTOCOL"] = @cipher[1] - meta["SSL_CIPHER_USEKEYSIZE"] = @cipher[2].to_s - meta["SSL_CIPHER_ALGKEYSIZE"] = @cipher[3].to_s - end - meta - end - - # :startdoc: - end - - ## - #-- - # Fake WEBrick::HTTPRequest for lookup_server - - class SNIRequest - - ## - # The SNI hostname - - attr_reader :host - - ## - # The socket address of the server - - attr_reader :addr - - ## - # The port this request is for - - attr_reader :port - - ## - # Creates a new SNIRequest. - - def initialize(sslsocket, hostname) - @host = hostname - @addr = sslsocket.addr - @port = @addr[1] - end - end - - - ## - #-- - # Adds SSL functionality to WEBrick::HTTPServer - - class HTTPServer < ::WEBrick::GenericServer - ## - # ServerNameIndication callback - - def ssl_servername_callback(sslsocket, hostname = nil) - req = SNIRequest.new(sslsocket, hostname) - server = lookup_server(req) - server ? server.ssl_context : nil - end - - # :stopdoc: - - ## - # Check whether +server+ is also SSL server. - # Also +server+'s SSL context will be created. - - alias orig_virtual_host virtual_host - - def virtual_host(server) - if @config[:SSLEnable] && !server.ssl_context - raise ArgumentError, "virtual host must set SSLEnable to true" - end - orig_virtual_host(server) - end - - # :startdoc: - end -end diff --git a/tool/lib/webrick/httpserver.rb b/tool/lib/webrick/httpserver.rb deleted file mode 100644 index f3f948da3b..0000000000 --- a/tool/lib/webrick/httpserver.rb +++ /dev/null @@ -1,293 +0,0 @@ -# frozen_string_literal: false -# -# httpserver.rb -- HTTPServer Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpserver.rb,v 1.63 2002/10/01 17:16:32 gotoyuzo Exp $ - -require_relative 'server' -require_relative 'httputils' -require_relative 'httpstatus' -require_relative 'httprequest' -require_relative 'httpresponse' -require_relative 'httpservlet' -require_relative 'accesslog' - -module WEBrick - class HTTPServerError < ServerError; end - - ## - # An HTTP Server - - class HTTPServer < ::WEBrick::GenericServer - ## - # Creates a new HTTP server according to +config+ - # - # An HTTP server uses the following attributes: - # - # :AccessLog:: An array of access logs. See WEBrick::AccessLog - # :BindAddress:: Local address for the server to bind to - # :DocumentRoot:: Root path to serve files from - # :DocumentRootOptions:: Options for the default HTTPServlet::FileHandler - # :HTTPVersion:: The HTTP version of this server - # :Port:: Port to listen on - # :RequestCallback:: Called with a request and response before each - # request is serviced. - # :RequestTimeout:: Maximum time to wait between requests - # :ServerAlias:: Array of alternate names for this server for virtual - # hosting - # :ServerName:: Name for this server for virtual hosting - - def initialize(config={}, default=Config::HTTP) - super(config, default) - @http_version = HTTPVersion::convert(@config[:HTTPVersion]) - - @mount_tab = MountTable.new - if @config[:DocumentRoot] - mount("/", HTTPServlet::FileHandler, @config[:DocumentRoot], - @config[:DocumentRootOptions]) - end - - unless @config[:AccessLog] - @config[:AccessLog] = [ - [ $stderr, AccessLog::COMMON_LOG_FORMAT ], - [ $stderr, AccessLog::REFERER_LOG_FORMAT ] - ] - end - - @virtual_hosts = Array.new - end - - ## - # Processes requests on +sock+ - - def run(sock) - while true - req = create_request(@config) - res = create_response(@config) - server = self - begin - timeout = @config[:RequestTimeout] - while timeout > 0 - break if sock.to_io.wait_readable(0.5) - break if @status != :Running - timeout -= 0.5 - end - raise HTTPStatus::EOFError if timeout <= 0 || @status != :Running - raise HTTPStatus::EOFError if sock.eof? - req.parse(sock) - res.request_method = req.request_method - res.request_uri = req.request_uri - res.request_http_version = req.http_version - res.keep_alive = req.keep_alive? - server = lookup_server(req) || self - if callback = server[:RequestCallback] - callback.call(req, res) - elsif callback = server[:RequestHandler] - msg = ":RequestHandler is deprecated, please use :RequestCallback" - @logger.warn(msg) - callback.call(req, res) - end - server.service(req, res) - rescue HTTPStatus::EOFError, HTTPStatus::RequestTimeout => ex - res.set_error(ex) - rescue HTTPStatus::Error => ex - @logger.error(ex.message) - res.set_error(ex) - rescue HTTPStatus::Status => ex - res.status = ex.code - rescue StandardError => ex - @logger.error(ex) - res.set_error(ex, true) - ensure - if req.request_line - if req.keep_alive? && res.keep_alive? - req.fixup() - end - res.send_response(sock) - server.access_log(@config, req, res) - end - end - break if @http_version < "1.1" - break unless req.keep_alive? - break unless res.keep_alive? - end - end - - ## - # Services +req+ and fills in +res+ - - def service(req, res) - if req.unparsed_uri == "*" - if req.request_method == "OPTIONS" - do_OPTIONS(req, res) - raise HTTPStatus::OK - end - raise HTTPStatus::NotFound, "`#{req.unparsed_uri}' not found." - end - - servlet, options, script_name, path_info = search_servlet(req.path) - raise HTTPStatus::NotFound, "`#{req.path}' not found." unless servlet - req.script_name = script_name - req.path_info = path_info - si = servlet.get_instance(self, *options) - @logger.debug(format("%s is invoked.", si.class.name)) - si.service(req, res) - end - - ## - # The default OPTIONS request handler says GET, HEAD, POST and OPTIONS - # requests are allowed. - - def do_OPTIONS(req, res) - res["allow"] = "GET,HEAD,POST,OPTIONS" - end - - ## - # Mounts +servlet+ on +dir+ passing +options+ to the servlet at creation - # time - - def mount(dir, servlet, *options) - @logger.debug(sprintf("%s is mounted on %s.", servlet.inspect, dir)) - @mount_tab[dir] = [ servlet, options ] - end - - ## - # Mounts +proc+ or +block+ on +dir+ and calls it with a - # WEBrick::HTTPRequest and WEBrick::HTTPResponse - - def mount_proc(dir, proc=nil, &block) - proc ||= block - raise HTTPServerError, "must pass a proc or block" unless proc - mount(dir, HTTPServlet::ProcHandler.new(proc)) - end - - ## - # Unmounts +dir+ - - def unmount(dir) - @logger.debug(sprintf("unmount %s.", dir)) - @mount_tab.delete(dir) - end - alias umount unmount - - ## - # Finds a servlet for +path+ - - def search_servlet(path) - script_name, path_info = @mount_tab.scan(path) - servlet, options = @mount_tab[script_name] - if servlet - [ servlet, options, script_name, path_info ] - end - end - - ## - # Adds +server+ as a virtual host. - - def virtual_host(server) - @virtual_hosts << server - @virtual_hosts = @virtual_hosts.sort_by{|s| - num = 0 - num -= 4 if s[:BindAddress] - num -= 2 if s[:Port] - num -= 1 if s[:ServerName] - num - } - end - - ## - # Finds the appropriate virtual host to handle +req+ - - def lookup_server(req) - @virtual_hosts.find{|s| - (s[:BindAddress].nil? || req.addr[3] == s[:BindAddress]) && - (s[:Port].nil? || req.port == s[:Port]) && - ((s[:ServerName].nil? || req.host == s[:ServerName]) || - (!s[:ServerAlias].nil? && s[:ServerAlias].find{|h| h === req.host})) - } - end - - ## - # Logs +req+ and +res+ in the access logs. +config+ is used for the - # server name. - - def access_log(config, req, res) - param = AccessLog::setup_params(config, req, res) - @config[:AccessLog].each{|logger, fmt| - logger << AccessLog::format(fmt+"\n", param) - } - end - - ## - # Creates the HTTPRequest used when handling the HTTP - # request. Can be overridden by subclasses. - def create_request(with_webrick_config) - HTTPRequest.new(with_webrick_config) - end - - ## - # Creates the HTTPResponse used when handling the HTTP - # request. Can be overridden by subclasses. - def create_response(with_webrick_config) - HTTPResponse.new(with_webrick_config) - end - - ## - # Mount table for the path a servlet is mounted on in the directory space - # of the server. Users of WEBrick can only access this indirectly via - # WEBrick::HTTPServer#mount, WEBrick::HTTPServer#unmount and - # WEBrick::HTTPServer#search_servlet - - class MountTable # :nodoc: - def initialize - @tab = Hash.new - compile - end - - def [](dir) - dir = normalize(dir) - @tab[dir] - end - - def []=(dir, val) - dir = normalize(dir) - @tab[dir] = val - compile - val - end - - def delete(dir) - dir = normalize(dir) - res = @tab.delete(dir) - compile - res - end - - def scan(path) - @scanner =~ path - [ $&, $' ] - end - - private - - def compile - k = @tab.keys - k.sort! - k.reverse! - k.collect!{|path| Regexp.escape(path) } - @scanner = Regexp.new("\\A(" + k.join("|") +")(?=/|\\z)") - end - - def normalize(dir) - ret = dir ? dir.dup : "" - ret.sub!(%r|/+\z|, "") - ret - end - end - end -end diff --git a/tool/lib/webrick/httpservlet.rb b/tool/lib/webrick/httpservlet.rb deleted file mode 100644 index da49a1405b..0000000000 --- a/tool/lib/webrick/httpservlet.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: false -# -# httpservlet.rb -- HTTPServlet Utility File -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpservlet.rb,v 1.21 2003/02/23 12:24:46 gotoyuzo Exp $ - -require_relative 'httpservlet/abstract' -require_relative 'httpservlet/filehandler' -require_relative 'httpservlet/cgihandler' -require_relative 'httpservlet/erbhandler' -require_relative 'httpservlet/prochandler' - -module WEBrick - module HTTPServlet - FileHandler.add_handler("cgi", CGIHandler) - FileHandler.add_handler("rhtml", ERBHandler) - end -end diff --git a/tool/lib/webrick/httpservlet/abstract.rb b/tool/lib/webrick/httpservlet/abstract.rb deleted file mode 100644 index bccb091861..0000000000 --- a/tool/lib/webrick/httpservlet/abstract.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: false -# -# httpservlet.rb -- HTTPServlet Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: abstract.rb,v 1.24 2003/07/11 11:16:46 gotoyuzo Exp $ - -require_relative '../htmlutils' -require_relative '../httputils' -require_relative '../httpstatus' - -module WEBrick - module HTTPServlet - class HTTPServletError < StandardError; end - - ## - # AbstractServlet allows HTTP server modules to be reused across multiple - # servers and allows encapsulation of functionality. - # - # By default a servlet will respond to GET, HEAD (through an alias to GET) - # and OPTIONS requests. - # - # By default a new servlet is initialized for every request. A servlet - # instance can be reused by overriding ::get_instance in the - # AbstractServlet subclass. - # - # == A Simple Servlet - # - # class Simple < WEBrick::HTTPServlet::AbstractServlet - # def do_GET request, response - # status, content_type, body = do_stuff_with request - # - # response.status = status - # response['Content-Type'] = content_type - # response.body = body - # end - # - # def do_stuff_with request - # return 200, 'text/plain', 'you got a page' - # end - # end - # - # This servlet can be mounted on a server at a given path: - # - # server.mount '/simple', Simple - # - # == Servlet Configuration - # - # Servlets can be configured via initialize. The first argument is the - # HTTP server the servlet is being initialized for. - # - # class Configurable < Simple - # def initialize server, color, size - # super server - # @color = color - # @size = size - # end - # - # def do_stuff_with request - # content = "<p " \ - # %q{style="color: #{@color}; font-size: #{@size}"} \ - # ">Hello, World!" - # - # return 200, "text/html", content - # end - # end - # - # This servlet must be provided two arguments at mount time: - # - # server.mount '/configurable', Configurable, 'red', '2em' - - class AbstractServlet - - ## - # Factory for servlet instances that will handle a request from +server+ - # using +options+ from the mount point. By default a new servlet - # instance is created for every call. - - def self.get_instance(server, *options) - self.new(server, *options) - end - - ## - # Initializes a new servlet for +server+ using +options+ which are - # stored as-is in +@options+. +@logger+ is also provided. - - def initialize(server, *options) - @server = @config = server - @logger = @server[:Logger] - @options = options - end - - ## - # Dispatches to a +do_+ method based on +req+ if such a method is - # available. (+do_GET+ for a GET request). Raises a MethodNotAllowed - # exception if the method is not implemented. - - def service(req, res) - method_name = "do_" + req.request_method.gsub(/-/, "_") - if respond_to?(method_name) - __send__(method_name, req, res) - else - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - end - end - - ## - # Raises a NotFound exception - - def do_GET(req, res) - raise HTTPStatus::NotFound, "not found." - end - - ## - # Dispatches to do_GET - - def do_HEAD(req, res) - do_GET(req, res) - end - - ## - # Returns the allowed HTTP request methods - - def do_OPTIONS(req, res) - m = self.methods.grep(/\Ado_([A-Z]+)\z/) {$1} - m.sort! - res["allow"] = m.join(",") - end - - private - - ## - # Redirects to a path ending in / - - def redirect_to_directory_uri(req, res) - if req.path[-1] != ?/ - location = WEBrick::HTTPUtils.escape_path(req.path + "/") - if req.query_string && req.query_string.bytesize > 0 - location << "?" << req.query_string - end - res.set_redirect(HTTPStatus::MovedPermanently, location) - end - end - end - - end -end diff --git a/tool/lib/webrick/httpservlet/cgi_runner.rb b/tool/lib/webrick/httpservlet/cgi_runner.rb deleted file mode 100644 index 0398c16749..0000000000 --- a/tool/lib/webrick/httpservlet/cgi_runner.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: false -# -# cgi_runner.rb -- CGI launcher. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cgi_runner.rb,v 1.9 2002/09/25 11:33:15 gotoyuzo Exp $ - -def sysread(io, size) - buf = "" - while size > 0 - tmp = io.sysread(size) - buf << tmp - size -= tmp.bytesize - end - return buf -end - -STDIN.binmode - -len = sysread(STDIN, 8).to_i -out = sysread(STDIN, len) -STDOUT.reopen(File.open(out, "w")) - -len = sysread(STDIN, 8).to_i -err = sysread(STDIN, len) -STDERR.reopen(File.open(err, "w")) - -len = sysread(STDIN, 8).to_i -dump = sysread(STDIN, len) -hash = Marshal.restore(dump) -ENV.keys.each{|name| ENV.delete(name) } -hash.each{|k, v| ENV[k] = v if v } - -dir = File::dirname(ENV["SCRIPT_FILENAME"]) -Dir::chdir dir - -if ARGV[0] - argv = ARGV.dup - argv << ENV["SCRIPT_FILENAME"] - exec(*argv) - # NOTREACHED -end -exec ENV["SCRIPT_FILENAME"] diff --git a/tool/lib/webrick/httpservlet/cgihandler.rb b/tool/lib/webrick/httpservlet/cgihandler.rb deleted file mode 100644 index 4457770b7a..0000000000 --- a/tool/lib/webrick/httpservlet/cgihandler.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: false -# -# cgihandler.rb -- CGIHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cgihandler.rb,v 1.27 2003/03/21 19:56:01 gotoyuzo Exp $ - -require 'rbconfig' -require 'tempfile' -require_relative '../config' -require_relative 'abstract' - -module WEBrick - module HTTPServlet - - ## - # Servlet for handling CGI scripts - # - # Example: - # - # server.mount('/cgi/my_script', WEBrick::HTTPServlet::CGIHandler, - # '/path/to/my_script') - - class CGIHandler < AbstractServlet - Ruby = RbConfig.ruby # :nodoc: - CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: - CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb".freeze].freeze # :nodoc: - - ## - # Creates a new CGI script servlet for the script at +name+ - - def initialize(server, name) - super(server, name) - @script_filename = name - @tempdir = server[:TempDir] - interpreter = server[:CGIInterpreter] - if interpreter.is_a?(Array) - @cgicmd = CGIRunnerArray + interpreter - else - @cgicmd = "#{CGIRunner} #{interpreter}" - end - end - - # :stopdoc: - - def do_GET(req, res) - cgi_in = IO::popen(@cgicmd, "wb") - cgi_out = Tempfile.new("webrick.cgiout.", @tempdir, mode: IO::BINARY) - cgi_out.set_encoding("ASCII-8BIT") - cgi_err = Tempfile.new("webrick.cgierr.", @tempdir, mode: IO::BINARY) - cgi_err.set_encoding("ASCII-8BIT") - begin - cgi_in.sync = true - meta = req.meta_vars - meta["SCRIPT_FILENAME"] = @script_filename - meta["PATH"] = @config[:CGIPathEnv] - meta.delete("HTTP_PROXY") - if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM - meta["SystemRoot"] = ENV["SystemRoot"] - end - dump = Marshal.dump(meta) - - cgi_in.write("%8d" % cgi_out.path.bytesize) - cgi_in.write(cgi_out.path) - cgi_in.write("%8d" % cgi_err.path.bytesize) - cgi_in.write(cgi_err.path) - cgi_in.write("%8d" % dump.bytesize) - cgi_in.write(dump) - - req.body { |chunk| cgi_in.write(chunk) } - ensure - cgi_in.close - status = $?.exitstatus - sleep 0.1 if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM - data = cgi_out.read - cgi_out.close(true) - if errmsg = cgi_err.read - if errmsg.bytesize > 0 - @logger.error("CGIHandler: #{@script_filename}:\n" + errmsg) - end - end - cgi_err.close(true) - end - - if status != 0 - @logger.error("CGIHandler: #{@script_filename} exit with #{status}") - end - - data = "" unless data - raw_header, body = data.split(/^[\xd\xa]+/, 2) - raise HTTPStatus::InternalServerError, - "Premature end of script headers: #{@script_filename}" if body.nil? - - begin - header = HTTPUtils::parse_header(raw_header) - if /^(\d+)/ =~ header['status'][0] - res.status = $1.to_i - header.delete('status') - end - if header.has_key?('location') - # RFC 3875 6.2.3, 6.2.4 - res.status = 302 unless (300...400) === res.status - end - if header.has_key?('set-cookie') - header['set-cookie'].each{|k| - res.cookies << Cookie.parse_set_cookie(k) - } - header.delete('set-cookie') - end - header.each{|key, val| res[key] = val.join(", ") } - rescue => ex - raise HTTPStatus::InternalServerError, ex.message - end - res.body = body - end - alias do_POST do_GET - - # :startdoc: - end - - end -end diff --git a/tool/lib/webrick/httpservlet/erbhandler.rb b/tool/lib/webrick/httpservlet/erbhandler.rb deleted file mode 100644 index cd09e5f216..0000000000 --- a/tool/lib/webrick/httpservlet/erbhandler.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: false -# -# erbhandler.rb -- ERBHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: erbhandler.rb,v 1.25 2003/02/24 19:25:31 gotoyuzo Exp $ - -require_relative 'abstract' - -require 'erb' - -module WEBrick - module HTTPServlet - - ## - # ERBHandler evaluates an ERB file and returns the result. This handler - # is automatically used if there are .rhtml files in a directory served by - # the FileHandler. - # - # ERBHandler supports GET and POST methods. - # - # The ERB file is evaluated with the local variables +servlet_request+ and - # +servlet_response+ which are a WEBrick::HTTPRequest and - # WEBrick::HTTPResponse respectively. - # - # Example .rhtml file: - # - # Request to <%= servlet_request.request_uri %> - # - # Query params <%= servlet_request.query.inspect %> - - class ERBHandler < AbstractServlet - - ## - # Creates a new ERBHandler on +server+ that will evaluate and serve the - # ERB file +name+ - - def initialize(server, name) - super(server, name) - @script_filename = name - end - - ## - # Handles GET requests - - def do_GET(req, res) - unless defined?(ERB) - @logger.warn "#{self.class}: ERB not defined." - raise HTTPStatus::Forbidden, "ERBHandler cannot work." - end - begin - data = File.open(@script_filename, &:read) - res.body = evaluate(ERB.new(data), req, res) - res['content-type'] ||= - HTTPUtils::mime_type(@script_filename, @config[:MimeTypes]) - rescue StandardError - raise - rescue Exception => ex - @logger.error(ex) - raise HTTPStatus::InternalServerError, ex.message - end - end - - ## - # Handles POST requests - - alias do_POST do_GET - - private - - ## - # Evaluates +erb+ providing +servlet_request+ and +servlet_response+ as - # local variables. - - def evaluate(erb, servlet_request, servlet_response) - Module.new.module_eval{ - servlet_request.meta_vars - servlet_request.query - erb.result(binding) - } - end - end - end -end diff --git a/tool/lib/webrick/httpservlet/filehandler.rb b/tool/lib/webrick/httpservlet/filehandler.rb deleted file mode 100644 index 010df0e918..0000000000 --- a/tool/lib/webrick/httpservlet/filehandler.rb +++ /dev/null @@ -1,552 +0,0 @@ -# frozen_string_literal: false -# -# filehandler.rb -- FileHandler Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: filehandler.rb,v 1.44 2003/06/07 01:34:51 gotoyuzo Exp $ - -require 'time' - -require_relative '../htmlutils' -require_relative '../httputils' -require_relative '../httpstatus' - -module WEBrick - module HTTPServlet - - ## - # Servlet for serving a single file. You probably want to use the - # FileHandler servlet instead as it handles directories and fancy indexes. - # - # Example: - # - # server.mount('/my_page.txt', WEBrick::HTTPServlet::DefaultFileHandler, - # '/path/to/my_page.txt') - # - # This servlet handles If-Modified-Since and Range requests. - - class DefaultFileHandler < AbstractServlet - - ## - # Creates a DefaultFileHandler instance for the file at +local_path+. - - def initialize(server, local_path) - super(server, local_path) - @local_path = local_path - end - - # :stopdoc: - - def do_GET(req, res) - st = File::stat(@local_path) - mtime = st.mtime - res['etag'] = sprintf("%x-%x-%x", st.ino, st.size, st.mtime.to_i) - - if not_modified?(req, res, mtime, res['etag']) - res.body = '' - raise HTTPStatus::NotModified - elsif req['range'] - make_partial_content(req, res, @local_path, st.size) - raise HTTPStatus::PartialContent - else - mtype = HTTPUtils::mime_type(@local_path, @config[:MimeTypes]) - res['content-type'] = mtype - res['content-length'] = st.size.to_s - res['last-modified'] = mtime.httpdate - res.body = File.open(@local_path, "rb") - end - end - - def not_modified?(req, res, mtime, etag) - if ir = req['if-range'] - begin - if Time.httpdate(ir) >= mtime - return true - end - rescue - if HTTPUtils::split_header_value(ir).member?(res['etag']) - return true - end - end - end - - if (ims = req['if-modified-since']) && Time.parse(ims) >= mtime - return true - end - - if (inm = req['if-none-match']) && - HTTPUtils::split_header_value(inm).member?(res['etag']) - return true - end - - return false - end - - # returns a lambda for webrick/httpresponse.rb send_body_proc - def multipart_body(body, parts, boundary, mtype, filesize) - lambda do |socket| - begin - begin - first = parts.shift - last = parts.shift - socket.write( - "--#{boundary}#{CRLF}" \ - "Content-Type: #{mtype}#{CRLF}" \ - "Content-Range: bytes #{first}-#{last}/#{filesize}#{CRLF}" \ - "#{CRLF}" - ) - - begin - IO.copy_stream(body, socket, last - first + 1, first) - rescue NotImplementedError - body.seek(first, IO::SEEK_SET) - IO.copy_stream(body, socket, last - first + 1) - end - socket.write(CRLF) - end while parts[0] - socket.write("--#{boundary}--#{CRLF}") - ensure - body.close - end - end - end - - def make_partial_content(req, res, filename, filesize) - mtype = HTTPUtils::mime_type(filename, @config[:MimeTypes]) - unless ranges = HTTPUtils::parse_range_header(req['range']) - raise HTTPStatus::BadRequest, - "Unrecognized range-spec: \"#{req['range']}\"" - end - File.open(filename, "rb"){|io| - if ranges.size > 1 - time = Time.now - boundary = "#{time.sec}_#{time.usec}_#{Process::pid}" - parts = [] - ranges.each {|range| - prange = prepare_range(range, filesize) - next if prange[0] < 0 - parts.concat(prange) - } - raise HTTPStatus::RequestRangeNotSatisfiable if parts.empty? - res["content-type"] = "multipart/byteranges; boundary=#{boundary}" - if req.http_version < '1.1' - res['connection'] = 'close' - else - res.chunked = true - end - res.body = multipart_body(io.dup, parts, boundary, mtype, filesize) - elsif range = ranges[0] - first, last = prepare_range(range, filesize) - raise HTTPStatus::RequestRangeNotSatisfiable if first < 0 - res['content-type'] = mtype - res['content-range'] = "bytes #{first}-#{last}/#{filesize}" - res['content-length'] = (last - first + 1).to_s - res.body = io.dup - else - raise HTTPStatus::BadRequest - end - } - end - - def prepare_range(range, filesize) - first = range.first < 0 ? filesize + range.first : range.first - return -1, -1 if first < 0 || first >= filesize - last = range.last < 0 ? filesize + range.last : range.last - last = filesize - 1 if last >= filesize - return first, last - end - - # :startdoc: - end - - ## - # Serves a directory including fancy indexing and a variety of other - # options. - # - # Example: - # - # server.mount('/assets', WEBrick::HTTPServlet::FileHandler, - # '/path/to/assets') - - class FileHandler < AbstractServlet - HandlerTable = Hash.new # :nodoc: - - ## - # Allow custom handling of requests for files with +suffix+ by class - # +handler+ - - def self.add_handler(suffix, handler) - HandlerTable[suffix] = handler - end - - ## - # Remove custom handling of requests for files with +suffix+ - - def self.remove_handler(suffix) - HandlerTable.delete(suffix) - end - - ## - # Creates a FileHandler servlet on +server+ that serves files starting - # at directory +root+ - # - # +options+ may be a Hash containing keys from - # WEBrick::Config::FileHandler or +true+ or +false+. - # - # If +options+ is true or false then +:FancyIndexing+ is enabled or - # disabled respectively. - - def initialize(server, root, options={}, default=Config::FileHandler) - @config = server.config - @logger = @config[:Logger] - @root = File.expand_path(root) - if options == true || options == false - options = { :FancyIndexing => options } - end - @options = default.dup.update(options) - end - - # :stopdoc: - - def set_filesystem_encoding(str) - enc = Encoding.find('filesystem') - if enc == Encoding::US_ASCII - str.b - else - str.dup.force_encoding(enc) - end - end - - def service(req, res) - # if this class is mounted on "/" and /~username is requested. - # we're going to override path information before invoking service. - if defined?(Etc) && @options[:UserDir] && req.script_name.empty? - if %r|^(/~([^/]+))| =~ req.path_info - script_name, user = $1, $2 - path_info = $' - begin - passwd = Etc::getpwnam(user) - @root = File::join(passwd.dir, @options[:UserDir]) - req.script_name = script_name - req.path_info = path_info - rescue - @logger.debug "#{self.class}#do_GET: getpwnam(#{user}) failed" - end - end - end - prevent_directory_traversal(req, res) - super(req, res) - end - - def do_GET(req, res) - unless exec_handler(req, res) - set_dir_list(req, res) - end - end - - def do_POST(req, res) - unless exec_handler(req, res) - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - def do_OPTIONS(req, res) - unless exec_handler(req, res) - super(req, res) - end - end - - # ToDo - # RFC2518: HTTP Extensions for Distributed Authoring -- WEBDAV - # - # PROPFIND PROPPATCH MKCOL DELETE PUT COPY MOVE - # LOCK UNLOCK - - # RFC3253: Versioning Extensions to WebDAV - # (Web Distributed Authoring and Versioning) - # - # VERSION-CONTROL REPORT CHECKOUT CHECK_IN UNCHECKOUT - # MKWORKSPACE UPDATE LABEL MERGE ACTIVITY - - private - - def trailing_pathsep?(path) - # check for trailing path separator: - # File.dirname("/aaaa/bbbb/") #=> "/aaaa") - # File.dirname("/aaaa/bbbb/x") #=> "/aaaa/bbbb") - # File.dirname("/aaaa/bbbb") #=> "/aaaa") - # File.dirname("/aaaa/bbbbx") #=> "/aaaa") - return File.dirname(path) != File.dirname(path+"x") - end - - def prevent_directory_traversal(req, res) - # Preventing directory traversal on Windows platforms; - # Backslashes (0x5c) in path_info are not interpreted as special - # character in URI notation. So the value of path_info should be - # normalize before accessing to the filesystem. - - # dirty hack for filesystem encoding; in nature, File.expand_path - # should not be used for path normalization. [Bug #3345] - path = req.path_info.dup.force_encoding(Encoding.find("filesystem")) - if trailing_pathsep?(req.path_info) - # File.expand_path removes the trailing path separator. - # Adding a character is a workaround to save it. - # File.expand_path("/aaa/") #=> "/aaa" - # File.expand_path("/aaa/" + "x") #=> "/aaa/x" - expanded = File.expand_path(path + "x") - expanded.chop! # remove trailing "x" - else - expanded = File.expand_path(path) - end - expanded.force_encoding(req.path_info.encoding) - req.path_info = expanded - end - - def exec_handler(req, res) - raise HTTPStatus::NotFound, "`#{req.path}' not found." unless @root - if set_filename(req, res) - handler = get_handler(req, res) - call_callback(:HandlerCallback, req, res) - h = handler.get_instance(@config, res.filename) - h.service(req, res) - return true - end - call_callback(:HandlerCallback, req, res) - return false - end - - def get_handler(req, res) - suffix1 = (/\.(\w+)\z/ =~ res.filename) && $1.downcase - if /\.(\w+)\.([\w\-]+)\z/ =~ res.filename - if @options[:AcceptableLanguages].include?($2.downcase) - suffix2 = $1.downcase - end - end - handler_table = @options[:HandlerTable] - return handler_table[suffix1] || handler_table[suffix2] || - HandlerTable[suffix1] || HandlerTable[suffix2] || - DefaultFileHandler - end - - def set_filename(req, res) - res.filename = @root - path_info = req.path_info.scan(%r|/[^/]*|) - - path_info.unshift("") # dummy for checking @root dir - while base = path_info.first - base = set_filesystem_encoding(base) - break if base == "/" - break unless File.directory?(File.expand_path(res.filename + base)) - shift_path_info(req, res, path_info) - call_callback(:DirectoryCallback, req, res) - end - - if base = path_info.first - base = set_filesystem_encoding(base) - if base == "/" - if file = search_index_file(req, res) - shift_path_info(req, res, path_info, file) - call_callback(:FileCallback, req, res) - return true - end - shift_path_info(req, res, path_info) - elsif file = search_file(req, res, base) - shift_path_info(req, res, path_info, file) - call_callback(:FileCallback, req, res) - return true - else - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - return false - end - - def check_filename(req, res, name) - if nondisclosure_name?(name) || windows_ambiguous_name?(name) - @logger.warn("the request refers nondisclosure name `#{name}'.") - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - def shift_path_info(req, res, path_info, base=nil) - tmp = path_info.shift - base = base || set_filesystem_encoding(tmp) - req.path_info = path_info.join - req.script_name << base - res.filename = File.expand_path(res.filename + base) - check_filename(req, res, File.basename(res.filename)) - end - - def search_index_file(req, res) - @config[:DirectoryIndex].each{|index| - if file = search_file(req, res, "/"+index) - return file - end - } - return nil - end - - def search_file(req, res, basename) - langs = @options[:AcceptableLanguages] - path = res.filename + basename - if File.file?(path) - return basename - elsif langs.size > 0 - req.accept_language.each{|lang| - path_with_lang = path + ".#{lang}" - if langs.member?(lang) && File.file?(path_with_lang) - return basename + ".#{lang}" - end - } - (langs - req.accept_language).each{|lang| - path_with_lang = path + ".#{lang}" - if File.file?(path_with_lang) - return basename + ".#{lang}" - end - } - end - return nil - end - - def call_callback(callback_name, req, res) - if cb = @options[callback_name] - cb.call(req, res) - end - end - - def windows_ambiguous_name?(name) - return true if /[. ]+\z/ =~ name - return true if /::\$DATA\z/ =~ name - return false - end - - def nondisclosure_name?(name) - @options[:NondisclosureName].each{|pattern| - if File.fnmatch(pattern, name, File::FNM_CASEFOLD) - return true - end - } - return false - end - - def set_dir_list(req, res) - redirect_to_directory_uri(req, res) - unless @options[:FancyIndexing] - raise HTTPStatus::Forbidden, "no access permission to `#{req.path}'" - end - local_path = res.filename - list = Dir::entries(local_path).collect{|name| - next if name == "." || name == ".." - next if nondisclosure_name?(name) - next if windows_ambiguous_name?(name) - st = (File::stat(File.join(local_path, name)) rescue nil) - if st.nil? - [ name, nil, -1 ] - elsif st.directory? - [ name + "/", st.mtime, -1 ] - else - [ name, st.mtime, st.size ] - end - } - list.compact! - - query = req.query - - d0 = nil - idx = nil - %w[N M S].each_with_index do |q, i| - if d = query.delete(q) - idx ||= i - d0 ||= d - end - end - d0 ||= "A" - idx ||= 0 - d1 = (d0 == "A") ? "D" : "A" - - if d0 == "A" - list.sort!{|a,b| a[idx] <=> b[idx] } - else - list.sort!{|a,b| b[idx] <=> a[idx] } - end - - namewidth = query["NameWidth"] - if namewidth == "*" - namewidth = nil - elsif !namewidth or (namewidth = namewidth.to_i) < 2 - namewidth = 25 - end - query = query.inject('') {|s, (k, v)| s << '&' << HTMLUtils::escape("#{k}=#{v}")} - - type = "text/html" - case enc = Encoding.find('filesystem') - when Encoding::US_ASCII, Encoding::ASCII_8BIT - else - type << "; charset=\"#{enc.name}\"" - end - res['content-type'] = type - - title = "Index of #{HTMLUtils::escape(req.path)}" - res.body = <<-_end_of_html_ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> -<HTML> - <HEAD> - <TITLE>#{title}</TITLE> - <style type="text/css"> - <!-- - .name, .mtime { text-align: left; } - .size { text-align: right; } - td { text-overflow: ellipsis; white-space: nowrap; overflow: hidden; } - table { border-collapse: collapse; } - tr th { border-bottom: 2px groove; } - //--> - </style> - </HEAD> - <BODY> - <H1>#{title}</H1> - _end_of_html_ - - res.body << "<TABLE width=\"100%\"><THEAD><TR>\n" - res.body << "<TH class=\"name\"><A HREF=\"?N=#{d1}#{query}\">Name</A></TH>" - res.body << "<TH class=\"mtime\"><A HREF=\"?M=#{d1}#{query}\">Last modified</A></TH>" - res.body << "<TH class=\"size\"><A HREF=\"?S=#{d1}#{query}\">Size</A></TH>\n" - res.body << "</TR></THEAD>\n" - res.body << "<TBODY>\n" - - query.sub!(/\A&/, '?') - list.unshift [ "..", File::mtime(local_path+"/.."), -1 ] - list.each{ |name, time, size| - if name == ".." - dname = "Parent Directory" - elsif namewidth and name.size > namewidth - dname = name[0...(namewidth - 2)] << '..' - else - dname = name - end - s = "<TR><TD class=\"name\"><A HREF=\"#{HTTPUtils::escape(name)}#{query if name.end_with?('/')}\">#{HTMLUtils::escape(dname)}</A></TD>" - s << "<TD class=\"mtime\">" << (time ? time.strftime("%Y/%m/%d %H:%M") : "") << "</TD>" - s << "<TD class=\"size\">" << (size >= 0 ? size.to_s : "-") << "</TD></TR>\n" - res.body << s - } - res.body << "</TBODY></TABLE>" - res.body << "<HR>" - - res.body << <<-_end_of_html_ - <ADDRESS> - #{HTMLUtils::escape(@config[:ServerSoftware])}<BR> - at #{req.host}:#{req.port} - </ADDRESS> - </BODY> -</HTML> - _end_of_html_ - end - - # :startdoc: - end - end -end diff --git a/tool/lib/webrick/httpservlet/prochandler.rb b/tool/lib/webrick/httpservlet/prochandler.rb deleted file mode 100644 index 599ffc4340..0000000000 --- a/tool/lib/webrick/httpservlet/prochandler.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: false -# -# prochandler.rb -- ProcHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: prochandler.rb,v 1.7 2002/09/21 12:23:42 gotoyuzo Exp $ - -require_relative 'abstract' - -module WEBrick - module HTTPServlet - - ## - # Mounts a proc at a path that accepts a request and response. - # - # Instead of mounting this servlet with WEBrick::HTTPServer#mount use - # WEBrick::HTTPServer#mount_proc: - # - # server.mount_proc '/' do |req, res| - # res.body = 'it worked!' - # res.status = 200 - # end - - class ProcHandler < AbstractServlet - # :stopdoc: - def get_instance(server, *options) - self - end - - def initialize(proc) - @proc = proc - end - - def do_GET(request, response) - @proc.call(request, response) - end - - alias do_POST do_GET - # :startdoc: - end - - end -end diff --git a/tool/lib/webrick/httpstatus.rb b/tool/lib/webrick/httpstatus.rb deleted file mode 100644 index c811f21964..0000000000 --- a/tool/lib/webrick/httpstatus.rb +++ /dev/null @@ -1,194 +0,0 @@ -# frozen_string_literal: false -#-- -# httpstatus.rb -- HTTPStatus Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpstatus.rb,v 1.11 2003/03/24 20:18:55 gotoyuzo Exp $ - -require_relative 'accesslog' - -module WEBrick - - ## - # This module is used to manager HTTP status codes. - # - # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html for more - # information. - module HTTPStatus - - ## - # Root of the HTTP status class hierarchy - class Status < StandardError - class << self - attr_reader :code, :reason_phrase # :nodoc: - end - - # Returns the HTTP status code - def code() self::class::code end - - # Returns the HTTP status description - def reason_phrase() self::class::reason_phrase end - - alias to_i code # :nodoc: - end - - # Root of the HTTP info statuses - class Info < Status; end - # Root of the HTTP success statuses - class Success < Status; end - # Root of the HTTP redirect statuses - class Redirect < Status; end - # Root of the HTTP error statuses - class Error < Status; end - # Root of the HTTP client error statuses - class ClientError < Error; end - # Root of the HTTP server error statuses - class ServerError < Error; end - - class EOFError < StandardError; end - - # HTTP status codes and descriptions - StatusMessage = { # :nodoc: - 100 => 'Continue', - 101 => 'Switching Protocols', - 200 => 'OK', - 201 => 'Created', - 202 => 'Accepted', - 203 => 'Non-Authoritative Information', - 204 => 'No Content', - 205 => 'Reset Content', - 206 => 'Partial Content', - 207 => 'Multi-Status', - 300 => 'Multiple Choices', - 301 => 'Moved Permanently', - 302 => 'Found', - 303 => 'See Other', - 304 => 'Not Modified', - 305 => 'Use Proxy', - 307 => 'Temporary Redirect', - 400 => 'Bad Request', - 401 => 'Unauthorized', - 402 => 'Payment Required', - 403 => 'Forbidden', - 404 => 'Not Found', - 405 => 'Method Not Allowed', - 406 => 'Not Acceptable', - 407 => 'Proxy Authentication Required', - 408 => 'Request Timeout', - 409 => 'Conflict', - 410 => 'Gone', - 411 => 'Length Required', - 412 => 'Precondition Failed', - 413 => 'Request Entity Too Large', - 414 => 'Request-URI Too Large', - 415 => 'Unsupported Media Type', - 416 => 'Request Range Not Satisfiable', - 417 => 'Expectation Failed', - 422 => 'Unprocessable Entity', - 423 => 'Locked', - 424 => 'Failed Dependency', - 426 => 'Upgrade Required', - 428 => 'Precondition Required', - 429 => 'Too Many Requests', - 431 => 'Request Header Fields Too Large', - 451 => 'Unavailable For Legal Reasons', - 500 => 'Internal Server Error', - 501 => 'Not Implemented', - 502 => 'Bad Gateway', - 503 => 'Service Unavailable', - 504 => 'Gateway Timeout', - 505 => 'HTTP Version Not Supported', - 507 => 'Insufficient Storage', - 511 => 'Network Authentication Required', - } - - # Maps a status code to the corresponding Status class - CodeToError = {} # :nodoc: - - # Creates a status or error class for each status code and - # populates the CodeToError map. - StatusMessage.each{|code, message| - message.freeze - var_name = message.gsub(/[ \-]/,'_').upcase - err_name = message.gsub(/[ \-]/,'') - - case code - when 100...200; parent = Info - when 200...300; parent = Success - when 300...400; parent = Redirect - when 400...500; parent = ClientError - when 500...600; parent = ServerError - end - - const_set("RC_#{var_name}", code) - err_class = Class.new(parent) - err_class.instance_variable_set(:@code, code) - err_class.instance_variable_set(:@reason_phrase, message) - const_set(err_name, err_class) - CodeToError[code] = err_class - } - - ## - # Returns the description corresponding to the HTTP status +code+ - # - # WEBrick::HTTPStatus.reason_phrase 404 - # => "Not Found" - def reason_phrase(code) - StatusMessage[code.to_i] - end - - ## - # Is +code+ an informational status? - def info?(code) - code.to_i >= 100 and code.to_i < 200 - end - - ## - # Is +code+ a successful status? - def success?(code) - code.to_i >= 200 and code.to_i < 300 - end - - ## - # Is +code+ a redirection status? - def redirect?(code) - code.to_i >= 300 and code.to_i < 400 - end - - ## - # Is +code+ an error status? - def error?(code) - code.to_i >= 400 and code.to_i < 600 - end - - ## - # Is +code+ a client error status? - def client_error?(code) - code.to_i >= 400 and code.to_i < 500 - end - - ## - # Is +code+ a server error status? - def server_error?(code) - code.to_i >= 500 and code.to_i < 600 - end - - ## - # Returns the status class corresponding to +code+ - # - # WEBrick::HTTPStatus[302] - # => WEBrick::HTTPStatus::NotFound - # - def self.[](code) - CodeToError[code] - end - - module_function :reason_phrase - module_function :info?, :success?, :redirect?, :error? - module_function :client_error?, :server_error? - end -end diff --git a/tool/lib/webrick/httputils.rb b/tool/lib/webrick/httputils.rb deleted file mode 100644 index e21284ee7f..0000000000 --- a/tool/lib/webrick/httputils.rb +++ /dev/null @@ -1,512 +0,0 @@ -# frozen_string_literal: false -# -# httputils.rb -- HTTPUtils Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httputils.rb,v 1.34 2003/06/05 21:34:08 gotoyuzo Exp $ - -require 'socket' -require 'tempfile' - -module WEBrick - CR = "\x0d" # :nodoc: - LF = "\x0a" # :nodoc: - CRLF = "\x0d\x0a" # :nodoc: - - ## - # HTTPUtils provides utility methods for working with the HTTP protocol. - # - # This module is generally used internally by WEBrick - - module HTTPUtils - - ## - # Normalizes a request path. Raises an exception if the path cannot be - # normalized. - - def normalize_path(path) - raise "abnormal path `#{path}'" if path[0] != ?/ - ret = path.dup - - ret.gsub!(%r{/+}o, '/') # // => / - while ret.sub!(%r'/\.(?:/|\Z)', '/'); end # /. => / - while ret.sub!(%r'/(?!\.\./)[^/]+/\.\.(?:/|\Z)', '/'); end # /foo/.. => /foo - - raise "abnormal path `#{path}'" if %r{/\.\.(/|\Z)} =~ ret - ret - end - module_function :normalize_path - - ## - # Default mime types - - DefaultMimeTypes = { - "ai" => "application/postscript", - "asc" => "text/plain", - "avi" => "video/x-msvideo", - "bin" => "application/octet-stream", - "bmp" => "image/bmp", - "class" => "application/octet-stream", - "cer" => "application/pkix-cert", - "crl" => "application/pkix-crl", - "crt" => "application/x-x509-ca-cert", - #"crl" => "application/x-pkcs7-crl", - "css" => "text/css", - "dms" => "application/octet-stream", - "doc" => "application/msword", - "dvi" => "application/x-dvi", - "eps" => "application/postscript", - "etx" => "text/x-setext", - "exe" => "application/octet-stream", - "gif" => "image/gif", - "htm" => "text/html", - "html" => "text/html", - "jpe" => "image/jpeg", - "jpeg" => "image/jpeg", - "jpg" => "image/jpeg", - "js" => "application/javascript", - "json" => "application/json", - "lha" => "application/octet-stream", - "lzh" => "application/octet-stream", - "mjs" => "application/javascript", - "mov" => "video/quicktime", - "mpe" => "video/mpeg", - "mpeg" => "video/mpeg", - "mpg" => "video/mpeg", - "pbm" => "image/x-portable-bitmap", - "pdf" => "application/pdf", - "pgm" => "image/x-portable-graymap", - "png" => "image/png", - "pnm" => "image/x-portable-anymap", - "ppm" => "image/x-portable-pixmap", - "ppt" => "application/vnd.ms-powerpoint", - "ps" => "application/postscript", - "qt" => "video/quicktime", - "ras" => "image/x-cmu-raster", - "rb" => "text/plain", - "rd" => "text/plain", - "rtf" => "application/rtf", - "sgm" => "text/sgml", - "sgml" => "text/sgml", - "svg" => "image/svg+xml", - "tif" => "image/tiff", - "tiff" => "image/tiff", - "txt" => "text/plain", - "wasm" => "application/wasm", - "xbm" => "image/x-xbitmap", - "xhtml" => "text/html", - "xls" => "application/vnd.ms-excel", - "xml" => "text/xml", - "xpm" => "image/x-xpixmap", - "xwd" => "image/x-xwindowdump", - "zip" => "application/zip", - } - - ## - # Loads Apache-compatible mime.types in +file+. - - def load_mime_types(file) - # note: +file+ may be a "| command" for now; some people may - # rely on this, but currently we do not use this method by default. - File.open(file){ |io| - hash = Hash.new - io.each{ |line| - next if /^#/ =~ line - line.chomp! - mimetype, ext0 = line.split(/\s+/, 2) - next unless ext0 - next if ext0.empty? - ext0.split(/\s+/).each{ |ext| hash[ext] = mimetype } - } - hash - } - end - module_function :load_mime_types - - ## - # Returns the mime type of +filename+ from the list in +mime_tab+. If no - # mime type was found application/octet-stream is returned. - - def mime_type(filename, mime_tab) - suffix1 = (/\.(\w+)$/ =~ filename && $1.downcase) - suffix2 = (/\.(\w+)\.[\w\-]+$/ =~ filename && $1.downcase) - mime_tab[suffix1] || mime_tab[suffix2] || "application/octet-stream" - end - module_function :mime_type - - ## - # Parses an HTTP header +raw+ into a hash of header fields with an Array - # of values. - - def parse_header(raw) - header = Hash.new([].freeze) - field = nil - raw.each_line{|line| - case line - when /^([A-Za-z0-9!\#$%&'*+\-.^_`|~]+):\s*(.*?)\s*\z/om - field, value = $1, $2 - field.downcase! - header[field] = [] unless header.has_key?(field) - header[field] << value - when /^\s+(.*?)\s*\z/om - value = $1 - unless field - raise HTTPStatus::BadRequest, "bad header '#{line}'." - end - header[field][-1] << " " << value - else - raise HTTPStatus::BadRequest, "bad header '#{line}'." - end - } - header.each{|key, values| - values.each(&:strip!) - } - header - end - module_function :parse_header - - ## - # Splits a header value +str+ according to HTTP specification. - - def split_header_value(str) - str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",]+)+) - (?:,\s*|\Z)'xn).flatten - end - module_function :split_header_value - - ## - # Parses a Range header value +ranges_specifier+ - - def parse_range_header(ranges_specifier) - if /^bytes=(.*)/ =~ ranges_specifier - byte_range_set = split_header_value($1) - byte_range_set.collect{|range_spec| - case range_spec - when /^(\d+)-(\d+)/ then $1.to_i .. $2.to_i - when /^(\d+)-/ then $1.to_i .. -1 - when /^-(\d+)/ then -($1.to_i) .. -1 - else return nil - end - } - end - end - module_function :parse_range_header - - ## - # Parses q values in +value+ as used in Accept headers. - - def parse_qvalues(value) - tmp = [] - if value - parts = value.split(/,\s*/) - parts.each {|part| - if m = %r{^([^\s,]+?)(?:;\s*q=(\d+(?:\.\d+)?))?$}.match(part) - val = m[1] - q = (m[2] or 1).to_f - tmp.push([val, q]) - end - } - tmp = tmp.sort_by{|val, q| -q} - tmp.collect!{|val, q| val} - end - return tmp - end - module_function :parse_qvalues - - ## - # Removes quotes and escapes from +str+ - - def dequote(str) - ret = (/\A"(.*)"\Z/ =~ str) ? $1 : str.dup - ret.gsub!(/\\(.)/, "\\1") - ret - end - module_function :dequote - - ## - # Quotes and escapes quotes in +str+ - - def quote(str) - '"' << str.gsub(/[\\\"]/o, "\\\1") << '"' - end - module_function :quote - - ## - # Stores multipart form data. FormData objects are created when - # WEBrick::HTTPUtils.parse_form_data is called. - - class FormData < String - EmptyRawHeader = [].freeze # :nodoc: - EmptyHeader = {}.freeze # :nodoc: - - ## - # The name of the form data part - - attr_accessor :name - - ## - # The filename of the form data part - - attr_accessor :filename - - attr_accessor :next_data # :nodoc: - protected :next_data - - ## - # Creates a new FormData object. - # - # +args+ is an Array of form data entries. One FormData will be created - # for each entry. - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you - - def initialize(*args) - @name = @filename = @next_data = nil - if args.empty? - @raw_header = [] - @header = nil - super("") - else - @raw_header = EmptyRawHeader - @header = EmptyHeader - super(args.shift) - unless args.empty? - @next_data = self.class.new(*args) - end - end - end - - ## - # Retrieves the header at the first entry in +key+ - - def [](*key) - begin - @header[key[0].downcase].join(", ") - rescue StandardError, NameError - super - end - end - - ## - # Adds +str+ to this FormData which may be the body, a header or a - # header entry. - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you - - def <<(str) - if @header - super - elsif str == CRLF - @header = HTTPUtils::parse_header(@raw_header.join) - if cd = self['content-disposition'] - if /\s+name="(.*?)"/ =~ cd then @name = $1 end - if /\s+filename="(.*?)"/ =~ cd then @filename = $1 end - end - else - @raw_header << str - end - self - end - - ## - # Adds +data+ at the end of the chain of entries - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you. - - def append_data(data) - tmp = self - while tmp - unless tmp.next_data - tmp.next_data = data - break - end - tmp = tmp.next_data - end - self - end - - ## - # Yields each entry in this FormData - - def each_data - tmp = self - while tmp - next_data = tmp.next_data - yield(tmp) - tmp = next_data - end - end - - ## - # Returns all the FormData as an Array - - def list - ret = [] - each_data{|data| - ret << data.to_s - } - ret - end - - ## - # A FormData will behave like an Array - - alias :to_ary :list - - ## - # This FormData's body - - def to_s - String.new(self) - end - end - - ## - # Parses the query component of a URI in +str+ - - def parse_query(str) - query = Hash.new - if str - str.split(/[&;]/).each{|x| - next if x.empty? - key, val = x.split(/=/,2) - key = unescape_form(key) - val = unescape_form(val.to_s) - val = FormData.new(val) - val.name = key - if query.has_key?(key) - query[key].append_data(val) - next - end - query[key] = val - } - end - query - end - module_function :parse_query - - ## - # Parses form data in +io+ with the given +boundary+ - - def parse_form_data(io, boundary) - boundary_regexp = /\A--#{Regexp.quote(boundary)}(--)?#{CRLF}\z/ - form_data = Hash.new - return form_data unless io - data = nil - io.each_line{|line| - if boundary_regexp =~ line - if data - data.chop! - key = data.name - if form_data.has_key?(key) - form_data[key].append_data(data) - else - form_data[key] = data - end - end - data = FormData.new - next - else - if data - data << line - end - end - } - return form_data - end - module_function :parse_form_data - - ##### - - reserved = ';/?:@&=+$,' - num = '0123456789' - lowalpha = 'abcdefghijklmnopqrstuvwxyz' - upalpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - mark = '-_.!~*\'()' - unreserved = num + lowalpha + upalpha + mark - control = (0x0..0x1f).collect{|c| c.chr }.join + "\x7f" - space = " " - delims = '<>#%"' - unwise = '{}|\\^[]`' - nonascii = (0x80..0xff).collect{|c| c.chr }.join - - module_function - - # :stopdoc: - - def _make_regex(str) /([#{Regexp.escape(str)}])/n end - def _make_regex!(str) /([^#{Regexp.escape(str)}])/n end - def _escape(str, regex) - str = str.b - str.gsub!(regex) {"%%%02X" % $1.ord} - # %-escaped string should contain US-ASCII only - str.force_encoding(Encoding::US_ASCII) - end - def _unescape(str, regex) - str = str.b - str.gsub!(regex) {$1.hex.chr} - # encoding of %-unescaped string is unknown - str - end - - UNESCAPED = _make_regex(control+space+delims+unwise+nonascii) - UNESCAPED_FORM = _make_regex(reserved+control+delims+unwise+nonascii) - NONASCII = _make_regex(nonascii) - ESCAPED = /%([0-9a-fA-F]{2})/ - UNESCAPED_PCHAR = _make_regex!(unreserved+":@&=+$,") - - # :startdoc: - - ## - # Escapes HTTP reserved and unwise characters in +str+ - - def escape(str) - _escape(str, UNESCAPED) - end - - ## - # Unescapes HTTP reserved and unwise characters in +str+ - - def unescape(str) - _unescape(str, ESCAPED) - end - - ## - # Escapes form reserved characters in +str+ - - def escape_form(str) - ret = _escape(str, UNESCAPED_FORM) - ret.gsub!(/ /, "+") - ret - end - - ## - # Unescapes form reserved characters in +str+ - - def unescape_form(str) - _unescape(str.gsub(/\+/, " "), ESCAPED) - end - - ## - # Escapes path +str+ - - def escape_path(str) - result = "" - str.scan(%r{/([^/]*)}).each{|i| - result << "/" << _escape(i[0], UNESCAPED_PCHAR) - } - return result - end - - ## - # Escapes 8 bit characters in +str+ - - def escape8bit(str) - _escape(str, NONASCII) - end - end -end diff --git a/tool/lib/webrick/httpversion.rb b/tool/lib/webrick/httpversion.rb deleted file mode 100644 index 8a251944a2..0000000000 --- a/tool/lib/webrick/httpversion.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: false -#-- -# HTTPVersion.rb -- presentation of HTTP version -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpversion.rb,v 1.5 2002/09/21 12:23:37 gotoyuzo Exp $ - -module WEBrick - - ## - # Represents an HTTP protocol version - - class HTTPVersion - include Comparable - - ## - # The major protocol version number - - attr_accessor :major - - ## - # The minor protocol version number - - attr_accessor :minor - - ## - # Converts +version+ into an HTTPVersion - - def self.convert(version) - version.is_a?(self) ? version : new(version) - end - - ## - # Creates a new HTTPVersion from +version+. - - def initialize(version) - case version - when HTTPVersion - @major, @minor = version.major, version.minor - when String - if /^(\d+)\.(\d+)$/ =~ version - @major, @minor = $1.to_i, $2.to_i - end - end - if @major.nil? || @minor.nil? - raise ArgumentError, - format("cannot convert %s into %s", version.class, self.class) - end - end - - ## - # Compares this version with +other+ according to the HTTP specification - # rules. - - def <=>(other) - unless other.is_a?(self.class) - other = self.class.new(other) - end - if (ret = @major <=> other.major) == 0 - return @minor <=> other.minor - end - return ret - end - - ## - # The HTTP version as show in the HTTP request and response. For example, - # "1.1" - - def to_s - format("%d.%d", @major, @minor) - end - end -end diff --git a/tool/lib/webrick/log.rb b/tool/lib/webrick/log.rb deleted file mode 100644 index 2c1fdfe602..0000000000 --- a/tool/lib/webrick/log.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: false -#-- -# log.rb -- Log Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: log.rb,v 1.26 2002/10/06 17:06:10 gotoyuzo Exp $ - -module WEBrick - - ## - # A generic logging class - - class BasicLog - - # Fatal log level which indicates a server crash - - FATAL = 1 - - # Error log level which indicates a recoverable error - - ERROR = 2 - - # Warning log level which indicates a possible problem - - WARN = 3 - - # Information log level which indicates possibly useful information - - INFO = 4 - - # Debugging error level for messages used in server development or - # debugging - - DEBUG = 5 - - # log-level, messages above this level will be logged - attr_accessor :level - - ## - # Initializes a new logger for +log_file+ that outputs messages at +level+ - # or higher. +log_file+ can be a filename, an IO-like object that - # responds to #<< or nil which outputs to $stderr. - # - # If no level is given INFO is chosen by default - - def initialize(log_file=nil, level=nil) - @level = level || INFO - case log_file - when String - @log = File.open(log_file, "a+") - @log.sync = true - @opened = true - when NilClass - @log = $stderr - else - @log = log_file # requires "<<". (see BasicLog#log) - end - end - - ## - # Closes the logger (also closes the log device associated to the logger) - def close - @log.close if @opened - @log = nil - end - - ## - # Logs +data+ at +level+ if the given level is above the current log - # level. - - def log(level, data) - if @log && level <= @level - data += "\n" if /\n\Z/ !~ data - @log << data - end - end - - ## - # Synonym for log(INFO, obj.to_s) - def <<(obj) - log(INFO, obj.to_s) - end - - # Shortcut for logging a FATAL message - def fatal(msg) log(FATAL, "FATAL " << format(msg)); end - # Shortcut for logging an ERROR message - def error(msg) log(ERROR, "ERROR " << format(msg)); end - # Shortcut for logging a WARN message - def warn(msg) log(WARN, "WARN " << format(msg)); end - # Shortcut for logging an INFO message - def info(msg) log(INFO, "INFO " << format(msg)); end - # Shortcut for logging a DEBUG message - def debug(msg) log(DEBUG, "DEBUG " << format(msg)); end - - # Will the logger output FATAL messages? - def fatal?; @level >= FATAL; end - # Will the logger output ERROR messages? - def error?; @level >= ERROR; end - # Will the logger output WARN messages? - def warn?; @level >= WARN; end - # Will the logger output INFO messages? - def info?; @level >= INFO; end - # Will the logger output DEBUG messages? - def debug?; @level >= DEBUG; end - - private - - ## - # Formats +arg+ for the logger - # - # * If +arg+ is an Exception, it will format the error message and - # the back trace. - # * If +arg+ responds to #to_str, it will return it. - # * Otherwise it will return +arg+.inspect. - def format(arg) - if arg.is_a?(Exception) - "#{arg.class}: #{AccessLog.escape(arg.message)}\n\t" << - arg.backtrace.join("\n\t") << "\n" - elsif arg.respond_to?(:to_str) - AccessLog.escape(arg.to_str) - else - arg.inspect - end - end - end - - ## - # A logging class that prepends a timestamp to each message. - - class Log < BasicLog - # Format of the timestamp which is applied to each logged line. The - # default is <tt>"[%Y-%m-%d %H:%M:%S]"</tt> - attr_accessor :time_format - - ## - # Same as BasicLog#initialize - # - # You can set the timestamp format through #time_format - def initialize(log_file=nil, level=nil) - super(log_file, level) - @time_format = "[%Y-%m-%d %H:%M:%S]" - end - - ## - # Same as BasicLog#log - def log(level, data) - tmp = Time.now.strftime(@time_format) - tmp << " " << data - super(level, tmp) - end - end -end diff --git a/tool/lib/webrick/server.rb b/tool/lib/webrick/server.rb deleted file mode 100644 index fd6b7a61b5..0000000000 --- a/tool/lib/webrick/server.rb +++ /dev/null @@ -1,381 +0,0 @@ -# frozen_string_literal: false -# -# server.rb -- GenericServer Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: server.rb,v 1.62 2003/07/22 19:20:43 gotoyuzo Exp $ - -require 'socket' -require_relative 'config' -require_relative 'log' - -module WEBrick - - ## - # Server error exception - - class ServerError < StandardError; end - - ## - # Base server class - - class SimpleServer - - ## - # A SimpleServer only yields when you start it - - def SimpleServer.start - yield - end - end - - ## - # A generic module for daemonizing a process - - class Daemon - - ## - # Performs the standard operations for daemonizing a process. Runs a - # block, if given. - - def Daemon.start - Process.daemon - File.umask(0) - yield if block_given? - end - end - - ## - # Base TCP server class. You must subclass GenericServer and provide a #run - # method. - - class GenericServer - - ## - # The server status. One of :Stop, :Running or :Shutdown - - attr_reader :status - - ## - # The server configuration - - attr_reader :config - - ## - # The server logger. This is independent from the HTTP access log. - - attr_reader :logger - - ## - # Tokens control the number of outstanding clients. The - # <code>:MaxClients</code> configuration sets this. - - attr_reader :tokens - - ## - # Sockets listening for connections. - - attr_reader :listeners - - ## - # Creates a new generic server from +config+. The default configuration - # comes from +default+. - - def initialize(config={}, default=Config::General) - @config = default.dup.update(config) - @status = :Stop - @config[:Logger] ||= Log::new - @logger = @config[:Logger] - - @tokens = Thread::SizedQueue.new(@config[:MaxClients]) - @config[:MaxClients].times{ @tokens.push(nil) } - - webrickv = WEBrick::VERSION - rubyv = "#{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]" - @logger.info("WEBrick #{webrickv}") - @logger.info("ruby #{rubyv}") - - @listeners = [] - @shutdown_pipe = nil - unless @config[:DoNotListen] - raise ArgumentError, "Port must an integer" unless @config[:Port].to_s == @config[:Port].to_i.to_s - - @config[:Port] = @config[:Port].to_i - if @config[:Listen] - warn(":Listen option is deprecated; use GenericServer#listen", uplevel: 1) - end - listen(@config[:BindAddress], @config[:Port]) - if @config[:Port] == 0 - @config[:Port] = @listeners[0].addr[1] - end - end - end - - ## - # Retrieves +key+ from the configuration - - def [](key) - @config[key] - end - - ## - # Adds listeners from +address+ and +port+ to the server. See - # WEBrick::Utils::create_listeners for details. - - def listen(address, port) - @listeners += Utils::create_listeners(address, port) - end - - ## - # Starts the server and runs the +block+ for each connection. This method - # does not return until the server is stopped from a signal handler or - # another thread using #stop or #shutdown. - # - # If the block raises a subclass of StandardError the exception is logged - # and ignored. If an IOError or Errno::EBADF exception is raised the - # exception is ignored. If an Exception subclass is raised the exception - # is logged and re-raised which stops the server. - # - # To completely shut down a server call #shutdown from ensure: - # - # server = WEBrick::GenericServer.new - # # or WEBrick::HTTPServer.new - # - # begin - # server.start - # ensure - # server.shutdown - # end - - def start(&block) - raise ServerError, "already started." if @status != :Stop - server_type = @config[:ServerType] || SimpleServer - - setup_shutdown_pipe - - server_type.start{ - @logger.info \ - "#{self.class}#start: pid=#{$$} port=#{@config[:Port]}" - @status = :Running - call_callback(:StartCallback) - - shutdown_pipe = @shutdown_pipe - - thgroup = ThreadGroup.new - begin - while @status == :Running - begin - sp = shutdown_pipe[0] - if svrs = IO.select([sp, *@listeners]) - if svrs[0].include? sp - # swallow shutdown pipe - buf = String.new - nil while String === - sp.read_nonblock([sp.nread, 8].max, buf, exception: false) - break - end - svrs[0].each{|svr| - @tokens.pop # blocks while no token is there. - if sock = accept_client(svr) - unless config[:DoNotReverseLookup].nil? - sock.do_not_reverse_lookup = !!config[:DoNotReverseLookup] - end - th = start_thread(sock, &block) - th[:WEBrickThread] = true - thgroup.add(th) - else - @tokens.push(nil) - end - } - end - rescue Errno::EBADF, Errno::ENOTSOCK, IOError => ex - # if the listening socket was closed in GenericServer#shutdown, - # IO::select raise it. - rescue StandardError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - rescue Exception => ex - @logger.fatal ex - raise - end - end - ensure - cleanup_shutdown_pipe(shutdown_pipe) - cleanup_listener - @status = :Shutdown - @logger.info "going to shutdown ..." - thgroup.list.each{|th| th.join if th[:WEBrickThread] } - call_callback(:StopCallback) - @logger.info "#{self.class}#start done." - @status = :Stop - end - } - end - - ## - # Stops the server from accepting new connections. - - def stop - if @status == :Running - @status = :Shutdown - end - - alarm_shutdown_pipe {|f| f.write_nonblock("\0")} - end - - ## - # Shuts down the server and all listening sockets. New listeners must be - # provided to restart the server. - - def shutdown - stop - - alarm_shutdown_pipe(&:close) - end - - ## - # You must subclass GenericServer and implement \#run which accepts a TCP - # client socket - - def run(sock) - @logger.fatal "run() must be provided by user." - end - - private - - # :stopdoc: - - ## - # Accepts a TCP client socket from the TCP server socket +svr+ and returns - # the client socket. - - def accept_client(svr) - case sock = svr.to_io.accept_nonblock(exception: false) - when :wait_readable - nil - else - if svr.respond_to?(:start_immediately) - sock = OpenSSL::SSL::SSLSocket.new(sock, ssl_context) - sock.sync_close = true - # we cannot do OpenSSL::SSL::SSLSocket#accept here because - # a slow client can prevent us from accepting connections - # from other clients - end - sock - end - rescue Errno::ECONNRESET, Errno::ECONNABORTED, - Errno::EPROTO, Errno::EINVAL - nil - rescue StandardError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - nil - end - - ## - # Starts a server thread for the client socket +sock+ that runs the given - # +block+. - # - # Sets the socket to the <code>:WEBrickSocket</code> thread local variable - # in the thread. - # - # If any errors occur in the block they are logged and handled. - - def start_thread(sock, &block) - Thread.start{ - begin - Thread.current[:WEBrickSocket] = sock - begin - addr = sock.peeraddr - @logger.debug "accept: #{addr[3]}:#{addr[1]}" - rescue SocketError - @logger.debug "accept: <address unknown>" - raise - end - if sock.respond_to?(:sync_close=) && @config[:SSLStartImmediately] - WEBrick::Utils.timeout(@config[:RequestTimeout]) do - begin - sock.accept # OpenSSL::SSL::SSLSocket#accept - rescue Errno::ECONNRESET, Errno::ECONNABORTED, - Errno::EPROTO, Errno::EINVAL - Thread.exit - end - end - end - call_callback(:AcceptCallback, sock) - block ? block.call(sock) : run(sock) - rescue Errno::ENOTCONN - @logger.debug "Errno::ENOTCONN raised" - rescue ServerError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - rescue Exception => ex - @logger.error ex - ensure - @tokens.push(nil) - Thread.current[:WEBrickSocket] = nil - if addr - @logger.debug "close: #{addr[3]}:#{addr[1]}" - else - @logger.debug "close: <address unknown>" - end - sock.close - end - } - end - - ## - # Calls the callback +callback_name+ from the configuration with +args+ - - def call_callback(callback_name, *args) - @config[callback_name]&.call(*args) - end - - def setup_shutdown_pipe - return @shutdown_pipe ||= IO.pipe - end - - def cleanup_shutdown_pipe(shutdown_pipe) - @shutdown_pipe = nil - shutdown_pipe&.each(&:close) - end - - def alarm_shutdown_pipe - _, pipe = @shutdown_pipe # another thread may modify @shutdown_pipe. - if pipe - if !pipe.closed? - begin - yield pipe - rescue IOError # closed by another thread. - end - end - end - end - - def cleanup_listener - @listeners.each{|s| - if @logger.debug? - addr = s.addr - @logger.debug("close TCPSocket(#{addr[2]}, #{addr[1]})") - end - begin - s.shutdown - rescue Errno::ENOTCONN - # when `Errno::ENOTCONN: Socket is not connected' on some platforms, - # call #close instead of #shutdown. - # (ignore @config[:ShutdownSocketWithoutClose]) - s.close - else - unless @config[:ShutdownSocketWithoutClose] - s.close - end - end - } - @listeners.clear - end - end # end of GenericServer -end diff --git a/tool/lib/webrick/ssl.rb b/tool/lib/webrick/ssl.rb deleted file mode 100644 index e448095a12..0000000000 --- a/tool/lib/webrick/ssl.rb +++ /dev/null @@ -1,215 +0,0 @@ -# frozen_string_literal: false -# -# ssl.rb -- SSL/TLS enhancement for GenericServer -# -# Copyright (c) 2003 GOTOU Yuuzou All rights reserved. -# -# $Id$ - -require 'webrick' -require 'openssl' - -module WEBrick - module Config - svrsoft = General[:ServerSoftware] - osslv = ::OpenSSL::OPENSSL_VERSION.split[1] - - ## - # Default SSL server configuration. - # - # WEBrick can automatically create a self-signed certificate if - # <code>:SSLCertName</code> is set. For more information on the various - # SSL options see OpenSSL::SSL::SSLContext. - # - # :ServerSoftware :: - # The server software name used in the Server: header. - # :SSLEnable :: false, - # Enable SSL for this server. Defaults to false. - # :SSLCertificate :: - # The SSL certificate for the server. - # :SSLPrivateKey :: - # The SSL private key for the server certificate. - # :SSLClientCA :: nil, - # Array of certificates that will be sent to the client. - # :SSLExtraChainCert :: nil, - # Array of certificates that will be added to the certificate chain - # :SSLCACertificateFile :: nil, - # Path to a CA certificate file - # :SSLCACertificatePath :: nil, - # Path to a directory containing CA certificates - # :SSLCertificateStore :: nil, - # OpenSSL::X509::Store used for certificate validation of the client - # :SSLTmpDhCallback :: nil, - # Callback invoked when DH parameters are required. - # :SSLVerifyClient :: - # Sets whether the client is verified. This defaults to VERIFY_NONE - # which is typical for an HTTPS server. - # :SSLVerifyDepth :: - # Number of CA certificates to walk when verifying a certificate chain - # :SSLVerifyCallback :: - # Custom certificate verification callback - # :SSLServerNameCallback:: - # Custom servername indication callback - # :SSLTimeout :: - # Maximum session lifetime - # :SSLOptions :: - # Various SSL options - # :SSLCiphers :: - # Ciphers to be used - # :SSLStartImmediately :: - # Immediately start SSL upon connection? Defaults to true - # :SSLCertName :: - # SSL certificate name. Must be set to enable automatic certificate - # creation. - # :SSLCertComment :: - # Comment used during automatic certificate creation. - - SSL = { - :ServerSoftware => "#{svrsoft} OpenSSL/#{osslv}", - :SSLEnable => false, - :SSLCertificate => nil, - :SSLPrivateKey => nil, - :SSLClientCA => nil, - :SSLExtraChainCert => nil, - :SSLCACertificateFile => nil, - :SSLCACertificatePath => nil, - :SSLCertificateStore => nil, - :SSLTmpDhCallback => nil, - :SSLVerifyClient => ::OpenSSL::SSL::VERIFY_NONE, - :SSLVerifyDepth => nil, - :SSLVerifyCallback => nil, # custom verification - :SSLTimeout => nil, - :SSLOptions => nil, - :SSLCiphers => nil, - :SSLStartImmediately => true, - # Must specify if you use auto generated certificate. - :SSLCertName => nil, - :SSLCertComment => "Generated by Ruby/OpenSSL" - } - General.update(SSL) - end - - module Utils - ## - # Creates a self-signed certificate with the given number of +bits+, - # the issuer +cn+ and a +comment+ to be stored in the certificate. - - def create_self_signed_cert(bits, cn, comment) - rsa = OpenSSL::PKey::RSA.new(bits){|p, n| - case p - when 0; $stderr.putc "." # BN_generate_prime - when 1; $stderr.putc "+" # BN_generate_prime - when 2; $stderr.putc "*" # searching good prime, - # n = #of try, - # but also data from BN_generate_prime - when 3; $stderr.putc "\n" # found good prime, n==0 - p, n==1 - q, - # but also data from BN_generate_prime - else; $stderr.putc "*" # BN_generate_prime - end - } - cert = OpenSSL::X509::Certificate.new - cert.version = 2 - cert.serial = 1 - name = (cn.kind_of? String) ? OpenSSL::X509::Name.parse(cn) - : OpenSSL::X509::Name.new(cn) - cert.subject = name - cert.issuer = name - cert.not_before = Time.now - cert.not_after = Time.now + (365*24*60*60) - cert.public_key = rsa.public_key - - ef = OpenSSL::X509::ExtensionFactory.new(nil,cert) - ef.issuer_certificate = cert - cert.extensions = [ - ef.create_extension("basicConstraints","CA:FALSE"), - ef.create_extension("keyUsage", "keyEncipherment, digitalSignature, keyAgreement, dataEncipherment"), - ef.create_extension("subjectKeyIdentifier", "hash"), - ef.create_extension("extendedKeyUsage", "serverAuth"), - ef.create_extension("nsComment", comment), - ] - aki = ef.create_extension("authorityKeyIdentifier", - "keyid:always,issuer:always") - cert.add_extension(aki) - cert.sign(rsa, "SHA256") - - return [ cert, rsa ] - end - module_function :create_self_signed_cert - end - - ## - #-- - # Updates WEBrick::GenericServer with SSL functionality - - class GenericServer - - ## - # SSL context for the server when run in SSL mode - - def ssl_context # :nodoc: - @ssl_context ||= begin - if @config[:SSLEnable] - ssl_context = setup_ssl_context(@config) - @logger.info("\n" + @config[:SSLCertificate].to_text) - ssl_context - end - end - end - - undef listen - - ## - # Updates +listen+ to enable SSL when the SSL configuration is active. - - def listen(address, port) # :nodoc: - listeners = Utils::create_listeners(address, port) - if @config[:SSLEnable] - listeners.collect!{|svr| - ssvr = ::OpenSSL::SSL::SSLServer.new(svr, ssl_context) - ssvr.start_immediately = @config[:SSLStartImmediately] - ssvr - } - end - @listeners += listeners - setup_shutdown_pipe - end - - ## - # Sets up an SSL context for +config+ - - def setup_ssl_context(config) # :nodoc: - unless config[:SSLCertificate] - cn = config[:SSLCertName] - comment = config[:SSLCertComment] - cert, key = Utils::create_self_signed_cert(2048, cn, comment) - config[:SSLCertificate] = cert - config[:SSLPrivateKey] = key - end - ctx = OpenSSL::SSL::SSLContext.new - ctx.key = config[:SSLPrivateKey] - ctx.cert = config[:SSLCertificate] - ctx.client_ca = config[:SSLClientCA] - ctx.extra_chain_cert = config[:SSLExtraChainCert] - ctx.ca_file = config[:SSLCACertificateFile] - ctx.ca_path = config[:SSLCACertificatePath] - ctx.cert_store = config[:SSLCertificateStore] - ctx.tmp_dh_callback = config[:SSLTmpDhCallback] - ctx.verify_mode = config[:SSLVerifyClient] - ctx.verify_depth = config[:SSLVerifyDepth] - ctx.verify_callback = config[:SSLVerifyCallback] - ctx.servername_cb = config[:SSLServerNameCallback] || proc { |args| ssl_servername_callback(*args) } - ctx.timeout = config[:SSLTimeout] - ctx.options = config[:SSLOptions] - ctx.ciphers = config[:SSLCiphers] - ctx - end - - ## - # ServerNameIndication callback - - def ssl_servername_callback(sslsocket, hostname = nil) - # default - end - - end -end diff --git a/tool/lib/webrick/utils.rb b/tool/lib/webrick/utils.rb deleted file mode 100644 index a96d6f03fd..0000000000 --- a/tool/lib/webrick/utils.rb +++ /dev/null @@ -1,265 +0,0 @@ -# frozen_string_literal: false -# -# utils.rb -- Miscellaneous utilities -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: utils.rb,v 1.10 2003/02/16 22:22:54 gotoyuzo Exp $ - -require 'socket' -require 'io/nonblock' -require 'etc' - -module WEBrick - module Utils - ## - # Sets IO operations on +io+ to be non-blocking - def set_non_blocking(io) - io.nonblock = true if io.respond_to?(:nonblock=) - end - module_function :set_non_blocking - - ## - # Sets the close on exec flag for +io+ - def set_close_on_exec(io) - io.close_on_exec = true if io.respond_to?(:close_on_exec=) - end - module_function :set_close_on_exec - - ## - # Changes the process's uid and gid to the ones of +user+ - def su(user) - if pw = Etc.getpwnam(user) - Process::initgroups(user, pw.gid) - Process::Sys::setgid(pw.gid) - Process::Sys::setuid(pw.uid) - else - warn("WEBrick::Utils::su doesn't work on this platform", uplevel: 1) - end - end - module_function :su - - ## - # The server hostname - def getservername - Socket::gethostname - end - module_function :getservername - - ## - # Creates TCP server sockets bound to +address+:+port+ and returns them. - # - # It will create IPV4 and IPV6 sockets on all interfaces. - def create_listeners(address, port) - unless port - raise ArgumentError, "must specify port" - end - sockets = Socket.tcp_server_sockets(address, port) - sockets = sockets.map {|s| - s.autoclose = false - ts = TCPServer.for_fd(s.fileno) - s.close - ts - } - return sockets - end - module_function :create_listeners - - ## - # Characters used to generate random strings - RAND_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + - "0123456789" + - "abcdefghijklmnopqrstuvwxyz" - - ## - # Generates a random string of length +len+ - def random_string(len) - rand_max = RAND_CHARS.bytesize - ret = "" - len.times{ ret << RAND_CHARS[rand(rand_max)] } - ret - end - module_function :random_string - - ########### - - require "timeout" - require "singleton" - - ## - # Class used to manage timeout handlers across multiple threads. - # - # Timeout handlers should be managed by using the class methods which are - # synchronized. - # - # id = TimeoutHandler.register(10, Timeout::Error) - # begin - # sleep 20 - # puts 'foo' - # ensure - # TimeoutHandler.cancel(id) - # end - # - # will raise Timeout::Error - # - # id = TimeoutHandler.register(10, Timeout::Error) - # begin - # sleep 5 - # puts 'foo' - # ensure - # TimeoutHandler.cancel(id) - # end - # - # will print 'foo' - # - class TimeoutHandler - include Singleton - - ## - # Mutex used to synchronize access across threads - TimeoutMutex = Thread::Mutex.new # :nodoc: - - ## - # Registers a new timeout handler - # - # +time+:: Timeout in seconds - # +exception+:: Exception to raise when timeout elapsed - def TimeoutHandler.register(seconds, exception) - at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds - instance.register(Thread.current, at, exception) - end - - ## - # Cancels the timeout handler +id+ - def TimeoutHandler.cancel(id) - instance.cancel(Thread.current, id) - end - - def self.terminate - instance.terminate - end - - ## - # Creates a new TimeoutHandler. You should use ::register and ::cancel - # instead of creating the timeout handler directly. - def initialize - TimeoutMutex.synchronize{ - @timeout_info = Hash.new - } - @queue = Thread::Queue.new - @watcher = nil - end - - # :nodoc: - private \ - def watch - to_interrupt = [] - while true - now = Process.clock_gettime(Process::CLOCK_MONOTONIC) - wakeup = nil - to_interrupt.clear - TimeoutMutex.synchronize{ - @timeout_info.each {|thread, ary| - next unless ary - ary.each{|info| - time, exception = *info - if time < now - to_interrupt.push [thread, info.object_id, exception] - elsif !wakeup || time < wakeup - wakeup = time - end - } - } - } - to_interrupt.each {|arg| interrupt(*arg)} - if !wakeup - @queue.pop - elsif (wakeup -= now) > 0 - begin - (th = Thread.start {@queue.pop}).join(wakeup) - ensure - th&.kill&.join - end - end - @queue.clear - end - end - - # :nodoc: - private \ - def watcher - (w = @watcher)&.alive? and return w # usual case - TimeoutMutex.synchronize{ - (w = @watcher)&.alive? and next w # pathological check - @watcher = Thread.start(&method(:watch)) - } - end - - ## - # Interrupts the timeout handler +id+ and raises +exception+ - def interrupt(thread, id, exception) - if cancel(thread, id) && thread.alive? - thread.raise(exception, "execution timeout") - end - end - - ## - # Registers a new timeout handler - # - # +time+:: Timeout in seconds - # +exception+:: Exception to raise when timeout elapsed - def register(thread, time, exception) - info = nil - TimeoutMutex.synchronize{ - (@timeout_info[thread] ||= []) << (info = [time, exception]) - } - @queue.push nil - watcher - return info.object_id - end - - ## - # Cancels the timeout handler +id+ - def cancel(thread, id) - TimeoutMutex.synchronize{ - if ary = @timeout_info[thread] - ary.delete_if{|info| info.object_id == id } - if ary.empty? - @timeout_info.delete(thread) - end - return true - end - return false - } - end - - ## - def terminate - TimeoutMutex.synchronize{ - @timeout_info.clear - @watcher&.kill&.join - } - end - end - - ## - # Executes the passed block and raises +exception+ if execution takes more - # than +seconds+. - # - # If +seconds+ is zero or nil, simply executes the block - def timeout(seconds, exception=Timeout::Error) - return yield if seconds.nil? or seconds.zero? - # raise ThreadError, "timeout within critical session" if Thread.critical - id = TimeoutHandler.register(seconds, exception) - begin - yield(seconds) - ensure - TimeoutHandler.cancel(id) - end - end - module_function :timeout - end -end diff --git a/tool/lib/webrick/version.rb b/tool/lib/webrick/version.rb deleted file mode 100644 index b62988bdbb..0000000000 --- a/tool/lib/webrick/version.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: false -#-- -# version.rb -- version and release date -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: version.rb,v 1.74 2003/07/22 19:20:43 gotoyuzo Exp $ - -module WEBrick - - ## - # The WEBrick version - - VERSION = "1.7.0" -end diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md index c4a0f28f5b..f71118a913 100644 --- a/tool/lrama/NEWS.md +++ b/tool/lrama/NEWS.md @@ -1,14 +1,813 @@ # NEWS for Lrama -## Lrama 0.6.1 (2024-01-13) +## Lrama 0.7.1 (2025-12-24) + +### Optimize IELR + +Optimized performance to a level that allows for IELR testing in practical applications. + +https://github.com/ruby/lrama/pull/595 +https://github.com/ruby/lrama/pull/605 +https://github.com/ruby/lrama/pull/685 +https://github.com/ruby/lrama/pull/700 + +### Introduce counterexamples timeout + +Counterexample searches can sometimes take a long time, so we've added a timeout to abort the process after a set period. The current limits are: + +* 10 seconds per case +* 120 seconds total (cumulative) + +Please note that these are hard-coded and cannot be modified by the user in the current version. + +https://github.com/ruby/lrama/pull/623 + +### Optimize Counterexamples + +Optimized counterexample search performance. + +https://github.com/ruby/lrama/pull/607 +https://github.com/ruby/lrama/pull/610 +https://github.com/ruby/lrama/pull/614 +https://github.com/ruby/lrama/pull/622 +https://github.com/ruby/lrama/pull/627 +https://github.com/ruby/lrama/pull/629 +https://github.com/ruby/lrama/pull/659 + +### Support parameterized rule's arguments include inline + +Allow to use %inline directive with Parameterized rules arguments. When an inline rule is used as an argument to a Parameterized rule, it expands inline at the point of use. + +```yacc +%rule %inline op : '+' + | '-' + ; +%% +operation : op? + ; +``` + +This expands to: + +```yacc +operation : /* empty */ + | '+' + | '-' + ; +``` + +https://github.com/ruby/lrama/pull/637 + +### Render conflicts of each state on output file + +Added token information for conflicts in the output file. +These information are useful when a state has many actions. + +``` +State 1 + + 4 class: keyword_class • tSTRING "end" + 5 $@1: ε • [tSTRING] + 7 class: keyword_class • $@1 tSTRING '!' "end" $@2 + 8 $@3: ε • [tSTRING] + 10 class: keyword_class • $@3 tSTRING '?' "end" $@4 + + Conflict on tSTRING. shift/reduce($@1) + Conflict on tSTRING. shift/reduce($@3) + Conflict on tSTRING. reduce($@1)/reduce($@3) + + tSTRING shift, and go to state 6 + + tSTRING reduce using rule 5 ($@1) + tSTRING reduce using rule 8 ($@3) + + $@1 go to state 7 + $@3 go to state 8 +``` + +https://github.com/ruby/lrama/pull/541 + +### Render the origin of conflicted tokens on output file + +For example, for the grammar file like below: + +``` +%% + +program: expr + ; + +expr: expr '+' expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where `"plus"` (`'+'`) look ahead tokens come from: + +``` +State 6 + + 2 expr: expr • "plus" expr + 2 | expr "plus" expr • ["end of file", "plus"] + + Conflict on "plus". shift/reduce(expr) + "plus" comes from state 0 goto by expr + "plus" comes from state 5 goto by expr +``` + +state 0 and state 5 look like below: + +``` +State 0 + + 0 $accept: • program "end of file" + 1 program: • expr + 2 expr: • expr "plus" expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + program go to state 2 + expr go to state 3 + +State 5 + + 2 expr: • expr "plus" expr + 2 | expr "plus" • expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + expr go to state 6 +``` + +https://github.com/ruby/lrama/pull/726 + +### Render precedences usage information on output file -### Nested parameterizing rules +For example, for the grammar file like below: -Allow to pass an instantiated rule to other parameterizing rules. +``` +%left tPLUS +%right tUPLUS + +%% + +program: expr ; + +expr: tUPLUS expr + | expr tPLUS expr + | tNUMBER + ; +%% +``` + +Lrama generates output file which describes where these precedences are used to resolve conflicts: + +``` +Precedences + precedence on "unary+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + precedence on "+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + state 8. Conflict between reduce by "expr -> expr tPLUS expr" and shift "+" resolved as reduce (%left "+"). ``` + +https://github.com/ruby/lrama/pull/741 + +### Add support for reporting Rule Usage Frequency + +Support to report rule usage frequency statistics for analyzing grammar characteristics. +Run `exe/lrama --report=rules` to show how frequently each terminal and non-terminal symbol is used in the grammar rules. + +```console +$ exe/lrama --report=rules sample/calc.y +Rule Usage Frequency + 0 tSTRING (4 times) + 1 keyword_class (3 times) + 2 keyword_end (3 times) + 3 '+' (2 times) + 4 string (2 times) + 5 string_1 (2 times) + 6 '!' (1 times) + 7 '-' (1 times) + 8 '?' (1 times) + 9 EOI (1 times) + 10 class (1 times) + 11 program (1 times) + 12 string_2 (1 times) + 13 strings_1 (1 times) + 14 strings_2 (1 times) + 15 tNUMBER (1 times) +``` + +This feature provides insights into the language characteristics by showing: +- Which symbols are most frequently used in the grammar +- The distribution of terminal and non-terminal usage +- Potential areas for grammar optimization or refactoring + +The frequency statistics help developers understand the grammar structure and can be useful for: +- Grammar complexity analysis +- Performance optimization hints +- Language design decisions +- Documentation and educational purposes + +https://github.com/ruby/lrama/pull/677 + +### Render Split States information on output file + +For example, for the grammar file like below: + +``` +%token a +%token b +%token c +%define lr.type ielr + +%precedence tLOWEST +%precedence a +%precedence tHIGHEST + +%% + +S: a A B a + | b A B b + ; + +A: a C D E + ; + +B: c + | // empty + ; + +C: D + ; + +D: a + ; + +E: a + | %prec tHIGHEST // empty + ; + +%% +``` + +Lrama generates output file which describes where which new states are created when IELR is enabled: + +``` +Split States + + State 19 is split from state 4 + State 20 is split from state 9 + State 21 is split from state 14 +``` + +https://github.com/ruby/lrama/pull/624 + +### Add ioption support to the Standard library + +Support `ioption` (inline option) rule, which is expanded inline without creating intermediate rules. + +Unlike the regular `option` rule that generates a separate rule, `ioption` directly expands at the point of use: + +```yacc +program: ioption(number) expr + +// Expanded inline to: + +program: expr + | number expr +``` + +This differs from the regular `option` which would generate: + +```yacc +program: option(number) expr + +// Expanded to: + +program: option_number expr +option_number: %empty + | number +``` + +The `ioption` rule provides more compact grammar generation by avoiding intermediate rule creation, which can be beneficial for reducing the parser's rule count and potentially improving performance. + +This feature is inspired by Menhir's standard library and maintains compatibility with [Menhir's `ioption` behavior](https://github.com/let-def/menhir/blob/e8ba7bef219acd355798072c42abbd11335ecf09/src/standard.mly#L33-L41). + +https://github.com/ruby/lrama/pull/666 + +### Syntax Diagrams + +Lrama provides an API for generating HTML syntax diagrams. These visual diagrams are highly useful as grammar development tools and can also serve as a form of automatic self-documentation. + + + +If you use syntax diagrams, you add `--diagram` option. + +```console +$ exe/lrama --diagram sample.y +``` + +https://github.com/ruby/lrama/pull/523 + +### Support `--profile` option + +You can profile parser generation process without modification for Lrama source code. +Currently `--profile=call-stack` and `--profile=memory` are supported. + +```console +$ exe/lrama --profile=call-stack sample/calc.y +``` + +Then "tmp/stackprof-cpu-myapp.dump" is generated. + +https://github.com/ruby/lrama/pull/525 + +### Add support Start-Symbol: `%start` + +https://github.com/ruby/lrama/pull/576 + +## Lrama 0.7.0 (2025-01-21) + +### [EXPERIMENTAL] Support the generation of the IELR(1) parser described in this paper + +Support the generation of the IELR(1) parser described in this paper. +https://www.sciencedirect.com/science/article/pii/S0167642309001191 + +If you use IELR(1) parser, you can write the following directive in your grammar file. + +```yacc +%define lr.type ielr +``` + +But, currently IELR(1) parser is experimental feature. If you find any bugs, please report it to us. Thank you. + +### Support `-t` option as same as `--debug` option + +Support to `-t` option as same as `--debug` option. +These options align with Bison behavior. So same as `--debug` option. + +### Trace only explicit rules + +Support to trace only explicit rules. +If you use `--trace=rules` option, it shows include mid-rule actions. If you want to show only explicit rules, you can use `--trace=only-explicit-rules` option. + +Example: + +```yacc +%{ +%} +%union { + int i; +} +%token <i> number +%type <i> program +%% +program : number { printf("%d", $1); } number { $$ = $1 + $3; } + ; +%% +``` + +Result of `--trace=rules`: + +```console +$ exe/lrama --trace=rules sample.y +Grammar rules: +$accept -> program YYEOF +$@1 -> ε +program -> number $@1 number +``` + +Result of `--trace=only-explicit-rules`: + +```console +$ exe/lrama --trace=explicit-rules sample.y +Grammar rules: +$accept -> program YYEOF +program -> number number +``` + +## Lrama 0.6.11 (2024-12-23) + +### Add support for %type declarations using %nterm in Nonterminal Symbols + +Allow to use `%nterm` in Nonterminal Symbols for `%type` declarations. + +```yacc +%nterm <type> nonterminal… +``` + +This directive is also supported for compatibility with Bison, and only non-terminal symbols are allowed. In other words, definitions like the following will result in an error: + +```yacc +%{ +// Prologue +%} + +%token EOI 0 "EOI" +%nterm EOI + +%% + +program: /* empty */ + ; +``` + +It show an error message like the following: + +```command +❯ exe/lrama nterm.y +nterm.y:6:7: symbol EOI redeclared as a nonterminal +%nterm EOI + ^^^ +``` + +## Lrama 0.6.10 (2024-09-11) + +### Aliased Named References for actions of RHS in Parameterizing rules + +Allow to use aliased named references for actions of RHS in Parameterizing rules. + +```yacc +%rule sum(X, Y): X[summand] '+' Y[addend] { $$ = $summand + $addend } + ; +``` + +https://github.com/ruby/lrama/pull/410 + + +### Named References for actions of RHS in Parameterizing rules caller side + +Allow to use named references for actions of RHS in Parameterizing rules caller side. + +```yacc +opt_nl: '\n'?[nl] <str> { $$ = $nl; } + ; +``` + +https://github.com/ruby/lrama/pull/414 + +### Widen the definable position of Parameterizing rules + +Allow to define Parameterizing rules in the middle of the grammar. + +```yacc +%rule defined_option(X): /* empty */ + | X + ; + +%% + +program : defined_option(number) <i> + | defined_list(number) <i> + ; + +%rule defined_list(X): /* empty */ /* <--- here */ + | defined_list(X) number + ; +``` + +https://github.com/ruby/lrama/pull/420 + +### Report unused terminal symbols + +Support to report unused terminal symbols. +Run `exe/lrama --report=terms` to show unused terminal symbols. + +```console +$ exe/lrama --report=terms sample/calc.y + 11 Unused Terms + 0 YYerror + 1 YYUNDEF + 2 '\\\\' + 3 '\\13' + 4 keyword_class2 + 5 tNUMBER + 6 tPLUS + 7 tMINUS + 8 tEQ + 9 tEQEQ + 10 '>' +``` +https://github.com/ruby/lrama/pull/439 + +### Report unused rules + +Support to report unused rules. +Run `exe/lrama --report=rules` to show unused rules. + +```console +$ exe/lrama --report=rules sample/calc.y + 3 Unused Rules + 0 unused_option + 1 unused_list + 2 unused_nonempty_list +``` + +https://github.com/ruby/lrama/pull/441 + +### Ensure compatibility with Bison for `%locations` directive + +Support `%locations` directive to ensure compatibility with Bison. +Change to `%locations` directive not set by default. + +https://github.com/ruby/lrama/pull/446 + +### Diagnostics report for parameterized rules redefine + +Support to warning redefined parameterized rules. +Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterized rules. + +```console +$ exe/lrama -W sample/calc.y +parameterized rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) +``` + +https://github.com/ruby/lrama/pull/448 + +### Support `-v` and `--verbose` option + +Support to `-v` and `--verbose` option. +These options align with Bison behavior. So same as '--report=state' option. + +https://github.com/ruby/lrama/pull/457 + +## Lrama 0.6.9 (2024-05-02) + +### Callee side tag specification of Parameterizing rules + +Allow to specify tag on callee side of Parameterizing rules. + +```yacc +%union { + int i; +} + +%rule with_tag(X) <i>: X { $$ = $1; } + ; +``` + +### Named References for actions of RHS in Parameterizing rules + +Allow to use named references for actions of RHS in Parameterizing rules. + +```yacc +%rule option(number): /* empty */ + | number { $$ = $number; } + ; +``` + +## Lrama 0.6.8 (2024-04-29) + +### Nested Parameterizing rules with tag + +Allow to nested Parameterizing rules with tag. + +```yacc +%union { + int i; +} + +%rule nested_nested_option(X): /* empty */ + | X + ; + +%rule nested_option(X): /* empty */ + | nested_nested_option(X) <i> + ; + +%rule option(Y): /* empty */ + | nested_option(Y) <i> + ; +``` + +## Lrama 0.6.7 (2024-04-28) + +### RHS of user defined Parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. + +User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined Parameterizing rules. + +``` +%rule with_word_seps(X): /* empty */ + | X ' '+ + ; +``` + +## Lrama 0.6.6 (2024-04-27) + +### Trace actions + +Support trace actions for debugging. +Run `exe/lrama --trace=actions` to show grammar rules with actions. + +```console +$ exe/lrama --trace=actions sample/calc.y +Grammar rules with actions: +$accept -> list, YYEOF {} +list -> ε {} +list -> list, LF {} +list -> list, expr, LF { printf("=> %d\n", $2); } +expr -> NUM {} +expr -> expr, '+', expr { $$ = $1 + $3; } +expr -> expr, '-', expr { $$ = $1 - $3; } +expr -> expr, '*', expr { $$ = $1 * $3; } +expr -> expr, '/', expr { $$ = $1 / $3; } +expr -> '(', expr, ')' { $$ = $2; } +``` + +### Inlining + +Support inlining for rules. +The `%inline` directive causes all references to symbols to be replaced with its definition. + +```yacc +%rule %inline op: PLUS { + } + | TIMES { * } + ; + +%% + +expr : number { $$ = $1; } + | expr op expr { $$ = $1 $2 $3; } + ; +``` + +as same as + +```yacc +expr : number { $$ = $1; } + | expr '+' expr { $$ = $1 + $3; } + | expr '*' expr { $$ = $1 * $3; } + ; +``` + +## Lrama 0.6.5 (2024-03-25) + +### Typed Midrule Actions + +User can specify the type of mid-rule action by tag (`<bar>`) instead of specifying it with in an action. + +```yacc +primary: k_case expr_value terms? + { + $<val>$ = p->case_labels; + p->case_labels = Qnil; + } + case_body + k_end + { + ... + } +``` + +can be written as + +```yacc +primary: k_case expr_value terms? + { + $$ = p->case_labels; + p->case_labels = Qnil; + }<val> + case_body + k_end + { + ... + } +``` + +`%destructor` for midrule action is invoked only when tag is specified by Typed Midrule Actions. + +Difference from Bison's Typed Midrule Actions is that tag is postposed in Lrama however it's preposed in Bison. + +Bison supports this feature from 3.1. + +## Lrama 0.6.4 (2024-03-22) + +### Parameterizing rules (preceded, terminated, delimited) + +Support `preceded`, `terminated` and `delimited` rules. + +```text +program: preceded(opening, X) + +// Expanded to + +program: preceded_opening_X +preceded_opening_X: opening X +``` + +``` +program: terminated(X, closing) + +// Expanded to + +program: terminated_X_closing +terminated_X_closing: X closing +``` + +``` +program: delimited(opening, X, closing) + +// Expanded to + +program: delimited_opening_X_closing +delimited_opening_X_closing: opening X closing +``` + +https://github.com/ruby/lrama/pull/382 + +### Support `%destructor` declaration + +User can set codes for freeing semantic value resources by using `%destructor`. +In general, these resources are freed by actions or after parsing. +However, if syntax error happens in parsing, these codes may not be executed. +Codes associated to `%destructor` are executed when semantic value is popped from the stack by an error. + +```yacc +%token <val1> NUM +%type <val2> expr2 +%type <val3> expr + +%destructor { + printf("destructor for val1: %d\n", $$); +} <val1> // printer for TAG + +%destructor { + printf("destructor for val2: %d\n", $$); +} <val2> + +%destructor { + printf("destructor for expr: %d\n", $$); +} expr // printer for symbol +``` + +Bison supports this feature from 1.75b. + +https://github.com/ruby/lrama/pull/385 + +## Lrama 0.6.3 (2024-02-15) + +### Bring Your Own Stack + +Provide functionalities for Bring Your Own Stack. + +Ruby’s Ripper library requires their own semantic value stack to manage Ruby Objects returned by user defined callback method. Currently Ripper uses semantic value stack (`yyvsa`) which is used by parser to manage Node. This hack introduces some limitation on Ripper. For example, Ripper can not execute semantic analysis depending on Node structure. + +Lrama introduces two features to support another semantic value stack by parser generator users. + +1. Callback entry points + +User can emulate semantic value stack by these callbacks. +Lrama provides these five callbacks. Registered functions are called when each event happens. For example %after-shift function is called when shift happens on original semantic value stack. + +* `%after-shift` function_name +* `%before-reduce` function_name +* `%after-reduce` function_name +* `%after-shift-error-token` function_name +* `%after-pop-stack` function_name + +2. `$:n` variable to access index of each grammar symbols + +User also needs to access semantic value of their stack in grammar action. `$:n` provides the way to access to it. `$:n` is translated to the minus index from the top of the stack. +For example + +```yacc +primary: k_if expr_value then compstmt if_tail k_end + { + /*% ripper: if!($:2, $:4, $:5) %*/ + /* $:2 = -5, $:4 = -3, $:5 = -2. */ + } +``` + +https://github.com/ruby/lrama/pull/367 + +## Lrama 0.6.2 (2024-01-27) + +### %no-stdlib directive + +If `%no-stdlib` directive is set, Lrama doesn't load Lrama standard library for +parameterized rules, stdlib.y. + +https://github.com/ruby/lrama/pull/344 + +## Lrama 0.6.1 (2024-01-13) + +### Nested Parameterizing rules + +Allow to pass an instantiated rule to other Parameterizing rules. + +```yacc %rule constant(X) : X - ; + ; %rule option(Y) : /* empty */ | Y @@ -21,9 +820,9 @@ program : option(constant(number)) // Nested rule %% ``` -Allow to use nested parameterizing rules when define parameterizing rules. +Allow to use nested Parameterizing rules when define Parameterizing rules. -``` +```yacc %rule option(x) : /* empty */ | X ; @@ -46,11 +845,11 @@ https://github.com/ruby/lrama/pull/337 ## Lrama 0.6.0 (2023-12-25) -### User defined parameterizing rules +### User defined Parameterizing rules -Allow to define parameterizing rule by `%rule` directive. +Allow to define Parameterizing rule by `%rule` directive. -``` +```yacc %rule pair(X, Y): X Y { $$ = $1 + $2; } ; @@ -68,12 +867,12 @@ https://github.com/ruby/lrama/pull/285 ## Lrama 0.5.11 (2023-12-02) -### Type specification of parameterizing rules +### Type specification of Parameterizing rules Allow to specify type of rules by specifying tag, `<i>` in below example. Tag is post-modification style. -``` +```yacc %union { int i; } @@ -92,15 +891,15 @@ https://github.com/ruby/lrama/pull/272 ### Parameterizing rules (option, nonempty_list, list) -Support function call style parameterizing rules for `option`, `nonempty_list` and `list`. +Support function call style Parameterizing rules for `option`, `nonempty_list` and `list`. https://github.com/ruby/lrama/pull/197 ### Parameterizing rules (separated_list) -Support `separated_list` and `separated_nonempty_list` parameterizing rules. +Support `separated_list` and `separated_nonempty_list` Parameterizing rules. -``` +```text program: separated_list(',', number) // Expanded to @@ -131,7 +930,7 @@ https://github.com/ruby/lrama/pull/204 Parameterizing rules are template of rules. It's very common pattern to write "list" grammar rule like: -``` +```yacc opt_args: /* none */ | args ; @@ -154,7 +953,7 @@ https://github.com/ruby/lrama/pull/181 ### Racc parser -Replace Lrama's parser from hand written parser to LR parser generated by Racc. +Replace Lrama's parser from handwritten parser to LR parser generated by Racc. Lrama uses `--embedded` option to generate LR parser because Racc is changed from default gem to bundled gem by Ruby 3.3 (https://github.com/ruby/lrama/pull/132). https://github.com/ruby/lrama/pull/62 @@ -163,7 +962,7 @@ https://github.com/ruby/lrama/pull/62 ### Runtime configuration for error recovery -Meke error recovery function configurable on runtime by two new macros. +Make error recovery function configurable on runtime by two new macros. * `YYMAXREPAIR`: Expected to return max length of repair operations. `%parse-param` is passed to this function. * `YYERROR_RECOVERY_ENABLED`: Expected to return bool value to determine error recovery is enabled or not. `%parse-param` is passed to this function. @@ -186,7 +985,7 @@ https://github.com/ruby/lrama/pull/44 Instead of positional references like `$1` or `$$`, named references allow to access to symbol by name. -``` +```yacc primary: k_class cpath superclass bodystmt k_end { $primary = new_class($cpath, $bodystmt, $superclass); @@ -195,7 +994,7 @@ primary: k_class cpath superclass bodystmt k_end Alias name can be declared. -``` +```yacc expr[result]: expr[ex-left] '+' expr[ex.right] { $result = $[ex-left] + $[ex.right]; diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama index ba5fb06c82..710ac0cb96 100755 --- a/tool/lrama/exe/lrama +++ b/tool/lrama/exe/lrama @@ -1,6 +1,7 @@ #!/usr/bin/env ruby +# frozen_string_literal: true $LOAD_PATH << File.join(__dir__, "../lib") require "lrama" -Lrama::Command.new.run(ARGV.dup) +Lrama::Command.new(ARGV.dup).run diff --git a/tool/lrama/lib/lrama.rb b/tool/lrama/lib/lrama.rb index 9e517b0d71..56ba0044d4 100644 --- a/tool/lrama/lib/lrama.rb +++ b/tool/lrama/lib/lrama.rb @@ -1,17 +1,22 @@ -require "lrama/bitmap" -require "lrama/command" -require "lrama/context" -require "lrama/counterexamples" -require "lrama/digraph" -require "lrama/grammar" -require "lrama/lexer" -require "lrama/option_parser" -require "lrama/options" -require "lrama/output" -require "lrama/parser" -require "lrama/report" -require "lrama/state" -require "lrama/states" -require "lrama/states_reporter" -require "lrama/version" -require "lrama/warning" +# frozen_string_literal: true + +require_relative "lrama/bitmap" +require_relative "lrama/command" +require_relative "lrama/context" +require_relative "lrama/counterexamples" +require_relative "lrama/diagram" +require_relative "lrama/digraph" +require_relative "lrama/erb" +require_relative "lrama/grammar" +require_relative "lrama/lexer" +require_relative "lrama/logger" +require_relative "lrama/option_parser" +require_relative "lrama/options" +require_relative "lrama/output" +require_relative "lrama/parser" +require_relative "lrama/reporter" +require_relative "lrama/state" +require_relative "lrama/states" +require_relative "lrama/tracer" +require_relative "lrama/version" +require_relative "lrama/warnings" diff --git a/tool/lrama/lib/lrama/bitmap.rb b/tool/lrama/lib/lrama/bitmap.rb index 8349a23c34..88b255b012 100644 --- a/tool/lrama/lib/lrama/bitmap.rb +++ b/tool/lrama/lib/lrama/bitmap.rb @@ -1,5 +1,12 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama module Bitmap + # @rbs! + # type bitmap = Integer + + # @rbs (Array[Integer] ary) -> bitmap def self.from_array(ary) bit = 0 @@ -10,20 +17,31 @@ module Lrama bit end + # @rbs (Integer int) -> bitmap + def self.from_integer(int) + 1 << int + end + + # @rbs (bitmap int) -> Array[Integer] def self.to_array(int) - a = [] + a = [] #: Array[Integer] i = 0 - while int > 0 do - if int & 1 == 1 + len = int.bit_length + while i < len do + if int[i] == 1 a << i end i += 1 - int >>= 1 end a end + + # @rbs (bitmap int, Integer size) -> Array[bool] + def self.to_bool_array(int, size) + Array.new(size) { |i| int[i] == 1 } + end end end diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb index a39eed139b..17aad1a1c1 100644 --- a/tool/lrama/lib/lrama/command.rb +++ b/tool/lrama/lib/lrama/command.rb @@ -1,60 +1,120 @@ +# frozen_string_literal: true + module Lrama class Command - def run(argv) - begin - options = OptionParser.new.parse(argv) - rescue => e - message = e.message - message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty? - abort message - end + LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__))) + STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y') + + def initialize(argv) + @logger = Lrama::Logger.new + @options = OptionParser.parse(argv) + @tracer = Tracer.new(STDERR, **@options.trace_opts) + @reporter = Reporter.new(**@options.report_opts) + @warnings = Warnings.new(@logger, @options.warnings) + rescue => e + abort format_error_message(e.message) + end - Report::Duration.enable if options.trace_opts[:time] - - warning = Lrama::Warning.new - text = options.y.read - options.y.close if options.y != STDIN - parser = Lrama::Parser.new(text, options.grammar_file, options.debug) - begin - grammar = parser.parse - rescue => e - raise e if options.debug - message = e.message - message = message.gsub(/.+/, "\e[1m\\&\e[m") if Exception.to_tty? - abort message + def run + Lrama::Reporter::Profile::CallStack.report(@options.profile_opts[:call_stack]) do + Lrama::Reporter::Profile::Memory.report(@options.profile_opts[:memory]) do + execute_command_workflow + end end - states = Lrama::States.new(grammar, warning, trace_state: (options.trace_opts[:automaton] || options.trace_opts[:closure])) + end + + private + + def execute_command_workflow + @tracer.enable_duration + text = read_input + grammar = build_grammar(text) + states, context = compute_status(grammar) + render_reports(states) if @options.report_file + @tracer.trace(grammar) + render_diagram(grammar) + render_output(context, grammar) + states.validate!(@logger) + @warnings.warn(grammar, states) + end + + def read_input + text = @options.y.read + @options.y.close unless @options.y == STDIN + text + end + + def build_grammar(text) + grammar = + Lrama::Parser.new(text, @options.grammar_file, @options.debug, @options.locations, @options.define).parse + merge_stdlib(grammar) + prepare_grammar(grammar) + grammar + rescue => e + raise e if @options.debug + abort format_error_message(e.message) + end + + def format_error_message(message) + return message unless Exception.to_tty? + + message.gsub(/.+/, "\e[1m\\&\e[m") + end + + def merge_stdlib(grammar) + return if grammar.no_stdlib + + stdlib_text = File.read(STDLIB_FILE_PATH) + stdlib_grammar = Lrama::Parser.new( + stdlib_text, + STDLIB_FILE_PATH, + @options.debug, + @options.locations, + @options.define, + ).parse + + grammar.prepend_parameterized_rules(stdlib_grammar.parameterized_rules) + end + + def prepare_grammar(grammar) + grammar.prepare + grammar.validate! + end + + def compute_status(grammar) + states = Lrama::States.new(grammar, @tracer) states.compute - context = Lrama::Context.new(states) + states.compute_ielr if grammar.ielr_defined? + [states, Lrama::Context.new(states)] + end - if options.report_file - reporter = Lrama::StatesReporter.new(states) - File.open(options.report_file, "w+") do |f| - reporter.report(f, **options.report_opts) - end + def render_reports(states) + File.open(@options.report_file, "w+") do |f| + @reporter.report(f, states) end + end + + def render_diagram(grammar) + return unless @options.diagram - if options.trace_opts && options.trace_opts[:rules] - puts "Grammar rules:" - puts grammar.rules + File.open(@options.diagram_file, "w+") do |f| + Lrama::Diagram.render(out: f, grammar: grammar) end + end - File.open(options.outfile, "w+") do |f| + def render_output(context, grammar) + File.open(@options.outfile, "w+") do |f| Lrama::Output.new( out: f, - output_file_path: options.outfile, - template_name: options.skeleton, - grammar_file_path: options.grammar_file, - header_file_path: options.header_file, + output_file_path: @options.outfile, + template_name: @options.skeleton, + grammar_file_path: @options.grammar_file, + header_file_path: @options.header_file, context: context, grammar: grammar, - error_recovery: options.error_recovery, + error_recovery: @options.error_recovery, ).render end - - if warning.has_error? - exit false - end end end end diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb index 245c91a199..eb068c1b9e 100644 --- a/tool/lrama/lib/lrama/context.rb +++ b/tool/lrama/lib/lrama/context.rb @@ -1,9 +1,11 @@ -require "lrama/report/duration" +# frozen_string_literal: true + +require_relative "tracer/duration" module Lrama # This is passed to a template class Context - include Report::Duration + include Tracer::Duration ErrorActionNumber = -Float::INFINITY BaseMin = -Float::INFINITY @@ -41,7 +43,7 @@ module Lrama def yyfinal @states.states.find do |state| state.items.find do |item| - item.rule.lhs.accept_symbol? && item.end_of_rule? + item.lhs.accept_symbol? && item.end_of_rule? end end.id end @@ -221,7 +223,7 @@ module Lrama if state.reduces.map(&:selected_look_ahead).any? {|la| !la.empty? } # Iterate reduces with reverse order so that first rule is used. - state.reduces.reverse.each do |reduce| + state.reduces.reverse_each do |reduce| reduce.look_ahead.each do |term| actions[term.number] = rule_id_to_action_number(reduce.rule.id) end @@ -229,8 +231,8 @@ module Lrama end # Shift is selected when S/R conflict exists. - state.selected_term_transitions.each do |shift, next_state| - actions[shift.next_sym.number] = next_state.id + state.selected_term_transitions.each do |shift| + actions[shift.next_sym.number] = shift.to_state.id end state.resolved_conflicts.select do |conflict| @@ -253,7 +255,7 @@ module Lrama # If no default_reduction_rule, default behavior is an # error then replace ErrorActionNumber with zero. - if !state.default_reduction_rule + unless state.default_reduction_rule actions.map! do |e| if e == ErrorActionNumber 0 @@ -265,9 +267,9 @@ module Lrama s = actions.each_with_index.map do |n, i| [i, n] - end.select do |i, n| + end.reject do |i, n| # Remove default_reduction_rule entries - n != 0 + n == 0 end if s.count != 0 @@ -290,21 +292,18 @@ module Lrama # of a default nterm transition destination. @yydefgoto = Array.new(@states.nterms.count, 0) # Mapping from nterm to next_states - nterm_to_next_states = {} + nterm_to_to_states = {} @states.states.each do |state| - state.nterm_transitions.each do |shift, next_state| - key = shift.next_sym - nterm_to_next_states[key] ||= [] - nterm_to_next_states[key] << [state, next_state] # [from_state, to_state] + state.nterm_transitions.each do |goto| + key = goto.next_sym + nterm_to_to_states[key] ||= [] + nterm_to_to_states[key] << [state, goto.to_state] # [from_state, to_state] end end @states.nterms.each do |nterm| - if !(states = nterm_to_next_states[nterm]) - default_goto = 0 - not_default_gotos = [] - else + if (states = nterm_to_to_states[nterm]) default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first default_goto = default_state.id not_default_gotos = [] @@ -312,6 +311,9 @@ module Lrama next if to_state.id == default_goto not_default_gotos << [from_state.id, to_state.id] end + else + default_goto = 0 + not_default_gotos = [] end k = nterm_number_to_sequence_number(nterm.number) @@ -403,7 +405,7 @@ module Lrama @check = [] # Key is froms_and_tos, value is index position pushed = {} - userd_res = {} + used_res = {} lowzero = 0 high = 0 @@ -415,27 +417,25 @@ module Lrama res = lowzero - froms_and_tos.first[0] + # Find the smallest `res` such that `@table[res + from]` is empty for all `from` in `froms_and_tos` while true do - ok = true + advanced = false - froms_and_tos.each do |from, to| - loc = res + from - - if @table[loc] - # If the cell of table is set, can not use the cell. - ok = false - break - end + while used_res[res] + res += 1 + advanced = true end - if ok && userd_res[res] - ok = false + froms_and_tos.each do |from, to| + while @table[res + from] + res += 1 + advanced = true + end end - if ok + unless advanced + # no advance means that the current `res` satisfies the condition break - else - res += 1 end end @@ -456,13 +456,13 @@ module Lrama @base[state_id] = res pushed[froms_and_tos] = res - userd_res[res] = true + used_res[res] = true end @yylast = high # replace_ninf - @yypact_ninf = (@base.select {|i| i != BaseMin } + [0]).min - 1 + @yypact_ninf = (@base.reject {|i| i == BaseMin } + [0]).min - 1 @base.map! do |i| case i when BaseMin @@ -472,7 +472,7 @@ module Lrama end end - @yytable_ninf = (@table.compact.select {|i| i != ErrorActionNumber } + [0]).min - 1 + @yytable_ninf = (@table.compact.reject {|i| i == ErrorActionNumber } + [0]).min - 1 @table.map! do |i| case i when nil diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb index 046265da59..60d830d048 100644 --- a/tool/lrama/lib/lrama/counterexamples.rb +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -1,55 +1,116 @@ +# rbs_inline: enabled +# frozen_string_literal: true + require "set" +require "timeout" -require "lrama/counterexamples/derivation" -require "lrama/counterexamples/example" -require "lrama/counterexamples/path" -require "lrama/counterexamples/production_path" -require "lrama/counterexamples/start_path" -require "lrama/counterexamples/state_item" -require "lrama/counterexamples/transition_path" -require "lrama/counterexamples/triple" +require_relative "counterexamples/derivation" +require_relative "counterexamples/example" +require_relative "counterexamples/node" +require_relative "counterexamples/path" +require_relative "counterexamples/state_item" +require_relative "counterexamples/triple" module Lrama # See: https://www.cs.cornell.edu/andru/papers/cupex/cupex.pdf # 4. Constructing Nonunifying Counterexamples class Counterexamples - attr_reader :transitions, :productions - + PathSearchTimeLimit = 10 # 10 sec + CumulativeTimeLimit = 120 # 120 sec + + # @rbs! + # @states: States + # @iterate_count: Integer + # @total_duration: Float + # @exceed_cumulative_time_limit: bool + # @state_items: Hash[[State, State::Item], StateItem] + # @triples: Hash[Integer, Triple] + # @transitions: Hash[[StateItem, Grammar::Symbol], StateItem] + # @reverse_transitions: Hash[[StateItem, Grammar::Symbol], Set[StateItem]] + # @productions: Hash[StateItem, Set[StateItem]] + # @reverse_productions: Hash[[State, Grammar::Symbol], Set[StateItem]] # Grammar::Symbol is nterm + # @state_item_shift: Integer + + attr_reader :transitions #: Hash[[StateItem, Grammar::Symbol], StateItem] + attr_reader :productions #: Hash[StateItem, Set[StateItem]] + + # @rbs (States states) -> void def initialize(states) @states = states + @iterate_count = 0 + @total_duration = 0 + @exceed_cumulative_time_limit = false + @triples = {} + setup_state_items setup_transitions setup_productions end + # @rbs () -> "#<Counterexamples>" def to_s "#<Counterexamples>" end alias :inspect :to_s + # @rbs (State conflict_state) -> Array[Example] def compute(conflict_state) conflict_state.conflicts.flat_map do |conflict| + # Check cumulative time limit for not each path search method call but each conflict + # to avoid one of example's path to be nil. + next if @exceed_cumulative_time_limit + case conflict.type when :shift_reduce + # @type var conflict: State::ShiftReduceConflict shift_reduce_example(conflict_state, conflict) when :reduce_reduce + # @type var conflict: State::ReduceReduceConflict reduce_reduce_examples(conflict_state, conflict) end + rescue Timeout::Error => e + STDERR.puts "Counterexamples calculation for state #{conflict_state.id} #{e.message} with #{@iterate_count} iteration" + increment_total_duration(PathSearchTimeLimit) + nil end.compact end private + # @rbs (State state, State::Item item) -> StateItem + def get_state_item(state, item) + @state_items[[state, item]] + end + + # For optimization, create all StateItem in advance + # and use them by fetching an instance from `@state_items`. + # Do not create new StateItem instance in the shortest path search process + # to avoid miss hash lookup. + # + # @rbs () -> void + def setup_state_items + @state_items = {} + count = 0 + + @states.states.each do |state| + state.items.each do |item| + @state_items[[state, item]] = StateItem.new(count, state, item) + count += 1 + end + end + + @state_item_shift = Math.log(count, 2).ceil + end + + # @rbs () -> void def setup_transitions - # Hash [StateItem, Symbol] => StateItem @transitions = {} - # Hash [StateItem, Symbol] => Set(StateItem) @reverse_transitions = {} @states.states.each do |src_state| - trans = {} + trans = {} #: Hash[Grammar::Symbol, State] - src_state.transitions.each do |shift, next_state| - trans[shift.next_sym] = next_state + src_state.transitions.each do |transition| + trans[transition.next_sym] = transition.to_state end src_state.items.each do |src_item| @@ -59,11 +120,12 @@ module Lrama dest_state.kernels.each do |dest_item| next unless (src_item.rule == dest_item.rule) && (src_item.position + 1 == dest_item.position) - src_state_item = StateItem.new(src_state, src_item) - dest_state_item = StateItem.new(dest_state, dest_item) + src_state_item = get_state_item(src_state, src_item) + dest_state_item = get_state_item(dest_state, dest_item) @transitions[[src_state_item, sym]] = dest_state_item + # @type var key: [StateItem, Grammar::Symbol] key = [dest_state_item, sym] @reverse_transitions[key] ||= Set.new @reverse_transitions[key] << src_state_item @@ -72,21 +134,20 @@ module Lrama end end + # @rbs () -> void def setup_productions - # Hash [StateItem] => Set(Item) @productions = {} - # Hash [State, Symbol] => Set(Item). Symbol is nterm @reverse_productions = {} @states.states.each do |state| - # LHS => Set(Item) - h = {} + # Grammar::Symbol is LHS + h = {} #: Hash[Grammar::Symbol, Set[StateItem]] state.closure.each do |item| sym = item.lhs h[sym] ||= Set.new - h[sym] << item + h[sym] << get_state_item(state, item) end state.items.each do |item| @@ -94,108 +155,132 @@ module Lrama next if item.next_sym.term? sym = item.next_sym - state_item = StateItem.new(state, item) - key = [state, sym] - + state_item = get_state_item(state, item) @productions[state_item] = h[sym] + # @type var key: [State, Grammar::Symbol] + key = [state, sym] @reverse_productions[key] ||= Set.new - @reverse_productions[key] << item + @reverse_productions[key] << state_item end end end + # For optimization, use same Triple if it's already created. + # Do not create new Triple instance anywhere else + # to avoid miss hash lookup. + # + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> Triple + def get_triple(state_item, precise_lookahead_set) + key = (precise_lookahead_set << @state_item_shift) | state_item.id + @triples[key] ||= Triple.new(state_item, precise_lookahead_set) + end + + # @rbs (State conflict_state, State::ShiftReduceConflict conflict) -> Example def shift_reduce_example(conflict_state, conflict) conflict_symbol = conflict.symbols.first + # @type var shift_conflict_item: ::Lrama::State::Item shift_conflict_item = conflict_state.items.find { |item| item.next_sym == conflict_symbol } - path2 = shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) - path1 = find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) + end + path1 = with_timeout("#find_shift_conflict_shortest_path:") do + find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + end Example.new(path1, path2, conflict, conflict_symbol, self) end + # @rbs (State conflict_state, State::ReduceReduceConflict conflict) -> Example def reduce_reduce_examples(conflict_state, conflict) conflict_symbol = conflict.symbols.first - path1 = shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) - path2 = shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + path1 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) + end + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + end Example.new(path1, path2, conflict, conflict_symbol, self) end - def find_shift_conflict_shortest_path(reduce_path, conflict_state, conflict_item) - state_items = find_shift_conflict_shortest_state_items(reduce_path, conflict_state, conflict_item) - build_paths_from_state_items(state_items) - end + # @rbs (Array[StateItem]? reduce_state_items, State conflict_state, State::Item conflict_item) -> Array[StateItem] + def find_shift_conflict_shortest_path(reduce_state_items, conflict_state, conflict_item) + time1 = Time.now.to_f + @iterate_count = 0 - def find_shift_conflict_shortest_state_items(reduce_path, conflict_state, conflict_item) - target_state_item = StateItem.new(conflict_state, conflict_item) + target_state_item = get_state_item(conflict_state, conflict_item) result = [target_state_item] - reversed_reduce_path = reduce_path.to_a.reverse + reversed_state_items = reduce_state_items.to_a.reverse # Index for state_item i = 0 - while (path = reversed_reduce_path[i]) + while (state_item = reversed_state_items[i]) # Index for prev_state_item j = i + 1 _j = j - while (prev_path = reversed_reduce_path[j]) - if prev_path.production? + while (prev_state_item = reversed_state_items[j]) + if prev_state_item.type == :production j += 1 else break end end - state_item = path.to - prev_state_item = prev_path&.to - if target_state_item == state_item || target_state_item.item.start_item? - result.concat(reversed_reduce_path[_j..-1].map(&:to)) + result.concat( + reversed_state_items[_j..-1] #: Array[StateItem] + ) break end - if target_state_item.item.beginning_of_rule? - queue = [] - queue << [target_state_item] + if target_state_item.type == :production + queue = [] #: Array[Node[StateItem]] + queue << Node.new(target_state_item, nil) # Find reverse production while (sis = queue.shift) - si = sis.last + @iterate_count += 1 + si = sis.elem # Reach to start state if si.item.start_item? - sis.shift - result.concat(sis) + a = Node.to_a(sis).reverse + a.shift + result.concat(a) target_state_item = si break end - if !si.item.beginning_of_rule? + if si.type == :production + # @type var key: [State, Grammar::Symbol] + key = [si.state, si.item.lhs] + @reverse_productions[key].each do |state_item| + queue << Node.new(state_item, sis) + end + else + # @type var key: [StateItem, Grammar::Symbol] key = [si, si.item.previous_sym] @reverse_transitions[key].each do |prev_target_state_item| - next if prev_target_state_item.state != prev_state_item.state - sis.shift - result.concat(sis) + next if prev_target_state_item.state != prev_state_item&.state + a = Node.to_a(sis).reverse + a.shift + result.concat(a) result << prev_target_state_item target_state_item = prev_target_state_item i = j queue.clear break end - else - key = [si.state, si.item.lhs] - @reverse_productions[key].each do |item| - state_item = StateItem.new(si.state, item) - queue << (sis + [state_item]) - end end end else # Find reverse transition + # @type var key: [StateItem, Grammar::Symbol] key = [target_state_item, target_state_item.item.previous_sym] @reverse_transitions[key].each do |prev_target_state_item| - next if prev_target_state_item.state != prev_state_item.state + next if prev_target_state_item.state != prev_state_item&.state result << prev_target_state_item target_state_item = prev_target_state_item i = j @@ -204,68 +289,106 @@ module Lrama end end + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "find_shift_conflict_shortest_path #{@iterate_count} iteration", duration) + end + result.reverse end - def build_paths_from_state_items(state_items) - state_items.zip([nil] + state_items).map do |si, prev_si| - case - when prev_si.nil? - StartPath.new(si) - when si.item.beginning_of_rule? - ProductionPath.new(prev_si, si) - else - TransitionPath.new(prev_si, si) + # @rbs (StateItem target) -> Set[StateItem] + def reachable_state_items(target) + result = Set.new + queue = [target] + + while (state_item = queue.shift) + next if result.include?(state_item) + result << state_item + + @reverse_transitions[[state_item, state_item.item.previous_sym]]&.each do |prev_state_item| + queue << prev_state_item + end + + if state_item.item.beginning_of_rule? + @reverse_productions[[state_item.state, state_item.item.lhs]]&.each do |si| + queue << si + end end end + + result end + # @rbs (State conflict_state, State::Item conflict_reduce_item, Grammar::Symbol conflict_term) -> ::Array[StateItem]? def shortest_path(conflict_state, conflict_reduce_item, conflict_term) - # queue: is an array of [Triple, [Path]] - queue = [] - visited = {} - start_state = @states.states.first + time1 = Time.now.to_f + @iterate_count = 0 + + queue = [] #: Array[[Triple, Path]] + visited = {} #: Hash[Triple, true] + start_state = @states.states.first #: Lrama::State + conflict_term_bit = Bitmap::from_integer(conflict_term.number) raise "BUG: Start state should be just one kernel." if start_state.kernels.count != 1 + reachable = reachable_state_items(get_state_item(conflict_state, conflict_reduce_item)) + start = get_triple(get_state_item(start_state, start_state.kernels.first), Bitmap::from_integer(@states.eof_symbol.number)) + + queue << [start, Path.new(start.state_item, nil)] + + while (triple, path = queue.shift) + @iterate_count += 1 - start = Triple.new(start_state, start_state.kernels.first, Set.new([@states.eof_symbol])) + # Found + if (triple.state == conflict_state) && (triple.item == conflict_reduce_item) && (triple.l & conflict_term_bit != 0) + state_items = [path.state_item] - queue << [start, [StartPath.new(start.state_item)]] + while (path = path.parent) + state_items << path.state_item + end - while true - triple, paths = queue.shift + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) - next if visited[triple] - visited[triple] = true + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "shortest_path #{@iterate_count} iteration", duration) + end - # Found - if triple.state == conflict_state && triple.item == conflict_reduce_item && triple.l.include?(conflict_term) - return paths + return state_items.reverse end # transition - triple.state.transitions.each do |shift, next_state| - next unless triple.item.next_sym && triple.item.next_sym == shift.next_sym - next_state.kernels.each do |kernel| - next if kernel.rule != triple.item.rule - t = Triple.new(next_state, kernel, triple.l) - queue << [t, paths + [TransitionPath.new(triple.state_item, t.state_item)]] + next_state_item = @transitions[[triple.state_item, triple.item.next_sym]] + if next_state_item && reachable.include?(next_state_item) + # @type var t: Triple + t = get_triple(next_state_item, triple.l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] end end # production step - triple.state.closure.each do |item| - next unless triple.item.next_sym && triple.item.next_sym == item.lhs + @productions[triple.state_item]&.each do |si| + next unless reachable.include?(si) + l = follow_l(triple.item, triple.l) - t = Triple.new(triple.state, item, l) - queue << [t, paths + [ProductionPath.new(triple.state_item, t.state_item)]] + # @type var t: Triple + t = get_triple(si, l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end end - - break if queue.empty? end return nil end + # @rbs (State::Item item, Bitmap::bitmap current_l) -> Bitmap::bitmap def follow_l(item, current_l) # 1. follow_L (A -> X1 ... Xn-1 • Xn) = L # 2. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = {Xk+2} if Xk+2 is a terminal @@ -275,11 +398,28 @@ module Lrama when item.number_of_rest_symbols == 1 current_l when item.next_next_sym.term? - Set.new([item.next_next_sym]) + item.next_next_sym.number_bitmap when !item.next_next_sym.nullable - item.next_next_sym.first_set + item.next_next_sym.first_set_bitmap else - item.next_next_sym.first_set + follow_l(item.new_by_next_position, current_l) + item.next_next_sym.first_set_bitmap | follow_l(item.new_by_next_position, current_l) + end + end + + # @rbs [T] (String message) { -> T } -> T + def with_timeout(message) + Timeout.timeout(PathSearchTimeLimit, Timeout::Error, message + " timeout of #{PathSearchTimeLimit} sec exceeded") do + yield + end + end + + # @rbs (Float|Integer duration) -> void + def increment_total_duration(duration) + @total_duration += duration + + if !@exceed_cumulative_time_limit && @total_duration > CumulativeTimeLimit + @exceed_cumulative_time_limit = true + STDERR.puts "CumulativeTimeLimit #{CumulativeTimeLimit} sec exceeded then skip following Counterexamples calculation" end end end diff --git a/tool/lrama/lib/lrama/counterexamples/derivation.rb b/tool/lrama/lib/lrama/counterexamples/derivation.rb index 691e935356..a2b74767a9 100644 --- a/tool/lrama/lib/lrama/counterexamples/derivation.rb +++ b/tool/lrama/lib/lrama/counterexamples/derivation.rb @@ -1,32 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Counterexamples class Derivation - attr_reader :item, :left, :right - attr_writer :right + # @rbs! + # @item: State::Item + # @left: Derivation? + + attr_reader :item #: State::Item + attr_reader :left #: Derivation? + attr_accessor :right #: Derivation? - def initialize(item, left, right = nil) + # @rbs (State::Item item, Derivation? left) -> void + def initialize(item, left) @item = item @left = left - @right = right end + # @rbs () -> ::String def to_s "#<Derivation(#{item.display_name})>" end alias :inspect :to_s + # @rbs () -> Array[String] def render_strings_for_report - result = [] + result = [] #: Array[String] _render_for_report(self, 0, result, 0) result.map(&:rstrip) end + # @rbs () -> String def render_for_report render_strings_for_report.join("\n") end private + # @rbs (Derivation derivation, Integer offset, Array[String] strings, Integer index) -> Integer def _render_for_report(derivation, offset, strings, index) item = derivation.item if strings[index] @@ -42,18 +54,19 @@ module Lrama str << "#{item.next_sym.display_name}" length = _render_for_report(derivation.left, len, strings, index + 1) # I want String#ljust! - str << " " * (length - str.length) + str << " " * (length - str.length) if length > str.length else str << " • #{item.symbols_after_dot.map(&:display_name).join(" ")} " return str.length end if derivation.right&.left - length = _render_for_report(derivation.right.left, str.length, strings, index + 1) - str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " + left = derivation.right&.left #: Derivation + length = _render_for_report(left, str.length, strings, index + 1) + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore str << " " * (length - str.length) if length > str.length elsif item.next_next_sym - str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore end return str.length diff --git a/tool/lrama/lib/lrama/counterexamples/example.rb b/tool/lrama/lib/lrama/counterexamples/example.rb index 8f02d71fa4..c007f45af4 100644 --- a/tool/lrama/lib/lrama/counterexamples/example.rb +++ b/tool/lrama/lib/lrama/counterexamples/example.rb @@ -1,10 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Counterexamples class Example - attr_reader :path1, :path2, :conflict, :conflict_symbol + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @path1: ::Array[StateItem] + # @path2: ::Array[StateItem] + # @conflict: State::conflict + # @conflict_symbol: Grammar::Symbol + # @counterexamples: Counterexamples + # @derivations1: Derivation + # @derivations2: Derivation + + attr_reader :path1 #: ::Array[StateItem] + attr_reader :path2 #: ::Array[StateItem] + attr_reader :conflict #: State::conflict + attr_reader :conflict_symbol #: Grammar::Symbol # path1 is shift conflict when S/R conflict # path2 is always reduce conflict + # + # @rbs (Array[StateItem]? path1, Array[StateItem]? path2, State::conflict conflict, Grammar::Symbol conflict_symbol, Counterexamples counterexamples) -> void def initialize(path1, path2, conflict, conflict_symbol, counterexamples) @path1 = path1 @path2 = path2 @@ -13,66 +34,75 @@ module Lrama @counterexamples = counterexamples end + # @rbs () -> (:shift_reduce | :reduce_reduce) def type @conflict.type end + # @rbs () -> State::Item def path1_item - @path1.last.to.item + @path1.last.item end + # @rbs () -> State::Item def path2_item - @path2.last.to.item + @path2.last.item end + # @rbs () -> Derivation def derivations1 @derivations1 ||= _derivations(path1) end + # @rbs () -> Derivation def derivations2 @derivations2 ||= _derivations(path2) end private - def _derivations(paths) - derivation = nil + # @rbs (Array[StateItem] state_items) -> Derivation + def _derivations(state_items) + derivation = nil #: Derivation current = :production - lookahead_sym = paths.last.to.item.end_of_rule? ? @conflict_symbol : nil + last_state_item = state_items.last #: StateItem + lookahead_sym = last_state_item.item.end_of_rule? ? @conflict_symbol : nil - paths.reverse.each do |path| - item = path.to.item + state_items.reverse_each do |si| + item = si.item case current when :production - case path - when StartPath + case si.type + when :start derivation = Derivation.new(item, derivation) current = :start - when TransitionPath + when :transition derivation = Derivation.new(item, derivation) current = :transition - when ProductionPath + when :production derivation = Derivation.new(item, derivation) current = :production + else + raise "Unexpected. #{si}" end if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym) - state_item = @counterexamples.transitions[[path.to, item.next_sym]] - derivation2 = find_derivation_for_symbol(state_item, lookahead_sym) - derivation.right = derivation2 + si2 = @counterexamples.transitions[[si, item.next_sym]] + derivation2 = find_derivation_for_symbol(si2, lookahead_sym) + derivation.right = derivation2 # steep:ignore lookahead_sym = nil end when :transition - case path - when StartPath + case si.type + when :start derivation = Derivation.new(item, derivation) current = :start - when TransitionPath + when :transition # ignore current = :transition - when ProductionPath + when :production # ignore current = :production end @@ -86,8 +116,9 @@ module Lrama derivation end + # @rbs (StateItem state_item, Grammar::Symbol sym) -> Derivation? def find_derivation_for_symbol(state_item, sym) - queue = [] + queue = [] #: Array[Array[StateItem]] queue << [state_item] while (sis = queue.shift) @@ -97,7 +128,7 @@ module Lrama if next_sym == sym derivation = nil - sis.reverse.each do |si| + sis.reverse_each do |si| derivation = Derivation.new(si.item, derivation) end @@ -105,9 +136,8 @@ module Lrama end if next_sym.nterm? && next_sym.first_set.include?(sym) - @counterexamples.productions[si].each do |next_item| - next if next_item.empty_rule? - next_si = StateItem.new(si.state, next_item) + @counterexamples.productions[si].each do |next_si| + next if next_si.item.empty_rule? next if sis.include?(next_si) queue << (sis + [next_si]) end diff --git a/tool/lrama/lib/lrama/counterexamples/node.rb b/tool/lrama/lib/lrama/counterexamples/node.rb new file mode 100644 index 0000000000..9214a0e7f1 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/node.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + # @rbs generic E < Object -- Type of an element + class Node + attr_reader :elem #: E + attr_reader :next_node #: Node[E]? + + # @rbs [E < Object] (Node[E] node) -> Array[E] + def self.to_a(node) + a = [] # steep:ignore UnannotatedEmptyCollection + + while (node) + a << node.elem + node = node.next_node + end + + a + end + + # @rbs (E elem, Node[E]? next_node) -> void + def initialize(elem, next_node) + @elem = elem + @next_node = next_node + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/path.rb b/tool/lrama/lib/lrama/counterexamples/path.rb index edba67a3b6..6b1325f73b 100644 --- a/tool/lrama/lib/lrama/counterexamples/path.rb +++ b/tool/lrama/lib/lrama/counterexamples/path.rb @@ -1,21 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Counterexamples class Path - def initialize(from_state_item, to_state_item) - @from_state_item = from_state_item - @to_state_item = to_state_item - end + # @rbs! + # @state_item: StateItem + # @parent: Path? - def from - @from_state_item - end + attr_reader :state_item #: StateItem + attr_reader :parent #: Path? - def to - @to_state_item + # @rbs (StateItem state_item, Path? parent) -> void + def initialize(state_item, parent) + @state_item = state_item + @parent = parent end + # @rbs () -> ::String def to_s - "#<Path(#{type})>" + "#<Path>" end alias :inspect :to_s end diff --git a/tool/lrama/lib/lrama/counterexamples/production_path.rb b/tool/lrama/lib/lrama/counterexamples/production_path.rb deleted file mode 100644 index d7db688518..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/production_path.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Lrama - class Counterexamples - class ProductionPath < Path - def type - :production - end - - def transition? - false - end - - def production? - true - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/start_path.rb b/tool/lrama/lib/lrama/counterexamples/start_path.rb deleted file mode 100644 index 4a6821cd0f..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/start_path.rb +++ /dev/null @@ -1,21 +0,0 @@ -module Lrama - class Counterexamples - class StartPath < Path - def initialize(to_state_item) - super nil, to_state_item - end - - def type - :start - end - - def transition? - false - end - - def production? - false - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/state_item.rb b/tool/lrama/lib/lrama/counterexamples/state_item.rb index 930ff4a5f8..8c2481d793 100644 --- a/tool/lrama/lib/lrama/counterexamples/state_item.rb +++ b/tool/lrama/lib/lrama/counterexamples/state_item.rb @@ -1,6 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Counterexamples - class StateItem < Struct.new(:state, :item) + class StateItem + attr_reader :id #: Integer + attr_reader :state #: State + attr_reader :item #: State::Item + + # @rbs (Integer id, State state, State::Item item) -> void + def initialize(id, state, item) + @id = id + @state = state + @item = item + end + + # @rbs () -> (:start | :transition | :production) + def type + case + when item.start_item? + :start + when item.beginning_of_rule? + :production + else + :transition + end + end end end end diff --git a/tool/lrama/lib/lrama/counterexamples/transition_path.rb b/tool/lrama/lib/lrama/counterexamples/transition_path.rb deleted file mode 100644 index 96e611612a..0000000000 --- a/tool/lrama/lib/lrama/counterexamples/transition_path.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Lrama - class Counterexamples - class TransitionPath < Path - def type - :transition - end - - def transition? - true - end - - def production? - false - end - end - end -end diff --git a/tool/lrama/lib/lrama/counterexamples/triple.rb b/tool/lrama/lib/lrama/counterexamples/triple.rb index e802beccf4..98fe051f53 100644 --- a/tool/lrama/lib/lrama/counterexamples/triple.rb +++ b/tool/lrama/lib/lrama/counterexamples/triple.rb @@ -1,19 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Counterexamples - # s: state - # itm: item within s - # l: precise lookahead set - class Triple < Struct.new(:s, :itm, :l) - alias :state :s - alias :item :itm - alias :precise_lookahead_set :l + class Triple + attr_reader :precise_lookahead_set #: Bitmap::bitmap + + alias :l :precise_lookahead_set + + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> void + def initialize(state_item, precise_lookahead_set) + @state_item = state_item + @precise_lookahead_set = precise_lookahead_set + end + + # @rbs () -> State + def state + @state_item.state + end + alias :s :state + + # @rbs () -> State::Item + def item + @state_item.item + end + alias :itm :item + # @rbs () -> StateItem def state_item - StateItem.new(state, item) + @state_item end + # @rbs () -> ::String def inspect - "#{state.inspect}. #{item.display_name}. #{l.map(&:id).map(&:s_value)}" + "#{state.inspect}. #{item.display_name}. #{l.to_s(2)}" end alias :to_s :inspect end diff --git a/tool/lrama/lib/lrama/diagram.rb b/tool/lrama/lib/lrama/diagram.rb new file mode 100644 index 0000000000..985808933f --- /dev/null +++ b/tool/lrama/lib/lrama/diagram.rb @@ -0,0 +1,77 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Diagram + class << self + # @rbs (IO out, Grammar grammar, String template_name) -> void + def render(out:, grammar:, template_name: 'diagram/diagram.html') + return unless require_railroad_diagrams + new(out: out, grammar: grammar, template_name: template_name).render + end + + # @rbs () -> bool + def require_railroad_diagrams + require "railroad_diagrams" + true + rescue LoadError + warn "railroad_diagrams is not installed. Please run `bundle install`." + false + end + end + + # @rbs (IO out, Grammar grammar, String template_name) -> void + def initialize(out:, grammar:, template_name: 'diagram/diagram.html') + @grammar = grammar + @out = out + @template_name = template_name + end + + # @rbs () -> void + def render + RailroadDiagrams::TextDiagram.set_formatting(RailroadDiagrams::TextDiagram::PARTS_UNICODE) + @out << ERB.render(template_file, output: self) + end + + # @rbs () -> string + def default_style + RailroadDiagrams::Style::default_style + end + + # @rbs () -> string + def diagrams + result = +'' + @grammar.unique_rule_s_values.each do |s_value| + diagrams = + @grammar.select_rules_by_s_value(s_value).map { |r| r.to_diagrams } + add_diagram( + s_value, + RailroadDiagrams::Diagram.new( + RailroadDiagrams::Choice.new(0, *diagrams), + ), + result + ) + end + result + end + + private + + # @rbs () -> string + def template_dir + File.expand_path('../../template', __dir__) + end + + # @rbs () -> string + def template_file + File.join(template_dir, @template_name) + end + + # @rbs (String name, RailroadDiagrams::Diagram diagram, String result) -> void + def add_diagram(name, diagram, result) + result << "\n<h2 class=\"diagram-header\">#{RailroadDiagrams.escape_html(name)}</h2>" + diagram.write_svg(result.method(:<<)) + result << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/digraph.rb b/tool/lrama/lib/lrama/digraph.rb index bbaa86019f..52865f52dd 100644 --- a/tool/lrama/lib/lrama/digraph.rb +++ b/tool/lrama/lib/lrama/digraph.rb @@ -1,21 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama - # Algorithm Digraph of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # Digraph Algorithm of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # + # Digraph is an algorithm for graph data structure. + # The algorithm efficiently traverses SCC (Strongly Connected Component) of graph + # and merges nodes attributes within the same SCC. + # + # `compute_read_sets` and `compute_follow_sets` have the same structure. + # Graph of gotos and attributes of gotos are given then compute propagated attributes for each node. + # + # In the case of `compute_read_sets`: + # + # * Set of gotos is nodes of graph + # * `reads_relation` is edges of graph + # * `direct_read_sets` is nodes attributes + # + # In the case of `compute_follow_sets`: + # + # * Set of gotos is nodes of graph + # * `includes_relation` is edges of graph + # * `read_sets` is nodes attributes + # + # + # @rbs generic X < Object -- Type of a node + # @rbs generic Y < _Or -- Type of attribute sets assigned to a node which should support merge operation (#| method) class Digraph + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # interface _Or + # def |: (self) -> self + # end + # @sets: Array[X] + # @relation: Hash[X, Array[X]] + # @base_function: Hash[X, Y] + # @stack: Array[X] + # @h: Hash[X, (Integer|Float)?] + # @result: Hash[X, Y] + + # @rbs sets: Array[X] -- Nodes of graph + # @rbs relation: Hash[X, Array[X]] -- Edges of graph + # @rbs base_function: Hash[X, Y] -- Attributes of nodes + # @rbs return: void def initialize(sets, relation, base_function) + # X in the paper @sets = sets + # R in the paper @relation = relation + # F' in the paper @base_function = base_function + # S in the paper @stack = [] + # N in the paper @h = Hash.new(0) + # F in the paper @result = {} end + # @rbs () -> Hash[X, Y] def compute @sets.each do |x| next if @h[x] != 0 @@ -27,6 +79,7 @@ module Lrama private + # @rbs (X x) -> void def traverse(x) @stack.push(x) d = @stack.count diff --git a/tool/lrama/lib/lrama/erb.rb b/tool/lrama/lib/lrama/erb.rb new file mode 100644 index 0000000000..8f8be54811 --- /dev/null +++ b/tool/lrama/lib/lrama/erb.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "erb" + +module Lrama + class ERB + # @rbs (String file, **untyped kwargs) -> String + def self.render(file, **kwargs) + new(file).render(**kwargs) + end + + # @rbs (String file) -> void + def initialize(file) + input = File.read(file) + if ::ERB.instance_method(:initialize).parameters.last.first == :key + @erb = ::ERB.new(input, trim_mode: '-') + else + @erb = ::ERB.new(input, nil, '-') # steep:ignore UnexpectedPositionalArgument + end + @erb.filename = file + end + + # @rbs (**untyped kwargs) -> String + def render(**kwargs) + @erb.result_with_hash(kwargs) + end + end +end diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb index 9c500b381d..95a80bb01c 100644 --- a/tool/lrama/lib/lrama/grammar.rb +++ b/tool/lrama/lib/lrama/grammar.rb @@ -1,240 +1,324 @@ -require "lrama/grammar/auxiliary" -require "lrama/grammar/binding" -require "lrama/grammar/code" -require "lrama/grammar/counter" -require "lrama/grammar/error_token" -require "lrama/grammar/percent_code" -require "lrama/grammar/precedence" -require "lrama/grammar/printer" -require "lrama/grammar/reference" -require "lrama/grammar/rule" -require "lrama/grammar/rule_builder" -require "lrama/grammar/parameterizing_rule" -require "lrama/grammar/symbol" -require "lrama/grammar/type" -require "lrama/grammar/union" -require "lrama/lexer" +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "grammar/auxiliary" +require_relative "grammar/binding" +require_relative "grammar/code" +require_relative "grammar/counter" +require_relative "grammar/destructor" +require_relative "grammar/error_token" +require_relative "grammar/inline" +require_relative "grammar/parameterized" +require_relative "grammar/percent_code" +require_relative "grammar/precedence" +require_relative "grammar/printer" +require_relative "grammar/reference" +require_relative "grammar/rule" +require_relative "grammar/rule_builder" +require_relative "grammar/symbol" +require_relative "grammar/symbols" +require_relative "grammar/type" +require_relative "grammar/union" +require_relative "lexer" module Lrama # Grammar is the result of parsing an input grammar file class Grammar - attr_reader :percent_codes, :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux - attr_accessor :union, :expect, - :printers, :error_tokens, - :lex_param, :parse_param, :initial_action, - :symbols, :types, - :rules, :rule_builders, - :sym_to_rules - - def initialize(rule_counter) + # @rbs! + # + # interface _DelegatedMethods + # def rules: () -> Array[Rule] + # def accept_symbol: () -> Grammar::Symbol + # def eof_symbol: () -> Grammar::Symbol + # def undef_symbol: () -> Grammar::Symbol + # def precedences: () -> Array[Precedence] + # + # # delegate to @symbols_resolver + # def symbols: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def ielr_defined?: () -> bool + # end + # + # include Symbols::Resolver::_DelegatedMethods + # + # @rule_counter: Counter + # @percent_codes: Array[PercentCode] + # @printers: Array[Printer] + # @destructors: Array[Destructor] + # @error_tokens: Array[ErrorToken] + # @symbols_resolver: Symbols::Resolver + # @types: Array[Type] + # @rule_builders: Array[RuleBuilder] + # @rules: Array[Rule] + # @sym_to_rules: Hash[Integer, Array[Rule]] + # @parameterized_resolver: Parameterized::Resolver + # @empty_symbol: Grammar::Symbol + # @eof_symbol: Grammar::Symbol + # @error_symbol: Grammar::Symbol + # @undef_symbol: Grammar::Symbol + # @accept_symbol: Grammar::Symbol + # @aux: Auxiliary + # @no_stdlib: bool + # @locations: bool + # @define: Hash[String, String] + # @required: bool + # @union: Union + # @precedences: Array[Precedence] + # @start_nterm: Lrama::Lexer::Token::Base? + + extend Forwardable + + attr_reader :percent_codes #: Array[PercentCode] + attr_reader :eof_symbol #: Grammar::Symbol + attr_reader :error_symbol #: Grammar::Symbol + attr_reader :undef_symbol #: Grammar::Symbol + attr_reader :accept_symbol #: Grammar::Symbol + attr_reader :aux #: Auxiliary + attr_reader :parameterized_resolver #: Parameterized::Resolver + attr_reader :precedences #: Array[Precedence] + attr_accessor :union #: Union + attr_accessor :expect #: Integer + attr_accessor :printers #: Array[Printer] + attr_accessor :error_tokens #: Array[ErrorToken] + attr_accessor :lex_param #: String + attr_accessor :parse_param #: String + attr_accessor :initial_action #: Grammar::Code::InitialActionCode + attr_accessor :after_shift #: Lexer::Token::Base + attr_accessor :before_reduce #: Lexer::Token::Base + attr_accessor :after_reduce #: Lexer::Token::Base + attr_accessor :after_shift_error_token #: Lexer::Token::Base + attr_accessor :after_pop_stack #: Lexer::Token::Base + attr_accessor :symbols_resolver #: Symbols::Resolver + attr_accessor :types #: Array[Type] + attr_accessor :rules #: Array[Rule] + attr_accessor :rule_builders #: Array[RuleBuilder] + attr_accessor :sym_to_rules #: Hash[Integer, Array[Rule]] + attr_accessor :no_stdlib #: bool + attr_accessor :locations #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :required #: bool + + def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value, + :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, + :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type, + :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number! + + # @rbs (Counter rule_counter, bool locations, Hash[String, String] define) -> void + def initialize(rule_counter, locations, define = {}) @rule_counter = rule_counter # Code defined by "%code" @percent_codes = [] @printers = [] + @destructors = [] @error_tokens = [] - @symbols = [] + @symbols_resolver = Grammar::Symbols::Resolver.new @types = [] @rule_builders = [] @rules = [] @sym_to_rules = {} - @parameterizing_rule_resolver = ParameterizingRule::Resolver.new + @parameterized_resolver = Parameterized::Resolver.new @empty_symbol = nil @eof_symbol = nil @error_symbol = nil @undef_symbol = nil @accept_symbol = nil @aux = Auxiliary.new + @no_stdlib = false + @locations = locations + @define = define + @required = false + @precedences = [] + @start_nterm = nil append_special_symbols end + # @rbs (Counter rule_counter, Counter midrule_action_counter) -> RuleBuilder + def create_rule_builder(rule_counter, midrule_action_counter) + RuleBuilder.new(rule_counter, midrule_action_counter, @parameterized_resolver) + end + + # @rbs (id: Lexer::Token::Base, code: Lexer::Token::UserCode) -> Array[PercentCode] def add_percent_code(id:, code:) @percent_codes << PercentCode.new(id.s_value, code.s_value) end + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Destructor] + def add_destructor(ident_or_tags:, token_code:, lineno:) + @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Printer] def add_printer(ident_or_tags:, token_code:, lineno:) @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) end + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[ErrorToken] def add_error_token(ident_or_tags:, token_code:, lineno:) @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) end - def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) - if token_id && (sym = @symbols.find {|s| s.token_id == token_id }) - if replace - sym.id = id - sym.alias_name = alias_name - sym.tag = tag - end - - return sym - end - - if (sym = @symbols.find {|s| s.id == id }) - return sym - end - - sym = Symbol.new( - id: id, alias_name: alias_name, number: nil, tag: tag, - term: true, token_id: token_id, nullable: false - ) - @symbols << sym - @terms = nil - - return sym - end - - def add_nterm(id:, alias_name: nil, tag: nil) - return if @symbols.find {|s| s.id == id } - - sym = Symbol.new( - id: id, alias_name: alias_name, number: nil, tag: tag, - term: false, token_id: nil, nullable: nil, - ) - @symbols << sym - @nterms = nil - - return sym - end - + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> Array[Type] def add_type(id:, tag:) @types << Type.new(id: id, tag: tag) end - def add_nonassoc(sym, precedence) - set_precedence(sym, Precedence.new(type: :nonassoc, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_nonassoc(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :nonassoc, precedence: precedence, lineno: lineno)) end - def add_left(sym, precedence) - set_precedence(sym, Precedence.new(type: :left, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_left(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :left, precedence: precedence, lineno: lineno)) end - def add_right(sym, precedence) - set_precedence(sym, Precedence.new(type: :right, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_right(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :right, precedence: precedence, lineno: lineno)) end - def add_precedence(sym, precedence) - set_precedence(sym, Precedence.new(type: :precedence, precedence: precedence)) + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_precedence(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :precedence, precedence: precedence, lineno: lineno)) end + # @rbs (Lrama::Lexer::Token::Base id) -> Lrama::Lexer::Token::Base + def set_start_nterm(id) + # When multiple `%start` directives are defined, Bison does not generate an error, + # whereas Lrama does generate an error. + # Related Bison's specification are + # refs: https://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html + if @start_nterm.nil? + @start_nterm = id + else + start = @start_nterm #: Lrama::Lexer::Token::Base + raise "Start non-terminal is already set to #{start.s_value} (line: #{start.first_line}). Cannot set to #{id.s_value} (line: #{id.first_line})." + end + end + + # @rbs (Grammar::Symbol sym, Precedence precedence) -> (Precedence | bot) def set_precedence(sym, precedence) - raise "" if sym.nterm? + @precedences << precedence sym.precedence = precedence end + # @rbs (Grammar::Code::NoReferenceCode code, Integer lineno) -> Union def set_union(code, lineno) @union = Union.new(code: code, lineno: lineno) end + # @rbs (RuleBuilder builder) -> Array[RuleBuilder] def add_rule_builder(builder) @rule_builders << builder end - def add_parameterizing_rule(rule) - @parameterizing_rule_resolver.add_parameterizing_rule(rule) + # @rbs (Parameterized::Rule rule) -> Array[Parameterized::Rule] + def add_parameterized_rule(rule) + @parameterized_resolver.add_rule(rule) + end + + # @rbs () -> Array[Parameterized::Rule] + def parameterized_rules + @parameterized_resolver.rules end + # @rbs (Array[Parameterized::Rule] rules) -> Array[Parameterized::Rule] + def prepend_parameterized_rules(rules) + @parameterized_resolver.rules = rules + @parameterized_resolver.rules + end + + # @rbs (Integer prologue_first_lineno) -> Integer def prologue_first_lineno=(prologue_first_lineno) @aux.prologue_first_lineno = prologue_first_lineno end + # @rbs (String prologue) -> String def prologue=(prologue) @aux.prologue = prologue end + # @rbs (Integer epilogue_first_lineno) -> Integer def epilogue_first_lineno=(epilogue_first_lineno) @aux.epilogue_first_lineno = epilogue_first_lineno end + # @rbs (String epilogue) -> String def epilogue=(epilogue) @aux.epilogue = epilogue end + # @rbs () -> void def prepare + resolve_inline_rules normalize_rules collect_symbols set_lhs_and_rhs - fill_symbol_number fill_default_precedence + fill_symbols fill_sym_to_rules - fill_nterm_type - fill_symbol_printer - fill_symbol_error_token - @symbols.sort_by!(&:number) + sort_precedence compute_nullable compute_first_set + set_locations end # TODO: More validation methods # # * Validation for no_declared_type_reference + # + # @rbs () -> void def validate! - validate_symbol_number_uniqueness! - validate_symbol_alias_name_uniqueness! + @symbols_resolver.validate! + validate_no_precedence_for_nterm! validate_rule_lhs_is_nterm! + validate_duplicated_precedence! end - def find_symbol_by_s_value(s_value) - @symbols.find do |sym| - sym.id.s_value == s_value - end - end - - def find_symbol_by_s_value!(s_value) - find_symbol_by_s_value(s_value) || (raise "Symbol not found: #{s_value}") - end - - def find_symbol_by_id(id) - @symbols.find do |sym| - sym.id == id || sym.alias_name == id.s_value - end - end - - def find_symbol_by_id!(id) - find_symbol_by_id(id) || (raise "Symbol not found: #{id}") - end - - def find_symbol_by_number!(number) - sym = @symbols[number] - - raise "Symbol not found: #{number}" unless sym - raise "[BUG] Symbol number mismatch. #{number}, #{sym}" if sym.number != number - - sym - end - + # @rbs (Grammar::Symbol sym) -> Array[Rule] def find_rules_by_symbol!(sym) find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found") end + # @rbs (Grammar::Symbol sym) -> Array[Rule]? def find_rules_by_symbol(sym) @sym_to_rules[sym.number] end - def terms_count - terms.count - end - - def terms - @terms ||= @symbols.select(&:term?) + # @rbs (String s_value) -> Array[Rule] + def select_rules_by_s_value(s_value) + @rules.select {|rule| rule.lhs.id.s_value == s_value } end - def nterms_count - nterms.count + # @rbs () -> Array[String] + def unique_rule_s_values + @rules.map {|rule| rule.lhs.id.s_value }.uniq end - def nterms - @nterms ||= @symbols.select(&:nterm?) + # @rbs () -> bool + def ielr_defined? + @define.key?('lr.type') && @define['lr.type'] == 'ielr' end private + # @rbs () -> void + def sort_precedence + @precedences.sort_by! do |prec| + prec.symbol.number + end + @precedences.freeze + end + + # @rbs () -> Array[Grammar::Symbol] def compute_nullable @rules.each do |rule| case - when rule.rhs.empty? + when rule.empty_rule? rule.nullable = true when rule.rhs.any?(&:term) rule.nullable = false @@ -275,11 +359,12 @@ module Lrama rule.nullable = false end - nterms.select {|r| r.nullable.nil? }.each do |nterm| + nterms.select {|e| e.nullable.nil? }.each do |nterm| nterm.nullable = false end end + # @rbs () -> Array[Grammar::Symbol] def compute_first_set terms.each do |term| term.first_set = Set.new([term]).freeze @@ -315,18 +400,14 @@ module Lrama end end + # @rbs () -> Array[RuleBuilder] def setup_rules @rule_builders.each do |builder| - builder.setup_rules(@parameterizing_rule_resolver) + builder.setup_rules end end - def find_nterm_by_id!(id) - nterms.find do |nterm| - nterm.id == id - end || (raise "Nterm not found: #{id}") - end - + # @rbs () -> Grammar::Symbol def append_special_symbols # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) @@ -357,13 +438,23 @@ module Lrama @accept_symbol = term end - def normalize_rules - # Add $accept rule to the top of rules - lineno = @rule_builders.first ? @rule_builders.first.line : 0 - @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [@rule_builders.first.lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + # @rbs () -> void + def resolve_inline_rules + while @rule_builders.any?(&:has_inline_rules?) do + @rule_builders = @rule_builders.flat_map do |builder| + if builder.has_inline_rules? + Inline::Resolver.new(builder).resolve + else + builder + end + end + end + end + # @rbs () -> void + def normalize_rules + add_accept_rule setup_rules - @rule_builders.each do |builder| builder.rules.each do |rule| add_nterm(id: rule._lhs, tag: rule.lhs_tag) @@ -371,96 +462,42 @@ module Lrama end end - @rules.sort_by!(&:id) + nterms.freeze + @rules.sort_by!(&:id).freeze + end + + # Add $accept rule to the top of rules + def add_accept_rule + if @start_nterm + start = @start_nterm #: Lrama::Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [start, @eof_symbol.id], token_code: nil, lineno: start.line) + else + rule_builder = @rule_builders.first #: RuleBuilder + lineno = rule_builder ? rule_builder.line : 0 + lhs = rule_builder.lhs #: Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + end end # Collect symbols from rules + # + # @rbs () -> void def collect_symbols @rules.flat_map(&:_rhs).each do |s| case s when Lrama::Lexer::Token::Char add_term(id: s) - when Lrama::Lexer::Token + when Lrama::Lexer::Token::Base # skip else raise "Unknown class: #{s}" end end - end - - # Fill #number and #token_id - def fill_symbol_number - # Character literal in grammar file has - # token id corresponding to ASCII code by default, - # so start token_id from 256. - token_id = 256 - - # YYEMPTY = -2 - # YYEOF = 0 - # YYerror = 1 - # YYUNDEF = 2 - number = 3 - - nterm_token_id = 0 - used_numbers = {} - - @symbols.map(&:number).each do |n| - used_numbers[n] = true - end - - (@symbols.select(&:term?) + @symbols.select(&:nterm?)).each do |sym| - while used_numbers[number] do - number += 1 - end - - if sym.number.nil? - sym.number = number - number += 1 - end - - # If id is Token::Char, it uses ASCII code - if sym.term? && sym.token_id.nil? - if sym.id.is_a?(Lrama::Lexer::Token::Char) - # Ignore ' on the both sides - case sym.id.s_value[1..-2] - when "\\b" - sym.token_id = 8 - when "\\f" - sym.token_id = 12 - when "\\n" - sym.token_id = 10 - when "\\r" - sym.token_id = 13 - when "\\t" - sym.token_id = 9 - when "\\v" - sym.token_id = 11 - when "\"" - sym.token_id = 34 - when "'" - sym.token_id = 39 - when "\\\\" - sym.token_id = 92 - when /\A\\(\d+)\z/ - sym.token_id = Integer($1, 8) - when /\A(.)\z/ - sym.token_id = $1.bytes.first - else - raise "Unknown Char s_value #{sym}" - end - else - sym.token_id = token_id - token_id += 1 - end - end - if sym.nterm? && sym.token_id.nil? - sym.token_id = nterm_token_id - nterm_token_id += 1 - end - end + terms.freeze end + # @rbs () -> void def set_lhs_and_rhs @rules.each do |rule| rule.lhs = token_to_symbol(rule._lhs) if rule._lhs @@ -471,18 +508,11 @@ module Lrama end end - def token_to_symbol(token) - case token - when Lrama::Lexer::Token - find_symbol_by_id!(token) - else - raise "Unknown class: #{token}" - end - end - # Rule inherits precedence from the last term in RHS. # # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html + # + # @rbs () -> void def fill_default_precedence @rules.each do |rule| # Explicitly specified precedence has the highest priority @@ -497,6 +527,17 @@ module Lrama end end + # @rbs () -> Array[Grammar::Symbol] + def fill_symbols + fill_symbol_number + fill_nterm_type(@types) + fill_printer(@printers) + fill_destructor(@destructors) + fill_error_token(@error_tokens) + sort_by_number! + end + + # @rbs () -> Array[Rule] def fill_sym_to_rules @rules.each do |rule| key = rule.lhs.number @@ -505,80 +546,58 @@ module Lrama end end - # Fill nterm's tag defined by %type decl - def fill_nterm_type - @types.each do |type| - nterm = find_nterm_by_id!(type.id) - nterm.tag = type.tag - end - end - - def fill_symbol_printer - @symbols.each do |sym| - @printers.each do |printer| - printer.ident_or_tags.each do |ident_or_tag| - case ident_or_tag - when Lrama::Lexer::Token::Ident - sym.printer = printer if sym.id == ident_or_tag - when Lrama::Lexer::Token::Tag - sym.printer = printer if sym.tag == ident_or_tag - else - raise "Unknown token type. #{printer}" - end - end - end - end - end + # @rbs () -> void + def validate_no_precedence_for_nterm! + errors = [] #: Array[String] - def fill_symbol_error_token - @symbols.each do |sym| - @error_tokens.each do |error_token| - error_token.ident_or_tags.each do |ident_or_tag| - case ident_or_tag - when Lrama::Lexer::Token::Ident - sym.error_token = error_token if sym.id == ident_or_tag - when Lrama::Lexer::Token::Tag - sym.error_token = error_token if sym.tag == ident_or_tag - else - raise "Unknown token type. #{error_token}" - end - end - end - end - end + nterms.each do |nterm| + next if nterm.precedence.nil? - def validate_symbol_number_uniqueness! - invalid = @symbols.group_by(&:number).select do |number, syms| - syms.count > 1 + errors << "[BUG] Precedence #{nterm.name} (line: #{nterm.precedence.lineno}) is defined for nonterminal symbol (line: #{nterm.id.first_line}). Precedence can be defined for only terminal symbol." end - return if invalid.empty? + return if errors.empty? - raise "Symbol number is duplicated. #{invalid}" + raise errors.join("\n") end - def validate_symbol_alias_name_uniqueness! - invalid = @symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| - syms.count > 1 + # @rbs () -> void + def validate_rule_lhs_is_nterm! + errors = [] #: Array[String] + + rules.each do |rule| + next if rule.lhs.nterm? + + errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is terminal symbol. It should be nonterminal symbol." end - return if invalid.empty? + return if errors.empty? - raise "Symbol alias name is duplicated. #{invalid}" + raise errors.join("\n") end - def validate_rule_lhs_is_nterm! - errors = [] + # # @rbs () -> void + def validate_duplicated_precedence! + errors = [] #: Array[String] + seen = {} #: Hash[String, Precedence] - rules.each do |rule| - next if rule.lhs.nterm? - - errors << "[BUG] LHS of #{rule} (line: #{rule.lineno}) is term. It should be nterm." + precedences.each do |prec| + s_value = prec.s_value + if first = seen[s_value] + errors << "%#{prec.type} redeclaration for #{s_value} (line: #{prec.lineno}) previous declaration was %#{first.type} (line: #{first.lineno})" + else + seen[s_value] = prec + end end return if errors.empty? raise errors.join("\n") end + + # @rbs () -> void + def set_locations + @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? } + end end end diff --git a/tool/lrama/lib/lrama/grammar/auxiliary.rb b/tool/lrama/lib/lrama/grammar/auxiliary.rb index 933574b0f6..76cfb74d4d 100644 --- a/tool/lrama/lib/lrama/grammar/auxiliary.rb +++ b/tool/lrama/lib/lrama/grammar/auxiliary.rb @@ -1,7 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar # Grammar file information not used by States but by Output - class Auxiliary < Struct.new(:prologue_first_lineno, :prologue, :epilogue_first_lineno, :epilogue, keyword_init: true) + class Auxiliary + attr_accessor :prologue_first_lineno #: Integer? + attr_accessor :prologue #: String? + attr_accessor :epilogue_first_lineno #: Integer? + attr_accessor :epilogue #: String? end end end diff --git a/tool/lrama/lib/lrama/grammar/binding.rb b/tool/lrama/lib/lrama/grammar/binding.rb index e5ea3fb037..5940d153a9 100644 --- a/tool/lrama/lib/lrama/grammar/binding.rb +++ b/tool/lrama/lib/lrama/grammar/binding.rb @@ -1,22 +1,77 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class Binding - attr_reader :actual_args, :count + # @rbs @actual_args: Array[Lexer::Token::Base] + # @rbs @param_to_arg: Hash[String, Lexer::Token::Base] - def initialize(parameterizing_rule, actual_args) - @parameters = parameterizing_rule.parameters + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> void + def initialize(params, actual_args) @actual_args = actual_args - @parameter_to_arg = @parameters.zip(actual_args).map do |param, arg| + @param_to_arg = build_param_to_arg(params, @actual_args) + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def resolve_symbol(sym) + return create_instantiate_rule(sym) if sym.is_a?(Lexer::Token::InstantiateRule) + find_arg_for_param(sym) + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def concatenated_args_str(token) + "#{token.rule_name}_#{format_args(token)}" + end + + private + + # @rbs (Lexer::Token::InstantiateRule sym) -> Lexer::Token::InstantiateRule + def create_instantiate_rule(sym) + Lrama::Lexer::Token::InstantiateRule.new( + s_value: sym.s_value, + location: sym.location, + args: resolve_args(sym.args), + lhs_tag: sym.lhs_tag + ) + end + + # @rbs (Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def resolve_args(args) + args.map { |arg| resolve_symbol(arg) } + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def find_arg_for_param(sym) + if (arg = @param_to_arg[sym.s_value]&.dup) + arg.alias_name = sym.alias_name + arg + else + sym + end + end + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> Hash[String, Lexer::Token::Base?] + def build_param_to_arg(params, actual_args) + params.zip(actual_args).map do |param, arg| [param.s_value, arg] end.to_h end - def resolve_symbol(symbol) - if symbol.is_a?(Lexer::Token::InstantiateRule) - resolved_args = symbol.args.map { |arg| resolve_symbol(arg) } - Lrama::Lexer::Token::InstantiateRule.new(s_value: symbol.s_value, location: symbol.location, args: resolved_args, lhs_tag: symbol.lhs_tag) - else - @parameter_to_arg[symbol.s_value] || symbol + # @rbs (Lexer::Token::InstantiateRule token) -> String + def format_args(token) + token_to_args_s_values(token).join('_') + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Array[String] + def token_to_args_s_values(token) + token.args.flat_map do |arg| + resolved = resolve_symbol(arg) + if resolved.is_a?(Lexer::Token::InstantiateRule) + [resolved.s_value] + resolved.args.map(&:s_value) + else + [resolved.s_value] + end end end end diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb index d0bef75ef1..f1b860eeba 100644 --- a/tool/lrama/lib/lrama/grammar/code.rb +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -1,23 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + require "forwardable" -require "lrama/grammar/code/initial_action_code" -require "lrama/grammar/code/no_reference_code" -require "lrama/grammar/code/printer_code" -require "lrama/grammar/code/rule_action" +require_relative "code/destructor_code" +require_relative "code/initial_action_code" +require_relative "code/no_reference_code" +require_relative "code/printer_code" +require_relative "code/rule_action" module Lrama class Grammar class Code + # @rbs! + # + # # delegated + # def s_value: -> String + # def line: -> Integer + # def column: -> Integer + # def references: -> Array[Lrama::Grammar::Reference] + extend Forwardable def_delegators "token_code", :s_value, :line, :column, :references - attr_reader :type, :token_code + attr_reader :type #: ::Symbol + attr_reader :token_code #: Lexer::Token::UserCode + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode) -> void def initialize(type:, token_code:) @type = type @token_code = token_code end + # @rbs (Code other) -> bool def ==(other) self.class == other.class && self.type == other.type && @@ -25,10 +40,12 @@ module Lrama end # $$, $n, @$, @n are translated to C code + # + # @rbs () -> String def translated_code t_code = s_value.dup - references.reverse.each do |ref| + references.reverse_each do |ref| first_column = ref.first_column last_column = ref.last_column @@ -42,6 +59,7 @@ module Lrama private + # @rbs (Lrama::Grammar::Reference ref) -> bot def reference_to_c(ref) raise NotImplementedError.new("#reference_to_c is not implemented") end diff --git a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb new file mode 100644 index 0000000000..d71b62e513 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class DestructorCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb index 2b064f271e..cb36041524 100644 --- a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb @@ -1,3 +1,6 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class Code @@ -6,18 +9,26 @@ module Lrama # * ($$) yylval # * (@$) yylloc + # * ($:$) error # * ($1) error # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ "yylval" when ref.type == :at && ref.name == "$" # @$ "yylloc" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in initial_action." when ref.type == :dollar # $n raise "$#{ref.value} can not be used in initial_action." when ref.type == :at # @n raise "@#{ref.value} can not be used in initial_action." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in initial_action." else raise "Unexpected. #{self}, #{ref}" end diff --git a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb index ac6cdb8fba..1d39919979 100644 --- a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb @@ -1,3 +1,6 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class Code @@ -6,14 +9,20 @@ module Lrama # * ($$) error # * (@$) error + # * ($:$) error # * ($1) error # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> bot def reference_to_c(ref) case when ref.type == :dollar # $$, $n raise "$#{ref.value} can not be used in #{type}." when ref.type == :at # @$, @n raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:$, $:n + raise "$:#{ref.value} can not be used in #{type}." else raise "Unexpected. #{self}, #{ref}" end diff --git a/tool/lrama/lib/lrama/grammar/code/printer_code.rb b/tool/lrama/lib/lrama/grammar/code/printer_code.rb index 2b1f127f41..c6e25d5235 100644 --- a/tool/lrama/lib/lrama/grammar/code/printer_code.rb +++ b/tool/lrama/lib/lrama/grammar/code/printer_code.rb @@ -1,7 +1,18 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class Code class PrinterCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void def initialize(type:, token_code:, tag:) super(type: type, token_code: token_code) @tag = tag @@ -11,8 +22,12 @@ module Lrama # * ($$) *yyvaluep # * (@$) *yylocationp + # * ($:$) error # * ($1) error # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ @@ -20,10 +35,14 @@ module Lrama "((*yyvaluep).#{member})" when ref.type == :at && ref.name == "$" # @$ "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." when ref.type == :dollar # $n raise "$#{ref.value} can not be used in #{type}." when ref.type == :at # @n raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." else raise "Unexpected. #{self}, #{ref}" end diff --git a/tool/lrama/lib/lrama/grammar/code/rule_action.rb b/tool/lrama/lib/lrama/grammar/code/rule_action.rb index 76169b91ed..e71e93e5a5 100644 --- a/tool/lrama/lib/lrama/grammar/code/rule_action.rb +++ b/tool/lrama/lib/lrama/grammar/code/rule_action.rb @@ -1,7 +1,18 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class Code class RuleAction < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @rule: Rule + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, rule: Rule) -> void def initialize(type:, token_code:, rule:) super(type: type, token_code: token_code) @rule = rule @@ -11,8 +22,10 @@ module Lrama # * ($$) yyval # * (@$) yyloc + # * ($:$) error # * ($1) yyvsp[i] # * (@1) yylsp[i] + # * ($:1) i - 1 # # # Consider a rule like @@ -24,6 +37,8 @@ module Lrama # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } # "Position in grammar" $1 $2 $3 $4 $5 # "Index for yyvsp" -4 -3 -2 -1 0 + # "$:n" $:1 $:2 $:3 $:4 $:5 + # "index of $:n" -5 -4 -3 -2 -1 # # # For the first midrule action: @@ -31,27 +46,38 @@ module Lrama # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } # "Position in grammar" $1 # "Index for yyvsp" 0 + # "$:n" $:1 + # + # @rbs (Reference ref) -> String def reference_to_c(ref) case when ref.type == :dollar && ref.name == "$" # $$ tag = ref.ex_tag || lhs.tag raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag "(yyval.#{tag.member})" when ref.type == :at && ref.name == "$" # @$ "(yyloc)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:$ is not supported" when ref.type == :dollar # $n i = -position_in_rhs + ref.index tag = ref.ex_tag || rhs[ref.index - 1].tag raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag "(yyvsp[#{i}].#{tag.member})" when ref.type == :at # @n i = -position_in_rhs + ref.index "(yylsp[#{i}])" + when ref.type == :index # $:n + i = -position_in_rhs + ref.index + "(#{i} - 1)" else raise "Unexpected. #{self}, #{ref}" end end + # @rbs () -> Integer def position_in_rhs # If rule is not derived rule, User Code is only action at # the end of rule RHS. In such case, the action is located on @@ -59,18 +85,23 @@ module Lrama @rule.position_in_original_rule_rhs || @rule.rhs.count end - # If this is midrule action, RHS is a RHS of the original rule. + # If this is midrule action, RHS is an RHS of the original rule. + # + # @rbs () -> Array[Grammar::Symbol] def rhs (@rule.original_rule || @rule).rhs end - # Unlike `rhs`, LHS is always a LHS of the rule. + # Unlike `rhs`, LHS is always an LHS of the rule. + # + # @rbs () -> Grammar::Symbol def lhs @rule.lhs end + # @rbs (Reference ref) -> bot def raise_tag_not_found_error(ref) - raise "Tag is not specified for '$#{ref.value}' in '#{@rule.to_s}'" + raise "Tag is not specified for '$#{ref.value}' in '#{@rule.display_name}'" end end end diff --git a/tool/lrama/lib/lrama/grammar/counter.rb b/tool/lrama/lib/lrama/grammar/counter.rb index c13f4ec3e3..ced934309d 100644 --- a/tool/lrama/lib/lrama/grammar/counter.rb +++ b/tool/lrama/lib/lrama/grammar/counter.rb @@ -1,10 +1,22 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class Counter + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @number: Integer + + # @rbs (Integer number) -> void def initialize(number) @number = number end + # @rbs () -> Integer def increment n = @number @number += 1 diff --git a/tool/lrama/lib/lrama/grammar/destructor.rb b/tool/lrama/lib/lrama/grammar/destructor.rb new file mode 100644 index 0000000000..0ce8611e77 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/destructor.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Destructor + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident|Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::DestructorCode.new(type: :destructor, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/error_token.rb b/tool/lrama/lib/lrama/grammar/error_token.rb index 8efde7df33..9d9ed54ae2 100644 --- a/tool/lrama/lib/lrama/grammar/error_token.rb +++ b/tool/lrama/lib/lrama/grammar/error_token.rb @@ -1,6 +1,21 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar - class ErrorToken < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + class ErrorToken + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident | Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String def translated_code(tag) Code::PrinterCode.new(type: :error_token, token_code: token_code, tag: tag).translated_code end diff --git a/tool/lrama/lib/lrama/grammar/inline.rb b/tool/lrama/lib/lrama/grammar/inline.rb new file mode 100644 index 0000000000..c02ab6002b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative 'inline/resolver' diff --git a/tool/lrama/lib/lrama/grammar/inline/resolver.rb b/tool/lrama/lib/lrama/grammar/inline/resolver.rb new file mode 100644 index 0000000000..aca689ccfb --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline/resolver.rb @@ -0,0 +1,80 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Inline + class Resolver + # @rbs (Lrama::Grammar::RuleBuilder rule_builder) -> void + def initialize(rule_builder) + @rule_builder = rule_builder + end + + # @rbs () -> Array[Lrama::Grammar::RuleBuilder] + def resolve + resolved_builders = [] #: Array[Lrama::Grammar::RuleBuilder] + @rule_builder.rhs.each_with_index do |token, i| + if (rule = @rule_builder.parameterized_resolver.find_inline(token)) + rule.rhs.each do |rhs| + builder = build_rule(rhs, token, i, rule) + resolved_builders << builder + end + break + end + end + resolved_builders + end + + private + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Lrama::Lexer::Token token, Integer index, Lrama::Grammar::Parameterized::Rule rule) -> Lrama::Grammar::RuleBuilder + def build_rule(rhs, token, index, rule) + builder = RuleBuilder.new( + @rule_builder.rule_counter, + @rule_builder.midrule_action_counter, + @rule_builder.parameterized_resolver, + lhs_tag: @rule_builder.lhs_tag + ) + resolve_rhs(builder, rhs, index, token, rule) + builder.lhs = @rule_builder.lhs + builder.line = @rule_builder.line + builder.precedence_sym = @rule_builder.precedence_sym + builder.user_code = replace_user_code(rhs, index) + builder + end + + # @rbs (Lrama::Grammar::RuleBuilder builder, Lrama::Grammar::Parameterized::Rhs rhs, Integer index, Lrama::Lexer::Token token, Lrama::Grammar::Parameterized::Rule rule) -> void + def resolve_rhs(builder, rhs, index, token, rule) + @rule_builder.rhs.each_with_index do |tok, i| + if i == index + rhs.symbols.each do |sym| + if token.is_a?(Lexer::Token::InstantiateRule) + bindings = Binding.new(rule.parameters, token.args) + builder.add_rhs(bindings.resolve_symbol(sym)) + else + builder.add_rhs(sym) + end + end + else + builder.add_rhs(tok) + end + end + end + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Integer index) -> Lrama::Lexer::Token::UserCode + def replace_user_code(rhs, index) + user_code = @rule_builder.user_code + return user_code if rhs.user_code.nil? || user_code.nil? + + code = user_code.s_value.gsub(/\$#{index + 1}/, rhs.user_code.s_value) + user_code.references.each do |ref| + next if ref.index.nil? || ref.index <= index # nil は $$ の場合 + code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (rhs.symbols.count - 1)}") + code = code.gsub(/@#{ref.index}/, "@#{ref.index + (rhs.symbols.count - 1)}") + end + Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized.rb b/tool/lrama/lib/lrama/grammar/parameterized.rb new file mode 100644 index 0000000000..48db3433f3 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'parameterized/resolver' +require_relative 'parameterized/rhs' +require_relative 'parameterized/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb new file mode 100644 index 0000000000..558f308190 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Resolver + attr_accessor :rules #: Array[Rule] + attr_accessor :created_lhs_list #: Array[Lexer::Token::Base] + + # @rbs () -> void + def initialize + @rules = [] + @created_lhs_list = [] + end + + # @rbs (Rule rule) -> Array[Rule] + def add_rule(rule) + @rules << rule + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Rule? + def find_rule(token) + select_rules(@rules, token).last + end + + # @rbs (Lexer::Token::Base token) -> Rule? + def find_inline(token) + @rules.reverse.find { |rule| rule.name == token.s_value && rule.inline? } + end + + # @rbs (String lhs_s_value) -> Lexer::Token::Base? + def created_lhs(lhs_s_value) + @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } + end + + # @rbs () -> Array[Rule] + def redefined_rules + @rules.select { |rule| @rules.count { |r| r.name == rule.name && r.required_parameters_count == rule.required_parameters_count } > 1 } + end + + private + + # @rbs (Array[Rule] rules, Lexer::Token::InstantiateRule token) -> Array[Rule] + def select_rules(rules, token) + rules = reject_inline_rules(rules) + rules = select_rules_by_name(rules, token.rule_name) + rules = rules.select { |rule| rule.required_parameters_count == token.args_count } + if rules.empty? + raise "Invalid number of arguments. `#{token.rule_name}`" + else + rules + end + end + + # @rbs (Array[Rule] rules) -> Array[Rule] + def reject_inline_rules(rules) + rules.reject(&:inline?) + end + + # @rbs (Array[Rule] rules, String rule_name) -> Array[Rule] + def select_rules_by_name(rules, rule_name) + rules = rules.select { |rule| rule.name == rule_name } + if rules.empty? + raise "Parameterized rule does not exist. `#{rule_name}`" + else + rules + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb new file mode 100644 index 0000000000..663de49100 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rhs + attr_accessor :symbols #: Array[Lexer::Token::Base] + attr_accessor :user_code #: Lexer::Token::UserCode? + attr_accessor :precedence_sym #: Grammar::Symbol? + + # @rbs () -> void + def initialize + @symbols = [] + @user_code = nil + @precedence_sym = nil + end + + # @rbs (Grammar::Binding bindings) -> Lexer::Token::UserCode? + def resolve_user_code(bindings) + return unless user_code + + resolved = Lexer::Token::UserCode.new(s_value: user_code.s_value, location: user_code.location) + var_to_arg = {} #: Hash[String, String] + symbols.each do |sym| + resolved_sym = bindings.resolve_symbol(sym) + if resolved_sym != sym + var_to_arg[sym.s_value] = resolved_sym.s_value + end + end + + var_to_arg.each do |var, arg| + resolved.references.each do |ref| + if ref.name == var + ref.name = arg + end + end + end + + return resolved + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rule.rb b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb new file mode 100644 index 0000000000..7048be3cff --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb @@ -0,0 +1,36 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rule + attr_reader :name #: String + attr_reader :parameters #: Array[Lexer::Token::Base] + attr_reader :rhs #: Array[Rhs] + attr_reader :required_parameters_count #: Integer + attr_reader :tag #: Lexer::Token::Tag? + + # @rbs (String name, Array[Lexer::Token::Base] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void + def initialize(name, parameters, rhs, tag: nil, is_inline: false) + @name = name + @parameters = parameters + @rhs = rhs + @tag = tag + @is_inline = is_inline + @required_parameters_count = parameters.count + end + + # @rbs () -> String + def to_s + "#{@name}(#{@parameters.map(&:s_value).join(', ')})" + end + + # @rbs () -> bool + def inline? + @is_inline + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb deleted file mode 100644 index d371805f4b..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule.rb +++ /dev/null @@ -1,3 +0,0 @@ -require_relative 'parameterizing_rule/resolver' -require_relative 'parameterizing_rule/rhs' -require_relative 'parameterizing_rule/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb deleted file mode 100644 index f5de9d0bf3..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/resolver.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRule - class Resolver - attr_accessor :created_lhs_list - - def initialize - @rules = [] - @created_lhs_list = [] - end - - def add_parameterizing_rule(rule) - @rules << rule - end - - def defined?(token) - !select_rules(token).empty? - end - - def find(token) - select_rules(token).last - end - - def created_lhs(lhs_s_value) - @created_lhs_list.select { |created_lhs| created_lhs.s_value == lhs_s_value }.last - end - - private - - def select_rules(token) - @rules.select do |rule| - rule.name == token.rule_name && - rule.required_parameters_count == token.args_count - end - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb deleted file mode 100644 index 7f50be873c..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rhs.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRule - class Rhs - attr_accessor :symbols, :user_code, :precedence_sym - - def initialize - @symbols = [] - @user_code = nil - @precedence_sym = nil - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb deleted file mode 100644 index 9c1d46e4f5..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rule/rule.rb +++ /dev/null @@ -1,16 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRule - class Rule - attr_reader :name, :parameters, :rhs_list, :required_parameters_count - - def initialize(name, parameters, rhs_list) - @name = name - @parameters = parameters - @rhs_list = rhs_list - @required_parameters_count = parameters.count - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder.rb deleted file mode 100644 index 20950b9b36..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder.rb +++ /dev/null @@ -1,60 +0,0 @@ -require 'lrama/grammar/parameterizing_rules/builder/base' -require 'lrama/grammar/parameterizing_rules/builder/list' -require 'lrama/grammar/parameterizing_rules/builder/nonempty_list' -require 'lrama/grammar/parameterizing_rules/builder/option' -require 'lrama/grammar/parameterizing_rules/builder/separated_nonempty_list' -require 'lrama/grammar/parameterizing_rules/builder/separated_list' - -module Lrama - class Grammar - class ParameterizingRules - # Builder for parameterizing rules - class Builder - RULES = { - option: Lrama::Grammar::ParameterizingRules::Builder::Option, - "?": Lrama::Grammar::ParameterizingRules::Builder::Option, - nonempty_list: Lrama::Grammar::ParameterizingRules::Builder::NonemptyList, - "+": Lrama::Grammar::ParameterizingRules::Builder::NonemptyList, - list: Lrama::Grammar::ParameterizingRules::Builder::List, - "*": Lrama::Grammar::ParameterizingRules::Builder::List, - separated_nonempty_list: Lrama::Grammar::ParameterizingRules::Builder::SeparatedNonemptyList, - separated_list: Lrama::Grammar::ParameterizingRules::Builder::SeparatedList, - } - - def initialize(token, rule_counter, lhs_tag, user_code, precedence_sym, line) - @token = token - @key = token.s_value.to_sym - @rule_counter = rule_counter - @lhs_tag = lhs_tag - @user_code = user_code - @precedence_sym = precedence_sym - @line = line - @builder = nil - end - - def build - create_builder - @builder.build - end - - def build_token - create_builder - @builder.build_token - end - - private - - def create_builder - unless @builder - validate_key! - @builder = RULES[@key].new(@token, @rule_counter, @lhs_tag, @user_code, @precedence_sym, @line) - end - end - - def validate_key! - raise "Parameterizing rule does not exist. `#{@key}`" unless RULES.key?(@key) - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/base.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/base.rb deleted file mode 100644 index 5787714f0c..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/base.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRules - class Builder - # Base class for parameterizing rules builder - class Base - attr_reader :build_token - - def initialize(token, rule_counter, lhs_tag, user_code, precedence_sym, line) - @args = token.args - @token = @args.first - @rule_counter = rule_counter - @lhs_tag = lhs_tag - @user_code = user_code - @precedence_sym = precedence_sym - @line = line - @expected_argument_num = 1 - @build_token = nil - end - - def build - raise NotImplementedError - end - - private - - def validate_argument_number! - unless @args.count == @expected_argument_num - raise "Invalid number of arguments. expect: #{@expected_argument_num} actual: #{@args.count}" - end - end - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/list.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/list.rb deleted file mode 100644 index 248e1e7ad4..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/list.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRules - class Builder - # Builder for list of general parameterizing rules - class List < Base - - # program: list(number) - # - # => - # - # program: list_number - # list_number: ε - # list_number: list_number number - def build - validate_argument_number! - - rules = [] - @build_token = Lrama::Lexer::Token::Ident.new(s_value: "list_#{@token.s_value}") - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [@build_token, @token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules - end - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/nonempty_list.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/nonempty_list.rb deleted file mode 100644 index bcec1d823a..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/nonempty_list.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRules - class Builder - # Builder for nonempty list of general parameterizing rules - class NonemptyList < Base - - # program: nonempty_list(number) - # - # => - # - # program: nonempty_list_number - # nonempty_list_number: number - # nonempty_list_number: nonempty_list_number number - def build - validate_argument_number! - - rules = [] - @build_token = Lrama::Lexer::Token::Ident.new(s_value: "nonempty_list_#{@token.s_value}") - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [@token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [@build_token, @token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules - end - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/option.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/option.rb deleted file mode 100644 index 8be045ec30..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/option.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRules - class Builder - # Builder for option of general parameterizing rules - class Option < Base - - # program: option(number) - # - # => - # - # program: option_number - # option_number: ε - # option_number: number - def build - validate_argument_number! - - rules = [] - @build_token = Lrama::Lexer::Token::Ident.new(s_value: "option_#{@token.s_value}") - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [@token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules - end - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/separated_list.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/separated_list.rb deleted file mode 100644 index f9677cadbc..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/separated_list.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRules - class Builder - # Builder for separated list of general parameterizing rules - class SeparatedList < Base - def initialize(token, rule_counter, lhs_tag, user_code, precedence_sym, line) - super - @separator = @args[0] - @token = @args[1] - @expected_argument_num = 2 - end - - # program: separated_list(',', number) - # - # => - # - # program: separated_list_number - # separated_list_number: ε - # separated_list_number: separated_nonempty_list_number - # separated_nonempty_list_number: number - # separated_nonempty_list_number: separated_nonempty_list_number ',' number - def build - validate_argument_number! - - rules = [] - @build_token = Lrama::Lexer::Token::Ident.new(s_value: "separated_list_#{@token.s_value}") - separated_nonempty_list_token = Lrama::Lexer::Token::Ident.new(s_value: "separated_nonempty_list_#{@token.s_value}") - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [separated_nonempty_list_token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules << Rule.new(id: @rule_counter.increment, _lhs: separated_nonempty_list_token, _rhs: [@token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules << Rule.new(id: @rule_counter.increment, _lhs: separated_nonempty_list_token, _rhs: [separated_nonempty_list_token, @separator, @token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules - end - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/separated_nonempty_list.rb b/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/separated_nonempty_list.rb deleted file mode 100644 index ba6ecf24cc..0000000000 --- a/tool/lrama/lib/lrama/grammar/parameterizing_rules/builder/separated_nonempty_list.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Lrama - class Grammar - class ParameterizingRules - class Builder - # Builder for separated nonempty list of general parameterizing rules - class SeparatedNonemptyList < Base - def initialize(token, rule_counter, lhs_tag, user_code, precedence_sym, line) - super - @separator = @args[0] - @token = @args[1] - @expected_argument_num = 2 - end - - # program: separated_nonempty_list(',', number) - # - # => - # - # program: separated_nonempty_list_number - # separated_nonempty_list_number: number - # separated_nonempty_list_number: separated_nonempty_list_number ',' number - def build - validate_argument_number! - - rules = [] - @build_token = Lrama::Lexer::Token::Ident.new(s_value: "separated_nonempty_list_#{@token.s_value}") - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [@token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules << Rule.new(id: @rule_counter.increment, _lhs: @build_token, _rhs: [@build_token, @separator, @token], lhs_tag: @lhs_tag, token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line) - rules - end - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/grammar/percent_code.rb b/tool/lrama/lib/lrama/grammar/percent_code.rb index 8cbc5aef2c..9afb903056 100644 --- a/tool/lrama/lib/lrama/grammar/percent_code.rb +++ b/tool/lrama/lib/lrama/grammar/percent_code.rb @@ -1,8 +1,21 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class PercentCode - attr_reader :name, :code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @name: String + # @code: String + + attr_reader :name #: String + attr_reader :code #: String + # @rbs (String name, String code) -> void def initialize(name, code) @name = name @code = code diff --git a/tool/lrama/lib/lrama/grammar/precedence.rb b/tool/lrama/lib/lrama/grammar/precedence.rb index fed739b3c0..b4c6403372 100644 --- a/tool/lrama/lib/lrama/grammar/precedence.rb +++ b/tool/lrama/lib/lrama/grammar/precedence.rb @@ -1,11 +1,55 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar - class Precedence < Struct.new(:type, :precedence, keyword_init: true) + class Precedence < Struct.new(:type, :symbol, :precedence, :s_value, :lineno, keyword_init: true) include Comparable + # @rbs! + # type type_enum = :left | :right | :nonassoc | :precedence + # + # attr_accessor type: type_enum + # attr_accessor symbol: Grammar::Symbol + # attr_accessor precedence: Integer + # attr_accessor s_value: String + # attr_accessor lineno: Integer + # + # def initialize: (?type: type_enum, ?symbol: Grammar::Symbol, ?precedence: Integer, ?s_value: ::String, ?lineno: Integer) -> void + + attr_reader :used_by_lalr #: Array[State::ResolvedConflict] + attr_reader :used_by_ielr #: Array[State::ResolvedConflict] + # @rbs (Precedence other) -> Integer def <=>(other) self.precedence <=> other.precedence end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_lalr(resolved_conflict) + @used_by_lalr ||= [] #: Array[State::ResolvedConflict] + @used_by_lalr << resolved_conflict + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_ielr(resolved_conflict) + @used_by_ielr ||= [] #: Array[State::ResolvedConflict] + @used_by_ielr << resolved_conflict + end + + # @rbs () -> bool + def used_by? + used_by_lalr? || used_by_ielr? + end + + # @rbs () -> bool + def used_by_lalr? + !@used_by_lalr.nil? && !@used_by_lalr.empty? + end + + # @rbs () -> bool + def used_by_ielr? + !@used_by_ielr.nil? && !@used_by_ielr.empty? + end end end end diff --git a/tool/lrama/lib/lrama/grammar/printer.rb b/tool/lrama/lib/lrama/grammar/printer.rb index 8984a96e1a..490fe701db 100644 --- a/tool/lrama/lib/lrama/grammar/printer.rb +++ b/tool/lrama/lib/lrama/grammar/printer.rb @@ -1,6 +1,17 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class Printer < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + # @rbs! + # attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag] + # attr_accessor token_code: Lexer::Token::UserCode + # attr_accessor lineno: Integer + # + # def initialize: (?ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], ?token_code: Lexer::Token::UserCode, ?lineno: Integer) -> void + + # @rbs (Lexer::Token::Tag tag) -> String def translated_code(tag) Code::PrinterCode.new(type: :printer, token_code: token_code, tag: tag).translated_code end diff --git a/tool/lrama/lib/lrama/grammar/reference.rb b/tool/lrama/lib/lrama/grammar/reference.rb index 24c981298e..7e3badfecc 100644 --- a/tool/lrama/lib/lrama/grammar/reference.rb +++ b/tool/lrama/lib/lrama/grammar/reference.rb @@ -1,12 +1,28 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar # type: :dollar or :at # name: String (e.g. $$, $foo, $expr.right) - # index: Integer (e.g. $1) + # number: Integer (e.g. $1) + # index: Integer # ex_tag: "$<tag>1" (Optional) - class Reference < Struct.new(:type, :name, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + # @rbs! + # attr_accessor type: ::Symbol + # attr_accessor name: String + # attr_accessor number: Integer + # attr_accessor index: Integer + # attr_accessor ex_tag: Lexer::Token::Base? + # attr_accessor first_column: Integer + # attr_accessor last_column: Integer + # + # def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token::Base?, first_column: Integer, last_column: Integer) -> void + + # @rbs () -> (String|Integer) def value - name || index + name || number end end end diff --git a/tool/lrama/lib/lrama/grammar/rule.rb b/tool/lrama/lib/lrama/grammar/rule.rb index 2876472030..d00d6a8883 100644 --- a/tool/lrama/lib/lrama/grammar/rule.rb +++ b/tool/lrama/lib/lrama/grammar/rule.rb @@ -1,9 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar # _rhs holds original RHS element. Use rhs to refer to Symbol. class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true) - attr_accessor :original_rule + # @rbs! + # + # interface _DelegatedMethods + # def lhs: -> Grammar::Symbol + # def rhs: -> Array[Grammar::Symbol] + # end + # + # attr_accessor id: Integer + # attr_accessor _lhs: Lexer::Token::Base + # attr_accessor lhs: Grammar::Symbol + # attr_accessor lhs_tag: Lexer::Token::Tag? + # attr_accessor _rhs: Array[Lexer::Token::Base] + # attr_accessor rhs: Array[Grammar::Symbol] + # attr_accessor token_code: Lexer::Token::UserCode? + # attr_accessor position_in_original_rule_rhs: Integer + # attr_accessor nullable: bool + # attr_accessor precedence_sym: Grammar::Symbol? + # attr_accessor lineno: Integer? + # + # def initialize: ( + # ?id: Integer, ?_lhs: Lexer::Token::Base?, ?lhs: Lexer::Token::Base, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token::Base], ?rhs: Array[Grammar::Symbol], + # ?token_code: Lexer::Token::UserCode?, ?position_in_original_rule_rhs: Integer?, ?nullable: bool, + # ?precedence_sym: Grammar::Symbol?, ?lineno: Integer? + # ) -> void + + attr_accessor :original_rule #: Rule + # @rbs (Rule other) -> bool def ==(other) self.class == other.class && self.lhs == other.lhs && @@ -16,41 +45,91 @@ module Lrama self.lineno == other.lineno end - # TODO: Change this to display_name - def to_s + # @rbs () -> String + def display_name + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map {|r| r.id.s_value }.join(" ") + "#{l} -> #{r}" + end + + # @rbs () -> String + def display_name_without_action l = lhs.id.s_value - r = rhs.empty? ? "ε" : rhs.map {|r| r.id.s_value }.join(", ") + r = empty_rule? ? "ε" : rhs.map do |r| + r.id.s_value if r.first_set.any? + end.compact.join(" ") "#{l} -> #{r}" end + # @rbs () -> (RailroadDiagrams::Skip | RailroadDiagrams::Sequence) + def to_diagrams + if rhs.empty? + RailroadDiagrams::Skip.new + else + RailroadDiagrams::Sequence.new(*rhs_to_diagram) + end + end + # Used by #user_actions + # + # @rbs () -> String def as_comment l = lhs.id.s_value - r = rhs.empty? ? "%empty" : rhs.map(&:display_name).join(" ") + r = empty_rule? ? "%empty" : rhs.map(&:display_name).join(" ") "#{l}: #{r}" end + # @rbs () -> String + def with_actions + "#{display_name} {#{token_code&.s_value}}" + end + # opt_nl: ε <-- empty_rule # | '\n' <-- not empty_rule + # + # @rbs () -> bool def empty_rule? rhs.empty? end + # @rbs () -> Precedence? def precedence precedence_sym&.precedence end + # @rbs () -> bool def initial_rule? id == 0 end + # @rbs () -> String? def translated_code return nil unless token_code Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self).translated_code end + + # @rbs () -> bool + def contains_at_reference? + return false unless token_code + + token_code.references.any? {|r| r.type == :at } + end + + private + + # @rbs () -> Array[(RailroadDiagrams::Terminal | RailroadDiagrams::NonTerminal)] + def rhs_to_diagram + rhs.map do |r| + if r.term + RailroadDiagrams::Terminal.new(r.id.s_value) + else + RailroadDiagrams::NonTerminal.new(r.id.s_value) + end + end + end end end end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb index 757554f46d..34fdca6c86 100644 --- a/tool/lrama/lib/lrama/grammar/rule_builder.rb +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -1,14 +1,38 @@ -require 'lrama/grammar/parameterizing_rules/builder' +# rbs_inline: enabled +# frozen_string_literal: true module Lrama class Grammar class RuleBuilder - attr_accessor :lhs, :line - attr_reader :lhs_tag, :rhs, :user_code, :precedence_sym - - def initialize(rule_counter, midrule_action_counter, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @position_in_original_rule_rhs: Integer? + # @skip_preprocess_references: bool + # @rules: Array[Rule] + # @rule_builders_for_parameterized: Array[RuleBuilder] + # @rule_builders_for_derived_rules: Array[RuleBuilder] + # @parameterized_rules: Array[Rule] + # @midrule_action_rules: Array[Rule] + # @replaced_rhs: Array[Lexer::Token::Base]? + + attr_accessor :lhs #: Lexer::Token::Base? + attr_accessor :line #: Integer? + attr_reader :rule_counter #: Counter + attr_reader :midrule_action_counter #: Counter + attr_reader :parameterized_resolver #: Grammar::Parameterized::Resolver + attr_reader :lhs_tag #: Lexer::Token::Tag? + attr_reader :rhs #: Array[Lexer::Token::Base] + attr_reader :user_code #: Lexer::Token::UserCode? + attr_reader :precedence_sym #: Grammar::Symbol? + + # @rbs (Counter rule_counter, Counter midrule_action_counter, Grammar::Parameterized::Resolver parameterized_resolver, ?Integer position_in_original_rule_rhs, ?lhs_tag: Lexer::Token::Tag?, ?skip_preprocess_references: bool) -> void + def initialize(rule_counter, midrule_action_counter, parameterized_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) @rule_counter = rule_counter @midrule_action_counter = midrule_action_counter + @parameterized_resolver = parameterized_resolver @position_in_original_rule_rhs = position_in_original_rule_rhs @skip_preprocess_references = skip_preprocess_references @@ -18,69 +42,84 @@ module Lrama @user_code = nil @precedence_sym = nil @line = nil - @rule_builders_for_parameterizing_rules = [] + @rules = [] + @rule_builders_for_parameterized = [] @rule_builders_for_derived_rules = [] + @parameterized_rules = [] + @midrule_action_rules = [] end + # @rbs (Lexer::Token::Base rhs) -> void def add_rhs(rhs) - if !@line - @line = rhs.line - end + @line ||= rhs.line flush_user_code @rhs << rhs end + # @rbs (Lexer::Token::UserCode? user_code) -> void def user_code=(user_code) - if !@line - @line = user_code&.line - end + @line ||= user_code&.line flush_user_code @user_code = user_code end + # @rbs (Grammar::Symbol? precedence_sym) -> void def precedence_sym=(precedence_sym) flush_user_code @precedence_sym = precedence_sym end + # @rbs () -> void def complete_input freeze_rhs end - def setup_rules(parameterizing_rule_resolver) + # @rbs () -> void + def setup_rules preprocess_references unless @skip_preprocess_references - process_rhs(parameterizing_rule_resolver) + process_rhs + resolve_inline_rules build_rules end + # @rbs () -> Array[Grammar::Rule] def rules - @parameterizing_rules + @old_parameterizing_rules + @midrule_action_rules + @rules + @parameterized_rules + @midrule_action_rules + @rules + end + + # @rbs () -> bool + def has_inline_rules? + rhs.any? { |token| @parameterized_resolver.find_inline(token) } end private + # @rbs () -> void def freeze_rhs @rhs.freeze end + # @rbs () -> void def preprocess_references numberize_references end + # @rbs () -> void def build_rules - tokens = @replaced_rhs + tokens = @replaced_rhs #: Array[Lexer::Token::Base] + return if tokens.any? { |t| @parameterized_resolver.find_inline(t) } rule = Rule.new( id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code, position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line ) @rules = [rule] - @parameterizing_rules = @rule_builders_for_parameterizing_rules.map do |rule_builder| + @parameterized_rules = @rule_builders_for_parameterized.map do |rule_builder| rule_builder.rules end.flatten @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder| @@ -93,79 +132,82 @@ module Lrama # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on. # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`. - def process_rhs(parameterizing_rule_resolver) + # + # @rbs () -> void + def process_rhs return if @replaced_rhs - @replaced_rhs = [] - @old_parameterizing_rules = [] + replaced_rhs = [] #: Array[Lexer::Token::Base] rhs.each_with_index do |token, i| case token when Lrama::Lexer::Token::Char - @replaced_rhs << token + replaced_rhs << token when Lrama::Lexer::Token::Ident - @replaced_rhs << token + replaced_rhs << token when Lrama::Lexer::Token::InstantiateRule - if parameterizing_rule_resolver.defined?(token) - parameterizing_rule = parameterizing_rule_resolver.find(token) - raise "Unexpected token. #{token}" unless parameterizing_rule - - bindings = Binding.new(parameterizing_rule, token.args) - lhs_s_value = lhs_s_value(token, bindings) - if (created_lhs = parameterizing_rule_resolver.created_lhs(lhs_s_value)) - @replaced_rhs << created_lhs - else - lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) - @replaced_rhs << lhs_token - parameterizing_rule_resolver.created_lhs_list << lhs_token - parameterizing_rule.rhs_list.each do |r| - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, i, lhs_tag: token.lhs_tag, skip_preprocess_references: true) - rule_builder.lhs = lhs_token - r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } - rule_builder.line = line - rule_builder.user_code = r.user_code - rule_builder.precedence_sym = r.precedence_sym - rule_builder.complete_input - rule_builder.setup_rules(parameterizing_rule_resolver) - @rule_builders_for_parameterizing_rules << rule_builder - end - end + parameterized_rule = @parameterized_resolver.find_rule(token) + raise "Unexpected token. #{token}" unless parameterized_rule + + bindings = Binding.new(parameterized_rule.parameters, token.args) + lhs_s_value = bindings.concatenated_args_str(token) + if (created_lhs = @parameterized_resolver.created_lhs(lhs_s_value)) + replaced_rhs << created_lhs else - # TODO: Delete when the standard library will defined as a grammar file. - parameterizing_rule = ParameterizingRules::Builder.new(token, @rule_counter, token.lhs_tag, user_code, precedence_sym, line) - @old_parameterizing_rules = @old_parameterizing_rules + parameterizing_rule.build - @replaced_rhs << parameterizing_rule.build_token + lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) + replaced_rhs << lhs_token + @parameterized_resolver.created_lhs_list << lhs_token + parameterized_rule.rhs.each do |r| + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, lhs_tag: token.lhs_tag || parameterized_rule.tag) + rule_builder.lhs = lhs_token + r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } + rule_builder.line = line + rule_builder.precedence_sym = r.precedence_sym + rule_builder.user_code = r.resolve_user_code(bindings) + rule_builder.complete_input + rule_builder.setup_rules + @rule_builders_for_parameterized << rule_builder + end end when Lrama::Lexer::Token::UserCode prefix = token.referred ? "@" : "$@" + tag = token.tag || lhs_tag new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) - @replaced_rhs << new_token + replaced_rhs << new_token - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, i, lhs_tag: lhs_tag, skip_preprocess_references: true) + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, i, lhs_tag: tag, skip_preprocess_references: true) rule_builder.lhs = new_token rule_builder.user_code = token rule_builder.complete_input - rule_builder.setup_rules(parameterizing_rule_resolver) + rule_builder.setup_rules @rule_builders_for_derived_rules << rule_builder + when Lrama::Lexer::Token::Empty + # Noop else raise "Unexpected token. #{token}" end end + + @replaced_rhs = replaced_rhs end - def lhs_s_value(token, bindings) - s_values = token.args.map do |arg| - resolved = bindings.resolve_symbol(arg) - if resolved.is_a?(Lexer::Token::InstantiateRule) - [resolved.s_value, resolved.args.map(&:s_value)] - else - resolved.s_value + # @rbs () -> void + def resolve_inline_rules + while @rule_builders_for_parameterized.any?(&:has_inline_rules?) do + @rule_builders_for_parameterized = @rule_builders_for_parameterized.flat_map do |rule_builder| + if rule_builder.has_inline_rules? + inlined_builders = Inline::Resolver.new(rule_builder).resolve + inlined_builders.each { |builder| builder.setup_rules } + inlined_builders + else + rule_builder + end end end - "#{token.rule_name}_#{s_values.join('_')}" end + # @rbs () -> void def numberize_references # Bison n'th component is 1-origin (rhs + [user_code]).compact.each.with_index(1) do |token, i| @@ -173,11 +215,15 @@ module Lrama token.references.each do |ref| ref_name = ref.name - if ref_name && ref_name != '$' - if lhs.referred_by?(ref_name) + + if ref_name + if ref_name == '$' ref.name = '$' else - candidates = rhs.each_with_index.select {|token, i| token.referred_by?(ref_name) } + candidates = ([lhs] + rhs).each_with_index.select do |token, _i| + # @type var token: Lexer::Token::Base + token.referred_by?(ref_name) + end if candidates.size >= 2 token.invalid_ref(ref, "Referring symbol `#{ref_name}` is duplicated.") @@ -187,10 +233,18 @@ module Lrama token.invalid_ref(ref, "Referring symbol `#{ref_name}` is not found.") end - ref.index = referring_symbol[1] + 1 + if referring_symbol[1] == 0 # Refers to LHS + ref.name = '$' + else + ref.number = referring_symbol[1] + end end end + if ref.number + ref.index = ref.number + end + # TODO: Need to check index of @ too? next if ref.type == :at @@ -204,6 +258,7 @@ module Lrama end end + # @rbs () -> void def flush_user_code if (c = @user_code) @rhs << c diff --git a/tool/lrama/lib/lrama/grammar/stdlib.y b/tool/lrama/lib/lrama/grammar/stdlib.y new file mode 100644 index 0000000000..dd397c9e08 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/stdlib.y @@ -0,0 +1,142 @@ +/********************************************************************** + + stdlib.y + + This is lrama's standard library. It provides a number of + parameterized rule definitions, such as options and lists, + that should be useful in a number of situations. + +**********************************************************************/ + +%% + +// ------------------------------------------------------------------- +// Options + +/* + * program: option(X) + * + * => + * + * program: option_X + * option_X: %empty + * option_X: X + */ +%rule option(X) + : /* empty */ + | X + ; + + +/* + * program: ioption(X) + * + * => + * + * program: %empty + * program: X + */ +%rule %inline ioption(X) + : /* empty */ + | X + ; + +// ------------------------------------------------------------------- +// Sequences + +/* + * program: preceded(opening, X) + * + * => + * + * program: preceded_opening_X + * preceded_opening_X: opening X + */ +%rule preceded(opening, X) + : opening X { $$ = $2; } + ; + +/* + * program: terminated(X, closing) + * + * => + * + * program: terminated_X_closing + * terminated_X_closing: X closing + */ +%rule terminated(X, closing) + : X closing { $$ = $1; } + ; + +/* + * program: delimited(opening, X, closing) + * + * => + * + * program: delimited_opening_X_closing + * delimited_opening_X_closing: opening X closing + */ +%rule delimited(opening, X, closing) + : opening X closing { $$ = $2; } + ; + +// ------------------------------------------------------------------- +// Lists + +/* + * program: list(X) + * + * => + * + * program: list_X + * list_X: %empty + * list_X: list_X X + */ +%rule list(X) + : /* empty */ + | list(X) X + ; + +/* + * program: nonempty_list(X) + * + * => + * + * program: nonempty_list_X + * nonempty_list_X: X + * nonempty_list_X: nonempty_list_X X + */ +%rule nonempty_list(X) + : X + | nonempty_list(X) X + ; + +/* + * program: separated_nonempty_list(separator, X) + * + * => + * + * program: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separated_nonempty_list_separator_X separator X + */ +%rule separated_nonempty_list(separator, X) + : X + | separated_nonempty_list(separator, X) separator X + ; + +/* + * program: separated_list(separator, X) + * + * => + * + * program: separated_list_separator_X + * separated_list_separator_X: option_separated_nonempty_list_separator_X + * option_separated_nonempty_list_separator_X: %empty + * option_separated_nonempty_list_separator_X: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separator separated_nonempty_list_separator_X X + */ +%rule separated_list(separator, X) + : option(separated_nonempty_list(separator, X)) + ; diff --git a/tool/lrama/lib/lrama/grammar/symbol.rb b/tool/lrama/lib/lrama/grammar/symbol.rb index df866db716..07aee0c0a2 100644 --- a/tool/lrama/lib/lrama/grammar/symbol.rb +++ b/tool/lrama/lib/lrama/grammar/symbol.rb @@ -1,17 +1,36 @@ +# rbs_inline: enabled +# frozen_string_literal: true + # Symbol is both of nterm and term # `number` is both for nterm and term # `token_id` is tokentype for term, internal sequence number for nterm # -# TODO: Add validation for ASCII code range for Token::Char module Lrama class Grammar class Symbol - attr_accessor :id, :alias_name, :tag, :number, :token_id, :nullable, :precedence, :printer, :error_token, :first_set, :first_set_bitmap - attr_reader :term - attr_writer :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + attr_accessor :id #: Lexer::Token::Base + attr_accessor :alias_name #: String? + attr_reader :number #: Integer + attr_accessor :number_bitmap #: Bitmap::bitmap + attr_accessor :tag #: Lexer::Token::Tag? + attr_accessor :token_id #: Integer + attr_accessor :nullable #: bool + attr_accessor :precedence #: Precedence? + attr_accessor :printer #: Printer? + attr_accessor :destructor #: Destructor? + attr_accessor :error_token #: ErrorToken + attr_accessor :first_set #: Set[Grammar::Symbol] + attr_accessor :first_set_bitmap #: Bitmap::bitmap + attr_reader :term #: bool + attr_writer :eof_symbol #: bool + attr_writer :error_symbol #: bool + attr_writer :undef_symbol #: bool + attr_writer :accept_symbol #: bool - def initialize(id:, alias_name: nil, number: nil, tag: nil, term:, token_id: nil, nullable: nil, precedence: nil, printer: nil) + # @rbs (id: Lexer::Token::Base, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token::Tag?, + # ?token_id: Integer?, ?nullable: bool?, ?precedence: Precedence?, ?printer: Printer?) -> void + def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil) @id = id @alias_name = alias_name @number = number @@ -21,79 +40,108 @@ module Lrama @nullable = nullable @precedence = precedence @printer = printer + @destructor = destructor end + # @rbs (Integer) -> void + def number=(number) + @number = number + @number_bitmap = Bitmap::from_integer(number) + end + + # @rbs () -> bool def term? term end + # @rbs () -> bool def nterm? !term end + # @rbs () -> bool def eof_symbol? !!@eof_symbol end + # @rbs () -> bool def error_symbol? !!@error_symbol end + # @rbs () -> bool def undef_symbol? !!@undef_symbol end + # @rbs () -> bool def accept_symbol? !!@accept_symbol end + # @rbs () -> bool + def midrule? + return false if term? + + name.include?("$") || name.include?("@") + end + + # @rbs () -> String + def name + id.s_value + end + + # @rbs () -> String def display_name - alias_name || id.s_value + alias_name || name end # name for yysymbol_kind_t # # See: b4_symbol_kind_base # @type var name: String + # @rbs () -> String def enum_name case when accept_symbol? - name = "YYACCEPT" + res = "YYACCEPT" when eof_symbol? - name = "YYEOF" + res = "YYEOF" when term? && id.is_a?(Lrama::Lexer::Token::Char) - name = number.to_s + display_name + res = number.to_s + display_name when term? && id.is_a?(Lrama::Lexer::Token::Ident) - name = id.s_value - when nterm? && (id.s_value.include?("$") || id.s_value.include?("@")) - name = number.to_s + id.s_value + res = name + when midrule? + res = number.to_s + name when nterm? - name = id.s_value + res = name else raise "Unexpected #{self}" end - "YYSYMBOL_" + name.gsub(/\W+/, "_") + "YYSYMBOL_" + res.gsub(/\W+/, "_") end # comment for yysymbol_kind_t + # + # @rbs () -> String? def comment case when accept_symbol? # YYSYMBOL_YYACCEPT - id.s_value + name when eof_symbol? # YYEOF alias_name when (term? && 0 < token_id && token_id < 128) # YYSYMBOL_3_backslash_, YYSYMBOL_14_ - alias_name || id.s_value - when id.s_value.include?("$") || id.s_value.include?("@") + display_name + when midrule? # YYSYMBOL_21_1 - id.s_value + name else # YYSYMBOL_keyword_class, YYSYMBOL_strings_1 - alias_name || id.s_value + display_name end end end diff --git a/tool/lrama/lib/lrama/grammar/symbols.rb b/tool/lrama/lib/lrama/grammar/symbols.rb new file mode 100644 index 0000000000..337241d1b2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative "symbols/resolver" diff --git a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb new file mode 100644 index 0000000000..085a835d28 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb @@ -0,0 +1,362 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Symbols + class Resolver + # @rbs! + # + # interface _DelegatedMethods + # def symbols: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def add_nterm: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + # def add_term: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + # def find_symbol_by_number!: (Integer number) -> Grammar::Symbol + # def find_symbol_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol + # def token_to_symbol: (Lexer::Token::Base token) -> Grammar::Symbol + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def fill_nterm_type: (Array[Grammar::Type] types) -> void + # def fill_symbol_number: () -> void + # def fill_printer: (Array[Grammar::Printer] printers) -> void + # def fill_destructor: (Array[Destructor] destructors) -> (Destructor | bot) + # def fill_error_token: (Array[Grammar::ErrorToken] error_tokens) -> void + # def sort_by_number!: () -> Array[Grammar::Symbol] + # end + # + # @symbols: Array[Grammar::Symbol]? + # @number: Integer + # @used_numbers: Hash[Integer, bool] + + attr_reader :terms #: Array[Grammar::Symbol] + attr_reader :nterms #: Array[Grammar::Symbol] + + # @rbs () -> void + def initialize + @terms = [] + @nterms = [] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols + @symbols ||= (@terms + @nterms) + end + + # @rbs () -> Array[Grammar::Symbol] + def sort_by_number! + symbols.sort_by!(&:number) + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) + if token_id && (sym = find_symbol_by_token_id(token_id)) + if replace + sym.id = id + sym.alias_name = alias_name + sym.tag = tag + end + + return sym + end + + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + term = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: true, token_id: token_id, nullable: false + ) + @terms << term + term + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + def add_nterm(id:, alias_name: nil, tag: nil) + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + nterm = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: false, token_id: nil, nullable: nil, + ) + @nterms << nterm + nterm + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_term_by_s_value(s_value) + terms.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_symbol_by_s_value(s_value) + symbols.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol + def find_symbol_by_s_value!(s_value) + find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`") + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol? + def find_symbol_by_id(id) + symbols.find do |s| + s.id == id || s.alias_name == id.s_value + end + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_symbol_by_id!(id) + find_symbol_by_id(id) || (raise "Symbol not found. #{id}") + end + + # @rbs (Integer token_id) -> Grammar::Symbol? + def find_symbol_by_token_id(token_id) + symbols.find {|s| s.token_id == token_id } + end + + # @rbs (Integer number) -> Grammar::Symbol + def find_symbol_by_number!(number) + sym = symbols[number] + + raise "Symbol not found. number: `#{number}`" unless sym + raise "[BUG] Symbol number mismatch. #{number}, #{sym}" if sym.number != number + + sym + end + + # @rbs () -> void + def fill_symbol_number + # YYEMPTY = -2 + # YYEOF = 0 + # YYerror = 1 + # YYUNDEF = 2 + @number = 3 + fill_terms_number + fill_nterms_number + end + + # @rbs (Array[Grammar::Type] types) -> void + def fill_nterm_type(types) + types.each do |type| + nterm = find_nterm_by_id!(type.id) + nterm.tag = type.tag + end + end + + # @rbs (Array[Grammar::Printer] printers) -> void + def fill_printer(printers) + symbols.each do |sym| + printers.each do |printer| + printer.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.printer = printer if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.printer = printer if sym.tag == ident_or_tag + else + raise "Unknown token type. #{printer}" + end + end + end + end + end + + # @rbs (Array[Destructor] destructors) -> (Array[Grammar::Symbol] | bot) + def fill_destructor(destructors) + symbols.each do |sym| + destructors.each do |destructor| + destructor.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.destructor = destructor if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.destructor = destructor if sym.tag == ident_or_tag + else + raise "Unknown token type. #{destructor}" + end + end + end + end + end + + # @rbs (Array[Grammar::ErrorToken] error_tokens) -> void + def fill_error_token(error_tokens) + symbols.each do |sym| + error_tokens.each do |token| + token.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.error_token = token if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.error_token = token if sym.tag == ident_or_tag + else + raise "Unknown token type. #{token}" + end + end + end + end + end + + # @rbs (Lexer::Token::Base token) -> Grammar::Symbol + def token_to_symbol(token) + case token + when Lrama::Lexer::Token::Base + find_symbol_by_id!(token) + else + raise "Unknown class: #{token}" + end + end + + # @rbs () -> void + def validate! + validate_number_uniqueness! + validate_alias_name_uniqueness! + validate_symbols! + end + + private + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_nterm_by_id!(id) + @nterms.find do |s| + s.id == id + end || (raise "Symbol not found. #{id}") + end + + # @rbs () -> void + def fill_terms_number + # Character literal in grammar file has + # token id corresponding to ASCII code by default, + # so start token_id from 256. + token_id = 256 + + @terms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + # If id is Token::Char, it uses ASCII code + if sym.token_id.nil? + if sym.id.is_a?(Lrama::Lexer::Token::Char) + # Ignore ' on the both sides + case sym.id.s_value[1..-2] + when "\\b" + sym.token_id = 8 + when "\\f" + sym.token_id = 12 + when "\\n" + sym.token_id = 10 + when "\\r" + sym.token_id = 13 + when "\\t" + sym.token_id = 9 + when "\\v" + sym.token_id = 11 + when "\"" + sym.token_id = 34 + when "'" + sym.token_id = 39 + when "\\\\" + sym.token_id = 92 + when /\A\\(\d+)\z/ + unless (id = Integer($1, 8)).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + when /\A(.)\z/ + unless (id = $1&.bytes&.first).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + else + raise "Unknown Char s_value #{sym}" + end + else + sym.token_id = token_id + token_id += 1 + end + end + end + end + + # @rbs () -> void + def fill_nterms_number + token_id = 0 + + @nterms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + if sym.token_id.nil? + sym.token_id = token_id + token_id += 1 + end + end + end + + # @rbs () -> Hash[Integer, bool] + def used_numbers + return @used_numbers if defined?(@used_numbers) + + @used_numbers = {} + symbols.map(&:number).each do |n| + @used_numbers[n] = true + end + @used_numbers + end + + # @rbs () -> void + def validate_number_uniqueness! + invalid = symbols.group_by(&:number).select do |number, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol number is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_alias_name_uniqueness! + invalid = symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol alias name is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_symbols! + symbols.each { |sym| sym.id.validate } + errors = symbols.map { |sym| sym.id.errors }.flatten.compact + return if errors.empty? + + raise errors.join("\n") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/type.rb b/tool/lrama/lib/lrama/grammar/type.rb index 6b4b0961a1..c631769447 100644 --- a/tool/lrama/lib/lrama/grammar/type.rb +++ b/tool/lrama/lib/lrama/grammar/type.rb @@ -1,13 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar class Type - attr_reader :id, :tag + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @id: Lexer::Token::Base + # @tag: Lexer::Token::Tag + + attr_reader :id #: Lexer::Token::Base + attr_reader :tag #: Lexer::Token::Tag + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> void def initialize(id:, tag:) @id = id @tag = tag end + # @rbs (Grammar::Type other) -> bool def ==(other) self.class == other.class && self.id == other.id && diff --git a/tool/lrama/lib/lrama/grammar/union.rb b/tool/lrama/lib/lrama/grammar/union.rb index 854bffb5c1..774cc66fc6 100644 --- a/tool/lrama/lib/lrama/grammar/union.rb +++ b/tool/lrama/lib/lrama/grammar/union.rb @@ -1,6 +1,19 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Grammar - class Union < Struct.new(:code, :lineno, keyword_init: true) + class Union + attr_reader :code #: Grammar::Code::NoReferenceCode + attr_reader :lineno #: Integer + + # @rbs (code: Grammar::Code::NoReferenceCode, lineno: Integer) -> void + def initialize(code:, lineno:) + @code = code + @lineno = lineno + end + + # @rbs () -> String def braces_less_code # Braces is already removed by lexer code.s_value diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb index 746d50cee5..ce98b505a7 100644 --- a/tool/lrama/lib/lrama/lexer.rb +++ b/tool/lrama/lib/lrama/lexer.rb @@ -1,18 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + require "strscan" -require "lrama/lexer/grammar_file" -require "lrama/lexer/location" -require "lrama/lexer/token" + +require_relative "lexer/grammar_file" +require_relative "lexer/location" +require_relative "lexer/token" module Lrama class Lexer - attr_reader :head_line, :head_column, :line - attr_accessor :status, :end_symbol + # @rbs! + # + # type token = lexer_token | c_token + # + # type lexer_token = [String, Token::Token] | + # [::Symbol, Token::Tag] | + # [::Symbol, Token::Char] | + # [::Symbol, Token::Str] | + # [::Symbol, Token::Int] | + # [::Symbol, Token::Ident] + # + # type c_token = [:C_DECLARATION, Token::UserCode] - SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'] + attr_reader :head_line #: Integer + attr_reader :head_column #: Integer + attr_reader :line #: Integer + attr_accessor :status #: :initial | :c_declaration + attr_accessor :end_symbol #: String? + + SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze #: Array[String] PERCENT_TOKENS = %w( %union %token %type + %nterm %left %right %nonassoc @@ -20,17 +41,29 @@ module Lrama %define %require %printer + %destructor %lex-param %parse-param %initial-action %precedence %prec %error-token + %before-reduce + %after-reduce + %after-shift-error-token + %after-shift + %after-pop-stack %empty %code %rule - ) + %no-stdlib + %inline + %locations + %categories + %start + ).freeze #: Array[String] + # @rbs (GrammarFile grammar_file) -> void def initialize(grammar_file) @grammar_file = grammar_file @scanner = StringScanner.new(grammar_file.text) @@ -40,6 +73,7 @@ module Lrama @end_symbol = nil end + # @rbs () -> token? def next_token case @status when :initial @@ -49,10 +83,12 @@ module Lrama end end + # @rbs () -> Integer def column @scanner.pos - @head end + # @rbs () -> Location def location Location.new( grammar_file: @grammar_file, @@ -61,13 +97,14 @@ module Lrama ) end + # @rbs () -> lexer_token? def lex_token - while !@scanner.eos? do + until @scanner.eos? do case when @scanner.scan(/\n/) newline when @scanner.scan(/\s+/) - # noop + @scanner.matched.count("\n").times { newline } when @scanner.scan(/\/\*/) lex_comment when @scanner.scan(/\/\/.*(?<newline>\n)?/) @@ -83,11 +120,11 @@ module Lrama when @scanner.eos? return when @scanner.scan(/#{SYMBOLS.join('|')}/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/[\?\+\*]/) - return [@scanner.matched, @scanner.matched] + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/<\w+>/) return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/'.'/) @@ -95,9 +132,9 @@ module Lrama when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/".*?"/) - return [:STRING, %Q(#{@scanner.matched})] + return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] when @scanner.scan(/\d+/) - return [:INTEGER, Integer(@scanner.matched)] + return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)] when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location) type = @@ -108,69 +145,72 @@ module Lrama end return [type, token] else - raise ParseError, "Unexpected token: #{@scanner.peek(10).chomp}." + raise ParseError, location.generate_error_message("Unexpected token") # steep:ignore UnknownConstant end end + # @rbs () -> c_token def lex_c_code nested = 0 - code = '' + code = +'' reset_first_position - while !@scanner.eos? do + until @scanner.eos? do case when @scanner.scan(/{/) - code += @scanner.matched + code << @scanner.matched nested += 1 when @scanner.scan(/}/) if nested == 0 && @end_symbol == '}' @scanner.unscan return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] else - code += @scanner.matched + code << @scanner.matched nested -= 1 end when @scanner.check(/#{@end_symbol}/) return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] when @scanner.scan(/\n/) - code += @scanner.matched + code << @scanner.matched newline when @scanner.scan(/".*?"/) - code += %Q(#{@scanner.matched}) + code << %Q(#{@scanner.matched}) @line += @scanner.matched.count("\n") when @scanner.scan(/'.*?'/) - code += %Q(#{@scanner.matched}) + code << %Q(#{@scanner.matched}) when @scanner.scan(/[^\"'\{\}\n]+/) - code += @scanner.matched - when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) - code += @scanner.matched + code << @scanner.matched + when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) # steep:ignore + code << @scanner.matched else - code += @scanner.getch + code << @scanner.getch end end - raise ParseError, "Unexpected code: #{code}." + raise ParseError, location.generate_error_message("Unexpected code: #{code}") # steep:ignore UnknownConstant end private + # @rbs () -> void def lex_comment - while !@scanner.eos? do + until @scanner.eos? do case - when @scanner.scan(/\n/) - newline - when @scanner.scan(/\*\//) + when @scanner.scan_until(/[\s\S]*?\*\//) + @scanner.matched.count("\n").times { newline } return - else - @scanner.getch + when @scanner.scan_until(/\n/) + newline end end end + # @rbs () -> void def reset_first_position @head_line = line @head_column = column end + # @rbs () -> void def newline @line += 1 @head = @scanner.pos diff --git a/tool/lrama/lib/lrama/lexer/grammar_file.rb b/tool/lrama/lib/lrama/lexer/grammar_file.rb index 6be0767004..37e82ff18d 100644 --- a/tool/lrama/lib/lrama/lexer/grammar_file.rb +++ b/tool/lrama/lib/lrama/lexer/grammar_file.rb @@ -1,18 +1,37 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Lexer class GrammarFile - attr_reader :path, :text + class Text < String + # @rbs () -> String + def inspect + length <= 50 ? super : "#{self[0..47]}...".inspect + end + end + + attr_reader :path #: String + attr_reader :text #: String + # @rbs (String path, String text) -> void def initialize(path, text) @path = path - @text = text.freeze + @text = Text.new(text).freeze + end + + # @rbs () -> String + def inspect + "<#{self.class}: @path=#{path}, @text=#{text.inspect}>" end + # @rbs (GrammarFile other) -> bool def ==(other) self.class == other.class && self.path == other.path end + # @rbs () -> Array[String] def lines @lines ||= text.split("\n") end diff --git a/tool/lrama/lib/lrama/lexer/location.rb b/tool/lrama/lib/lrama/lexer/location.rb index aefce3e16b..4465576d53 100644 --- a/tool/lrama/lib/lrama/lexer/location.rb +++ b/tool/lrama/lib/lrama/lexer/location.rb @@ -1,8 +1,16 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Lexer class Location - attr_reader :grammar_file, :first_line, :first_column, :last_line, :last_column + attr_reader :grammar_file #: GrammarFile + attr_reader :first_line #: Integer + attr_reader :first_column #: Integer + attr_reader :last_line #: Integer + attr_reader :last_column #: Integer + # @rbs (grammar_file: GrammarFile, first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:) @grammar_file = grammar_file @first_line = first_line @@ -11,6 +19,7 @@ module Lrama @last_column = last_column end + # @rbs (Location other) -> bool def ==(other) self.class == other.class && self.grammar_file == other.grammar_file && @@ -20,6 +29,7 @@ module Lrama self.last_column == other.last_column end + # @rbs (Integer left, Integer right) -> Location def partial_location(left, right) offset = -first_column new_first_line = -1 @@ -50,42 +60,67 @@ module Lrama ) end + # @rbs () -> String def to_s "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})" end + # @rbs (String error_message) -> String def generate_error_message(error_message) <<~ERROR.chomp #{path}:#{first_line}:#{first_column}: #{error_message} - #{line_with_carets} + #{error_with_carets} ERROR end - def line_with_carets + # @rbs () -> String + def error_with_carets <<~TEXT - #{text} - #{carets} + #{formatted_first_lineno} | #{text} + #{line_number_padding} | #{carets_line} TEXT end private + # @rbs () -> String def path grammar_file.path end - def blanks - (text[0...first_column] or raise "#{first_column} is invalid").gsub(/[^\t]/, ' ') + # @rbs () -> String + def carets_line + leading_whitespace + highlight_marker + end + + # @rbs () -> String + def leading_whitespace + (text[0...first_column] or raise "Invalid first_column: #{first_column}") + .gsub(/[^\t]/, ' ') + end + + # @rbs () -> String + def highlight_marker + length = last_column - first_column + '^' + '~' * [0, length - 1].max + end + + # @rbs () -> String + def formatted_first_lineno + first_line.to_s.rjust(4) end - def carets - blanks + '^' * (last_column - first_column) + # @rbs () -> String + def line_number_padding + ' ' * formatted_first_lineno.length end + # @rbs () -> String def text @text ||= _text.join("\n") end + # @rbs () -> Array[String] def _text @_text ||=begin range = (first_line - 1)...last_line diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb index 5278e98725..37f77aa069 100644 --- a/tool/lrama/lib/lrama/lexer/token.rb +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -1,56 +1,20 @@ -require 'lrama/lexer/token/char' -require 'lrama/lexer/token/ident' -require 'lrama/lexer/token/instantiate_rule' -require 'lrama/lexer/token/tag' -require 'lrama/lexer/token/user_code' +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'token/base' +require_relative 'token/char' +require_relative 'token/empty' +require_relative 'token/ident' +require_relative 'token/instantiate_rule' +require_relative 'token/int' +require_relative 'token/str' +require_relative 'token/tag' +require_relative 'token/token' +require_relative 'token/user_code' module Lrama class Lexer - class Token - attr_reader :s_value, :location - attr_accessor :alias_name, :referred - - def initialize(s_value:, alias_name: nil, location: nil) - s_value.freeze - @s_value = s_value - @alias_name = alias_name - @location = location - end - - def to_s - "#{super} location: #{location}" - end - - def referred_by?(string) - [self.s_value, self.alias_name].compact.include?(string) - end - - def ==(other) - self.class == other.class && self.s_value == other.s_value - end - - def first_line - location.first_line - end - alias :line :first_line - - def first_column - location.first_column - end - alias :column :first_column - - def last_line - location.last_line - end - - def last_column - location.last_column - end - - def invalid_ref(ref, message) - location = self.location.partial_location(ref.first_column, ref.last_column) - raise location.generate_error_message(message) - end + module Token end end end diff --git a/tool/lrama/lib/lrama/lexer/token/base.rb b/tool/lrama/lib/lrama/lexer/token/base.rb new file mode 100644 index 0000000000..3df93bbc73 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/base.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Base + attr_reader :s_value #: String + attr_reader :location #: Location + attr_accessor :alias_name #: String + attr_accessor :referred #: bool + attr_reader :errors #: Array[String] + + # @rbs (s_value: String, ?alias_name: String, ?location: Location) -> void + def initialize(s_value:, alias_name: nil, location: nil) + s_value.freeze + @s_value = s_value + @alias_name = alias_name + @location = location + @errors = [] + end + + # @rbs () -> String + def to_s + "value: `#{s_value}`, location: #{location}" + end + + # @rbs (String string) -> bool + def referred_by?(string) + [self.s_value, self.alias_name].compact.include?(string) + end + + # @rbs (Lexer::Token::Base other) -> bool + def ==(other) + self.class == other.class && self.s_value == other.s_value + end + + # @rbs () -> Integer + def first_line + location.first_line + end + alias :line :first_line + + # @rbs () -> Integer + def first_column + location.first_column + end + alias :column :first_column + + # @rbs () -> Integer + def last_line + location.last_line + end + + # @rbs () -> Integer + def last_column + location.last_column + end + + # @rbs (Lrama::Grammar::Reference ref, String message) -> bot + def invalid_ref(ref, message) + location = self.location.partial_location(ref.first_column, ref.last_column) + raise location.generate_error_message(message) + end + + # @rbs () -> bool + def validate + true + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/char.rb b/tool/lrama/lib/lrama/lexer/token/char.rb index ec3560ca09..f4ef7c9fbc 100644 --- a/tool/lrama/lib/lrama/lexer/token/char.rb +++ b/tool/lrama/lib/lrama/lexer/token/char.rb @@ -1,7 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Lexer - class Token - class Char < Token + module Token + class Char < Base + # @rbs () -> void + def validate + validate_ascii_code_range + end + + private + + # @rbs () -> void + def validate_ascii_code_range + unless s_value.ascii_only? + errors << "Invalid character: `#{s_value}`. Only ASCII characters are allowed." + end + end end end end diff --git a/tool/lrama/lib/lrama/lexer/token/empty.rb b/tool/lrama/lib/lrama/lexer/token/empty.rb new file mode 100644 index 0000000000..375e256493 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/empty.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Empty < Base + def initialize(location: nil) + super(s_value: '%empty', location: location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/ident.rb b/tool/lrama/lib/lrama/lexer/token/ident.rb index e576eaeccd..4880be9073 100644 --- a/tool/lrama/lib/lrama/lexer/token/ident.rb +++ b/tool/lrama/lib/lrama/lexer/token/ident.rb @@ -1,7 +1,10 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Lexer - class Token - class Ident < Token + module Token + class Ident < Base end end end diff --git a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb index 1c4d1095c8..7051ba75a4 100644 --- a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb +++ b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb @@ -1,19 +1,26 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Lexer - class Token - class InstantiateRule < Token - attr_reader :args, :lhs_tag + module Token + class InstantiateRule < Base + attr_reader :args #: Array[Lexer::Token::Base] + attr_reader :lhs_tag #: Lexer::Token::Tag? + # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token::Base], ?lhs_tag: Lexer::Token::Tag?) -> void def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil) super s_value: s_value, alias_name: alias_name, location: location @args = args @lhs_tag = lhs_tag end + # @rbs () -> String def rule_name s_value end + # @rbs () -> Integer def args_count args.count end diff --git a/tool/lrama/lib/lrama/lexer/token/int.rb b/tool/lrama/lib/lrama/lexer/token/int.rb new file mode 100644 index 0000000000..7daf48d4d3 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/int.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Int < Base + # @rbs! + # def initialize: (s_value: Integer, ?alias_name: String, ?location: Location) -> void + # def s_value: () -> Integer + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/str.rb b/tool/lrama/lib/lrama/lexer/token/str.rb new file mode 100644 index 0000000000..cf9de6cf0f --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/str.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Str < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/tag.rb b/tool/lrama/lib/lrama/lexer/token/tag.rb index e54d773915..68c6268219 100644 --- a/tool/lrama/lib/lrama/lexer/token/tag.rb +++ b/tool/lrama/lib/lrama/lexer/token/tag.rb @@ -1,9 +1,13 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class Lexer - class Token - class Tag < Token - # Omit "<>" + module Token + class Tag < Base + # @rbs () -> String def member + # Omit "<>" s_value[1..-2] or raise "Unexpected Tag format (#{s_value})" end end diff --git a/tool/lrama/lib/lrama/lexer/token/token.rb b/tool/lrama/lib/lrama/lexer/token/token.rb new file mode 100644 index 0000000000..935797efc6 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/token.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Token < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb index 765ca2fb46..166f04954a 100644 --- a/tool/lrama/lib/lrama/lexer/token/user_code.rb +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -1,20 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + require "strscan" module Lrama class Lexer - class Token - class UserCode < Token + module Token + class UserCode < Base + attr_accessor :tag #: Lexer::Token::Tag + + # @rbs () -> Array[Lrama::Grammar::Reference] def references @references ||= _references end private + # @rbs () -> Array[Lrama::Grammar::Reference] def _references scanner = StringScanner.new(s_value) - references = [] + references = [] #: Array[Grammar::Reference] - while !scanner.eos? do + until scanner.eos? do case when reference = scan_reference(scanner) references << reference @@ -28,34 +35,72 @@ module Lrama references end + # @rbs (StringScanner scanner) -> Lrama::Grammar::Reference? def scan_reference(scanner) start = scanner.pos - case - # $ references - # It need to wrap an identifier with brackets to use ".-" for identifiers - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\$/) # $$, $<long>$ - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?(\d+)/) # $1, $2, $<long>1 - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, index: Integer(scanner[2]), ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?([a-zA-Z_][a-zA-Z0-9_]*)/) # $foo, $expr, $<long>program (named reference without brackets) - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos) - when scanner.scan(/\$(<[a-zA-Z0-9_]+>)?\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) - tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil - return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[2], ex_tag: tag, first_column: start, last_column: scanner.pos) + if scanner.scan(/ + # $ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + \$(<[a-zA-Z0-9_]+>)?(?: + (\$) # $$, $<long>$ + | (\d+) # $1, $2, $<long>1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $foo, $expr, $<long>program (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + ) + | + # @ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + @(?: + (\$) # @$ + | (\d+) # @1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # @foo, @expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # @[expr.right], @[expr-right] (named reference with brackets) + ) + | + # $: references + \$: + (?: + (\$) # $:$ + | (\d+) # $:1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $:foo, $:expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $:[expr.right], $:[expr-right] (named reference with brackets) + ) + /x) + case + # $ references + when scanner[2] # $$, $<long>$ + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[3] # $1, $2, $<long>1 + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[3]), index: Integer(scanner[3]), ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[4] # $foo, $expr, $<long>program (named reference without brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[4], ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[5] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[5], ex_tag: tag, first_column: start, last_column: scanner.pos) - # @ references - # It need to wrap an identifier with brackets to use ".-" for identifiers - when scanner.scan(/@\$/) # @$ - return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) - when scanner.scan(/@(\d+)/) # @1 - return Lrama::Grammar::Reference.new(type: :at, index: Integer(scanner[1]), first_column: start, last_column: scanner.pos) - when scanner.scan(/@([a-zA-Z][a-zA-Z0-9_]*)/) # @foo, @expr (named reference without brackets) - return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos) - when scanner.scan(/@\[([a-zA-Z_.][-a-zA-Z0-9_.]*)\]/) # @[expr.right], @[expr-right] (named reference with brackets) - return Lrama::Grammar::Reference.new(type: :at, name: scanner[1], first_column: start, last_column: scanner.pos) + # @ references + when scanner[6] # @$ + return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) + when scanner[7] # @1 + return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[7]), index: Integer(scanner[7]), first_column: start, last_column: scanner.pos) + when scanner[8] # @foo, @expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[8], first_column: start, last_column: scanner.pos) + when scanner[9] # @[expr.right], @[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[9], first_column: start, last_column: scanner.pos) + + # $: references + when scanner[10] # $:$ + return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) + when scanner[11] # $:1 + return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[11]), index: Integer(scanner[11]), first_column: start, last_column: scanner.pos) + when scanner[12] # $:foo, $:expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[12], first_column: start, last_column: scanner.pos) + when scanner[13] # $:[expr.right], $:[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[13], first_column: start, last_column: scanner.pos) + end end end end diff --git a/tool/lrama/lib/lrama/logger.rb b/tool/lrama/lib/lrama/logger.rb new file mode 100644 index 0000000000..291eea5296 --- /dev/null +++ b/tool/lrama/lib/lrama/logger.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Logger + # @rbs (IO out) -> void + def initialize(out = STDERR) + @out = out + end + + # @rbs () -> void + def line_break + @out << "\n" + end + + # @rbs (String message) -> void + def trace(message) + @out << message << "\n" + end + + # @rbs (String message) -> void + def warn(message) + @out << 'warning: ' << message << "\n" + end + + # @rbs (String message) -> void + def error(message) + @out << 'error: ' << message << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/option_parser.rb b/tool/lrama/lib/lrama/option_parser.rb index 560b269b06..5a15d59c7b 100644 --- a/tool/lrama/lib/lrama/option_parser.rb +++ b/tool/lrama/lib/lrama/option_parser.rb @@ -1,22 +1,40 @@ +# rbs_inline: enabled +# frozen_string_literal: true + require 'optparse' module Lrama # Handle option parsing for the command line interface. class OptionParser + # @rbs! + # @options: Lrama::Options + # @trace: Array[String] + # @report: Array[String] + # @profile: Array[String] + + # @rbs (Array[String]) -> Lrama::Options + def self.parse(argv) + new.parse(argv) + end + + # @rbs () -> void def initialize @options = Options.new @trace = [] @report = [] + @profile = [] end + # @rbs (Array[String]) -> Lrama::Options def parse(argv) parse_by_option_parser(argv) @options.trace_opts = validate_trace(@trace) @options.report_opts = validate_report(@report) + @options.profile_opts = validate_profile(@profile) @options.grammar_file = argv.shift - if !@options.grammar_file + unless @options.grammar_file abort "File should be specified\n" end @@ -44,6 +62,7 @@ module Lrama private + # @rbs (Array[String]) -> void def parse_by_option_parser(argv) ::OptionParser.new do |o| o.banner = <<~BANNER @@ -57,17 +76,58 @@ module Lrama o.separator '' o.separator 'Tuning the Parser:' o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v } - o.on('-t', 'reserved, do nothing') { } - o.on('--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.on('-t', '--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.separator " same as '-Dparse.trace'" + o.on('--locations', 'enable location support') {|v| @options.locations = true } + o.on('-D', '--define=NAME[=VALUE]', Array, "similar to '%define NAME VALUE'") do |v| + @options.define = v.each_with_object({}) do |item, hash| # steep:ignore UnannotatedEmptyCollection + key, value = item.split('=', 2) + hash[key] = value + end + end o.separator '' o.separator 'Output:' o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v } o.on('-d', 'also produce a header file') { @options.header = true } - o.on('-r', '--report=THINGS', Array, 'also produce details on the automaton') {|v| @report = v } + o.on('-r', '--report=REPORTS', Array, 'also produce details on the automaton') {|v| @report = v } + o.on_tail '' + o.on_tail 'REPORTS is a list of comma-separated words that can include:' + o.on_tail ' states describe the states' + o.on_tail ' itemsets complete the core item sets with their closure' + o.on_tail ' lookaheads explicitly associate lookahead tokens to items' + o.on_tail ' solved describe shift/reduce conflicts solving' + o.on_tail ' counterexamples, cex generate conflict counterexamples' + o.on_tail ' rules list unused rules' + o.on_tail ' terms list unused terminals' + o.on_tail ' verbose report detailed internal state and analysis results' + o.on_tail ' all include all the above reports' + o.on_tail ' none disable all reports' o.on('--report-file=FILE', 'also produce details on the automaton output to a file named FILE') {|v| @options.report_file = v } o.on('-o', '--output=FILE', 'leave output to FILE') {|v| @options.outfile = v } - o.on('--trace=THINGS', Array, 'also output trace logs at runtime') {|v| @trace = v } - o.on('-v', 'reserved, do nothing') { } + o.on('--trace=TRACES', Array, 'also output trace logs at runtime') {|v| @trace = v } + o.on_tail '' + o.on_tail 'TRACES is a list of comma-separated words that can include:' + o.on_tail ' automaton display states' + o.on_tail ' closure display states' + o.on_tail ' rules display grammar rules' + o.on_tail ' only-explicit-rules display only explicit grammar rules' + o.on_tail ' actions display grammar rules with actions' + o.on_tail ' time display generation time' + o.on_tail ' all include all the above traces' + o.on_tail ' none disable all traces' + o.on('--diagram=[FILE]', 'generate a diagram of the rules') do |v| + @options.diagram = true + @options.diagram_file = v if v + end + o.on('--profile=PROFILES', Array, 'profiles parser generation parts') {|v| @profile = v } + o.on_tail '' + o.on_tail 'PROFILES is a list of comma-separated words that can include:' + o.on_tail ' call-stack use sampling call-stack profiler (stackprof gem)' + o.on_tail ' memory use memory profiler (memory_profiler gem)' + o.on('-v', '--verbose', "same as '--report=state'") {|_v| @report << 'states' } + o.separator '' + o.separator 'Diagnostics:' + o.on('-W', '--warnings', 'report the warnings') {|v| @options.warnings = true } o.separator '' o.separator 'Error Recovery:' o.on('-e', 'enable error recovery') {|v| @options.error_recovery = true } @@ -75,50 +135,85 @@ module Lrama o.separator 'Other options:' o.on('-V', '--version', "output version information and exit") {|v| puts "lrama #{Lrama::VERSION}"; exit 0 } o.on('-h', '--help', "display this help and exit") {|v| puts o; exit 0 } - o.separator '' + o.on_tail o.parse!(argv) end end + ALIASED_REPORTS = { cex: :counterexamples }.freeze #: Hash[Symbol, Symbol] + VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze #: Array[Symbol] + + # @rbs (Array[String]) -> Hash[Symbol, bool] def validate_report(report) - bison_list = %w[states itemsets lookaheads solved counterexamples cex all none] - others = %w[verbose] - list = bison_list + others - not_supported = %w[cex none] h = { grammar: true } + return h if report.empty? + return {} if report == ['none'] + if report == ['all'] + VALID_REPORTS.each { |r| h[r] = true } + return h + end report.each do |r| - if list.include?(r) && !not_supported.include?(r) - h[r.to_sym] = true + aliased = aliased_report_option(r) + if VALID_REPORTS.include?(aliased) + h[aliased] = true else raise "Invalid report option \"#{r}\"." end end - if h[:all] - (bison_list - not_supported).each do |r| - h[r.to_sym] = true - end + return h + end - h.delete(:all) + # @rbs (String) -> Symbol + def aliased_report_option(opt) + (ALIASED_REPORTS[opt.to_sym] || opt).to_sym + end + + VALID_TRACES = %w[ + locations scan parse automaton bitsets closure + grammar rules only-explicit-rules actions resource + sets muscles tools m4-early m4 skeleton time ielr cex + ].freeze #: Array[String] + NOT_SUPPORTED_TRACES = %w[ + locations scan parse bitsets grammar resource + sets muscles tools m4-early m4 skeleton ielr cex + ].freeze #: Array[String] + SUPPORTED_TRACES = VALID_TRACES - NOT_SUPPORTED_TRACES #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_trace(trace) + h = {} #: Hash[Symbol, bool] + return h if trace.empty? || trace == ['none'] + all_traces = SUPPORTED_TRACES - %w[only-explicit-rules] + if trace == ['all'] + all_traces.each { |t| h[t.gsub(/-/, '_').to_sym] = true } + return h + end + + trace.each do |t| + if SUPPORTED_TRACES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid trace option \"#{t}\".\nValid options are [#{SUPPORTED_TRACES.join(", ")}]." + end end return h end - def validate_trace(trace) - list = %w[ - none locations scan parse automaton bitsets - closure grammar rules resource sets muscles tools - m4-early m4 skeleton time ielr cex all - ] - h = {} + VALID_PROFILES = %w[call-stack memory].freeze #: Array[String] - trace.each do |t| - if list.include?(t) - h[t.to_sym] = true + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_profile(profile) + h = {} #: Hash[Symbol, bool] + return h if profile.empty? + + profile.each do |t| + if VALID_PROFILES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true else - raise "Invalid trace option \"#{t}\"." + raise "Invalid profile option \"#{t}\".\nValid options are [#{VALID_PROFILES.join(", ")}]." end end diff --git a/tool/lrama/lib/lrama/options.rb b/tool/lrama/lib/lrama/options.rb index e63679bcf2..87aec62448 100644 --- a/tool/lrama/lib/lrama/options.rb +++ b/tool/lrama/lib/lrama/options.rb @@ -1,23 +1,46 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama # Command line options. class Options - attr_accessor :skeleton, :header, :header_file, - :report_file, :outfile, - :error_recovery, :grammar_file, - :trace_opts, :report_opts, :y, - :debug + attr_accessor :skeleton #: String + attr_accessor :locations #: bool + attr_accessor :header #: bool + attr_accessor :header_file #: String? + attr_accessor :report_file #: String? + attr_accessor :outfile #: String + attr_accessor :error_recovery #: bool + attr_accessor :grammar_file #: String + attr_accessor :trace_opts #: Hash[Symbol, bool]? + attr_accessor :report_opts #: Hash[Symbol, bool]? + attr_accessor :warnings #: bool + attr_accessor :y #: IO + attr_accessor :debug #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :diagram #: bool + attr_accessor :diagram_file #: String + attr_accessor :profile_opts #: Hash[Symbol, bool]? + # @rbs () -> void def initialize @skeleton = "bison/yacc.c" + @locations = false + @define = {} @header = false @header_file = nil @report_file = nil @outfile = "y.tab.c" @error_recovery = false - @grammar_file = nil + @grammar_file = '' @trace_opts = nil @report_opts = nil + @warnings = false @y = STDIN + @debug = false + @diagram = false + @diagram_file = "diagram.html" + @profile_opts = nil end end end diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb index 936a3de8d1..d527be8bd4 100644 --- a/tool/lrama/lib/lrama/output.rb +++ b/tool/lrama/lib/lrama/output.rb @@ -1,11 +1,12 @@ -require "erb" +# frozen_string_literal: true + require "forwardable" -require "lrama/report/duration" +require_relative "tracer/duration" module Lrama class Output extend Forwardable - include Report::Duration + include Tracer::Duration attr_reader :grammar_file_path, :context, :grammar, :error_recovery, :include_header @@ -16,8 +17,7 @@ module Lrama def initialize( out:, output_file_path:, template_name:, grammar_file_path:, - header_out: nil, header_file_path: nil, - context:, grammar:, error_recovery: false + context:, grammar:, header_out: nil, header_file_path: nil, error_recovery: false ) @out = out @output_file_path = output_file_path @@ -42,7 +42,7 @@ module Lrama end def render_partial(file) - render_template(partial_file(file)) + ERB.render(partial_file(file), context: @context, output: self) end def render @@ -64,37 +64,29 @@ module Lrama # A part of b4_token_enums def token_enums - str = "" - - @context.yytokentype.each do |s_value, token_id, display_name| + @context.yytokentype.map do |s_value, token_id, display_name| s = sprintf("%s = %d%s", s_value, token_id, token_id == yymaxutok ? "" : ",") if display_name - str << sprintf(" %-30s /* %s */\n", s, display_name) + sprintf(" %-30s /* %s */\n", s, display_name) else - str << sprintf(" %s\n", s) + sprintf(" %s\n", s) end - end - - str + end.join end # b4_symbol_enum def symbol_enum - str = "" - last_sym_number = @context.yysymbol_kind_t.last[1] - @context.yysymbol_kind_t.each do |s_value, sym_number, display_name| + @context.yysymbol_kind_t.map do |s_value, sym_number, display_name| s = sprintf("%s = %d%s", s_value, sym_number, (sym_number == last_sym_number) ? "" : ",") if display_name - str << sprintf(" %-40s /* %s */\n", s, display_name) + sprintf(" %-40s /* %s */\n", s, display_name) else - str << sprintf(" %s\n", s) + sprintf(" %s\n", s) end - end - - str + end.join end def yytranslate @@ -133,12 +125,10 @@ module Lrama end def symbol_actions_for_printer - str = "" - - @grammar.symbols.each do |sym| + @grammar.symbols.map do |sym| next unless sym.printer - str << <<-STR + <<-STR case #{sym.enum_name}: /* #{sym.comment} */ #line #{sym.printer.lineno} "#{@grammar_file_path}" {#{sym.printer.translated_code(sym.tag)}} @@ -146,9 +136,22 @@ module Lrama break; STR - end + end.join + end - str + def symbol_actions_for_destructor + @grammar.symbols.map do |sym| + next unless sym.destructor + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.destructor.lineno} "#{@grammar_file_path}" + {#{sym.destructor.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join end # b4_user_initial_action @@ -162,13 +165,66 @@ module Lrama STR end - def symbol_actions_for_error_token - str = "" + def after_shift_function(comment = "") + return "" unless @grammar.after_shift + + <<-STR + #{comment} +#line #{@grammar.after_shift.line} "#{@grammar_file_path}" + {#{@grammar.after_shift.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def before_reduce_function(comment = "") + return "" unless @grammar.before_reduce + + <<-STR + #{comment} +#line #{@grammar.before_reduce.line} "#{@grammar_file_path}" + {#{@grammar.before_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_reduce_function(comment = "") + return "" unless @grammar.after_reduce - @grammar.symbols.each do |sym| + <<-STR + #{comment} +#line #{@grammar.after_reduce.line} "#{@grammar_file_path}" + {#{@grammar.after_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_shift_error_token_function(comment = "") + return "" unless @grammar.after_shift_error_token + + <<-STR + #{comment} +#line #{@grammar.after_shift_error_token.line} "#{@grammar_file_path}" + {#{@grammar.after_shift_error_token.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def after_pop_stack_function(len, comment = "") + return "" unless @grammar.after_pop_stack + + <<-STR + #{comment} +#line #{@grammar.after_pop_stack.line} "#{@grammar_file_path}" + {#{@grammar.after_pop_stack.s_value}(#{len}#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def symbol_actions_for_error_token + @grammar.symbols.map do |sym| next unless sym.error_token - str << <<-STR + <<-STR case #{sym.enum_name}: /* #{sym.comment} */ #line #{sym.error_token.lineno} "#{@grammar_file_path}" {#{sym.error_token.translated_code(sym.tag)}} @@ -176,22 +232,18 @@ module Lrama break; STR - end - - str + end.join end # b4_user_actions def user_actions - str = "" - - @context.states.rules.each do |rule| + action = @context.states.rules.map do |rule| next unless rule.token_code code = rule.token_code spaces = " " * (code.column - 1) - str << <<-STR + <<-STR case #{rule.id + 1}: /* #{rule.as_comment} */ #line #{code.line} "#{@grammar_file_path}" #{spaces}{#{rule.translated_code}} @@ -199,14 +251,12 @@ module Lrama break; STR - end + end.join - str << <<-STR + action + <<-STR #line [@oline@] [@ofile@] STR - - str end def omit_blanks(param) @@ -270,7 +320,7 @@ module Lrama # b4_parse_param_use def parse_param_use(val, loc) - str = <<-STR + str = <<-STR.dup YY_USE (#{val}); YY_USE (#{loc}); STR @@ -284,7 +334,8 @@ module Lrama # b4_yylex_formals def yylex_formals - ary = ["&yylval", "&yylloc"] + ary = ["&yylval"] + ary << "&yylloc" if @grammar.locations if @grammar.lex_param ary << lex_param_name @@ -324,17 +375,9 @@ module Lrama def int_array_to_string(ary) last = ary.count - 1 - s = ary.each_with_index.each_slice(10).map do |slice| - str = " " - - slice.each do |e, i| - str << sprintf("%6d%s", e, (i == last) ? "" : ",") - end - - str - end - - s.join("\n") + ary.each_with_index.each_slice(10).map do |slice| + " " + slice.map { |e, i| sprintf("%6d%s", e, (i == last) ? "" : ",") }.join + end.join("\n") end def spec_mapped_header_file @@ -361,16 +404,10 @@ module Lrama private def eval_template(file, path) - tmp = render_template(file) + tmp = ERB.render(file, context: @context, output: self) replace_special_variables(tmp, path) end - def render_template(file) - erb = self.class.erb(File.read(file)) - erb.filename = file - erb.result_with_hash(context: @context, output: self) - end - def template_file File.join(template_dir, @template_name) end @@ -384,26 +421,24 @@ module Lrama end def template_dir - File.expand_path("../../../template", __FILE__) + File.expand_path('../../template', __dir__) end def string_array_to_string(ary) - str = "" + result = "" tmp = " " ary.each do |s| - s = s.gsub('\\', '\\\\\\\\') - s = s.gsub('"', '\\"') - - if (tmp + s + " \"\",").length > 75 - str << tmp << "\n" - tmp = " \"#{s}\"," + replaced = s.gsub('\\', '\\\\\\\\').gsub('"', '\\"') + if (tmp + replaced + " \"\",").length > 75 + result = "#{result}#{tmp}\n" + tmp = " \"#{replaced}\"," else - tmp << " \"#{s}\"," + tmp = "#{tmp} \"#{replaced}\"," end end - str << tmp + result + tmp end def replace_special_variables(str, ofile) diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb index 9434b18cc5..20c3ad347f 100644 --- a/tool/lrama/lib/lrama/parser.rb +++ b/tool/lrama/lib/lrama/parser.rb @@ -1,6 +1,7 @@ +# frozen_string_literal: true # # DO NOT MODIFY!!!! -# This file is automatically generated by Racc 1.7.3 +# This file is automatically generated by Racc 1.8.1 # from Racc grammar file "parser.y". # @@ -23,7 +24,7 @@ unless $".find {|p| p.end_with?('/racc/info.rb')} $".push "#{__dir__}/racc/info.rb" module Racc - VERSION = '1.7.3' + VERSION = '1.8.1' Version = VERSION Copyright = 'Copyright (c) 1999-2006 Minero Aoki' end @@ -31,10 +32,6 @@ end end -unless defined?(NotImplementedError) - NotImplementedError = NotImplementError # :nodoc: -end - module Racc class ParseError < StandardError; end end @@ -42,7 +39,7 @@ unless defined?(::ParseError) ParseError = Racc::ParseError # :nodoc: end -# Racc is a LALR(1) parser generator. +# Racc is an LALR(1) parser generator. # It is written in Ruby itself, and generates Ruby programs. # # == Command-line Reference @@ -658,26 +655,28 @@ end module Lrama class Parser < Racc::Parser -module_eval(<<'...end parser.y/module_eval...', 'parser.y', 500) +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 504) -include Lrama::Report::Duration +include Lrama::Tracer::Duration -def initialize(text, path, debug = false) +def initialize(text, path, debug = false, locations = false, define = {}) + @path = path @grammar_file = Lrama::Lexer::GrammarFile.new(path, text) - @yydebug = debug + @yydebug = debug || define.key?('parse.trace') @rule_counter = Lrama::Grammar::Counter.new(0) @midrule_action_counter = Lrama::Grammar::Counter.new(1) + @locations = locations + @define = define end def parse - report_duration(:parse) do + message = "parse '#{File.basename(@path)}'" + report_duration(message) do @lexer = Lrama::Lexer.new(@grammar_file) - @grammar = Lrama::Grammar.new(@rule_counter) + @grammar = Lrama::Grammar.new(@rule_counter, @locations, @define) @precedence_number = 0 reset_precs do_parse - @grammar.prepare - @grammar.validate! @grammar end end @@ -687,7 +686,14 @@ def next_token end def on_error(error_token_id, error_value, value_stack) - if error_value.is_a?(Lrama::Lexer::Token) + case error_value + when Lrama::Lexer::Token::Int + location = error_value.location + value = "#{error_value.s_value}" + when Lrama::Lexer::Token::Token + location = error_value.location + value = "\"#{error_value.s_value}\"" + when Lrama::Lexer::Token::Base location = error_value.location value = "'#{error_value.s_value}'" else @@ -701,7 +707,7 @@ def on_error(error_token_id, error_value, value_stack) end def on_action_error(error_message, error_value) - if error_value.is_a?(Lrama::Lexer::Token) + if error_value.is_a?(Lrama::Lexer::Token::Base) location = error_value.location else location = @lexer.location @@ -713,10 +719,15 @@ end private def reset_precs - @prec_seen = false + @opening_prec_seen = false + @trailing_prec_seen = false @code_after_prec = false end +def prec_seen? + @opening_prec_seen || @trailing_prec_seen +end + def begin_c_declaration(end_symbol) @lexer.status = :c_declaration @lexer.end_symbol = end_symbol @@ -734,302 +745,322 @@ end ##### State transition tables begin ### racc_action_table = [ - 85, 44, 86, 145, 144, 67, 44, 44, 145, 188, - 67, 67, 44, 6, 188, 7, 67, 147, 199, 44, - 143, 43, 147, 189, 58, 163, 164, 165, 189, 3, - 44, 40, 43, 8, 67, 63, 34, 44, 148, 43, - 41, 87, 40, 148, 190, 47, 44, 80, 43, 190, - 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 47, 21, 23, 24, 25, 26, 27, 28, 29, 30, - 31, 9, 44, 47, 43, 13, 14, 15, 16, 17, - 18, 50, 51, 19, 20, 21, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 44, 44, 43, 43, - 52, 70, 70, 44, 44, 43, 43, 53, 70, 70, - 44, 44, 43, 43, 67, 173, 44, 44, 43, 43, - 67, 173, 44, 44, 43, 43, 67, 173, 44, 44, - 43, 43, 67, 173, 44, 44, 43, 43, 67, 173, - 44, 44, 43, 43, 67, 173, 44, 44, 43, 43, - 67, 67, 44, 44, 43, 43, 67, 67, 44, 44, - 43, 43, 67, 67, 44, 44, 179, 43, 67, 67, - 44, 44, 179, 43, 67, 67, 44, 44, 179, 43, - 67, 163, 164, 165, 83, 44, 141, 43, 142, 192, - 54, 193, 163, 164, 165, 208, 210, 193, 193, 55, - 76, 77, 81, 83, 88, 88, 88, 90, 96, 100, - 101, 104, 104, 104, 104, 107, 110, 111, 113, 115, - 116, 117, 118, 119, 122, 126, 127, 128, 131, 132, - 133, 135, 150, 152, 153, 154, 155, 156, 157, 158, - 131, 160, 168, 169, 178, 183, 184, 186, 191, 178, - 83, 183, 205, 207, 83, 212, 83 ] + 98, 98, 99, 99, 87, 53, 53, 52, 178, 110, + 110, 97, 53, 53, 184, 178, 110, 110, 53, 181, + 184, 162, 110, 6, 163, 181, 181, 53, 53, 52, + 52, 181, 79, 79, 53, 53, 52, 52, 43, 79, + 79, 53, 4, 52, 5, 110, 88, 94, 182, 125, + 126, 163, 100, 100, 180, 193, 194, 195, 137, 185, + 188, 180, 4, 44, 5, 185, 188, 94, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 46, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 47, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 47, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 50, + 57, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 57, + 60, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 53, 53, 52, + 52, 110, 105, 53, 53, 52, 52, 110, 105, 53, + 53, 52, 52, 110, 105, 53, 53, 52, 52, 110, + 105, 53, 53, 52, 52, 110, 110, 53, 53, 52, + 209, 110, 110, 53, 53, 209, 52, 110, 110, 53, + 53, 209, 52, 110, 193, 194, 195, 137, 216, 222, + 229, 217, 217, 217, 53, 53, 52, 52, 193, 194, + 195, 57, 57, 57, 57, 66, 67, 68, 69, 70, + 72, 72, 72, 86, 89, 47, 57, 57, 113, 117, + 117, 79, 123, 124, 131, 47, 133, 137, 139, 143, + 149, 150, 151, 152, 133, 155, 156, 157, 110, 166, + 149, 169, 172, 173, 72, 175, 176, 183, 189, 166, + 196, 137, 200, 202, 137, 166, 211, 166, 137, 72, + 176, 218, 176, 72, 72, 227, 137, 72 ] racc_action_check = [ - 42, 130, 42, 130, 129, 130, 159, 177, 159, 177, - 159, 177, 196, 2, 196, 2, 196, 130, 188, 26, - 129, 26, 159, 177, 26, 188, 188, 188, 196, 1, - 27, 9, 27, 3, 27, 27, 7, 14, 130, 14, - 13, 42, 35, 159, 177, 15, 57, 35, 57, 196, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 16, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 4, 58, 17, 58, 4, 4, 4, 4, 4, - 4, 18, 19, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 28, 29, 28, 29, - 21, 28, 29, 30, 31, 30, 31, 23, 30, 31, - 154, 69, 154, 69, 154, 154, 155, 70, 155, 70, - 155, 155, 156, 96, 156, 96, 156, 156, 170, 98, - 170, 98, 170, 170, 174, 104, 174, 104, 174, 174, - 175, 106, 175, 106, 175, 175, 62, 63, 62, 63, - 62, 63, 101, 103, 101, 103, 101, 103, 123, 148, - 123, 148, 123, 148, 160, 190, 160, 190, 160, 190, - 191, 193, 191, 193, 191, 193, 199, 120, 199, 120, - 199, 146, 146, 146, 146, 124, 125, 124, 125, 180, - 24, 180, 181, 181, 181, 202, 206, 202, 206, 25, - 32, 33, 38, 39, 46, 48, 49, 50, 56, 60, - 61, 68, 73, 74, 75, 76, 82, 83, 89, 91, - 92, 93, 94, 95, 99, 107, 108, 109, 110, 111, - 112, 114, 134, 136, 137, 138, 139, 140, 141, 142, - 143, 145, 149, 151, 157, 162, 166, 176, 179, 186, - 187, 192, 195, 200, 205, 211, 212 ] + 51, 97, 51, 97, 41, 75, 165, 75, 165, 75, + 165, 51, 171, 190, 171, 190, 171, 190, 201, 165, + 201, 148, 201, 1, 148, 171, 190, 36, 37, 36, + 37, 201, 36, 37, 38, 39, 38, 39, 5, 38, + 39, 117, 0, 117, 0, 117, 41, 46, 168, 88, + 88, 168, 51, 97, 165, 177, 177, 177, 177, 171, + 171, 190, 2, 6, 2, 201, 201, 90, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 9, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 10, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 11, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 3, 3, 12, + 14, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 15, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 8, 8, 16, + 17, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 18, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 63, 13, 63, + 13, 63, 63, 64, 73, 64, 73, 64, 64, 65, + 78, 65, 78, 65, 65, 106, 79, 106, 79, 106, + 106, 118, 180, 118, 180, 118, 180, 188, 196, 188, + 196, 188, 196, 202, 217, 202, 217, 202, 217, 218, + 113, 218, 113, 218, 186, 186, 186, 186, 208, 213, + 226, 208, 213, 226, 114, 123, 114, 123, 210, 210, + 210, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 40, 42, 47, 55, 60, 71, 74, + 76, 80, 81, 87, 91, 92, 93, 94, 102, 116, + 124, 125, 126, 127, 133, 136, 137, 138, 144, 150, + 151, 153, 156, 158, 162, 163, 164, 170, 174, 176, + 178, 179, 182, 184, 187, 189, 199, 200, 204, 205, + 207, 209, 212, 214, 216, 221, 222, 228 ] racc_action_pointer = [ - nil, 29, 3, 33, 62, nil, nil, 29, nil, 27, - nil, nil, nil, 34, 34, 26, 41, 54, 76, 63, - nil, 81, nil, 88, 171, 180, 16, 27, 93, 94, - 100, 101, 195, 199, nil, 38, nil, nil, 180, 159, - nil, nil, -5, nil, nil, nil, 185, nil, 186, 187, - 188, nil, nil, nil, nil, nil, 200, 43, 69, nil, - 203, 202, 143, 144, nil, nil, nil, nil, 203, 108, - 114, nil, nil, 204, 205, 206, 181, nil, nil, nil, - nil, nil, 180, 212, nil, nil, nil, nil, nil, 216, - nil, 217, 218, 219, 220, 221, 120, nil, 126, 217, - nil, 149, nil, 150, 132, nil, 138, 220, 215, 225, - 189, 184, 228, nil, 229, nil, nil, nil, nil, nil, - 174, nil, nil, 155, 182, 151, nil, nil, nil, -18, - -2, nil, nil, nil, 212, nil, 213, 214, 215, 216, - 217, 202, 234, 201, nil, 207, 140, nil, 156, 222, - nil, 223, nil, nil, 107, 113, 119, 205, nil, 3, - 161, nil, 237, nil, nil, nil, 244, nil, nil, nil, - 125, nil, nil, nil, 131, 137, 209, 4, nil, 214, - 154, 151, nil, nil, nil, nil, 210, 206, -16, nil, - 162, 167, 243, 168, nil, 232, 9, nil, nil, 173, - 251, nil, 160, nil, nil, 210, 161, nil, nil, nil, - nil, 235, 212, nil ] + 32, 23, 52, 93, nil, 31, 63, nil, 123, 68, + 74, 84, 103, 165, 94, 111, 123, 135, 141, nil, + nil, nil, nil, nil, 215, 216, 217, 218, 230, 231, + 232, 233, 234, 232, 233, 234, 24, 25, 31, 32, + 238, -1, 242, nil, nil, nil, 43, 232, nil, nil, + nil, -5, nil, nil, nil, 230, nil, nil, nil, nil, + 231, nil, nil, 164, 170, 176, nil, nil, nil, nil, + nil, 240, nil, 171, 241, 2, 242, nil, 177, 183, + 243, 244, nil, nil, nil, nil, nil, 209, 45, nil, + 63, 245, 242, 243, 202, nil, nil, -4, nil, nil, + nil, nil, 256, nil, nil, nil, 182, nil, nil, nil, + nil, nil, nil, 207, 221, nil, 253, 38, 188, nil, + nil, nil, nil, 222, 255, 215, 218, 252, nil, nil, + nil, nil, nil, 251, nil, nil, 219, 261, 250, nil, + nil, nil, nil, nil, 261, nil, nil, nil, -24, nil, + 219, 265, nil, 269, nil, nil, 216, nil, 256, nil, + nil, nil, 266, 270, 227, 3, nil, nil, 3, nil, + 228, 9, nil, nil, 232, nil, 229, 3, 236, 226, + 189, nil, 236, nil, 239, nil, 162, 229, 194, 235, + 10, nil, nil, nil, nil, nil, 195, nil, nil, 284, + 237, 15, 200, nil, 233, 281, nil, 241, 173, 247, + 176, nil, 243, 174, 285, nil, 286, 201, 206, nil, + nil, 278, 241, nil, nil, nil, 175, nil, 289, nil, + nil ] racc_action_default = [ - -2, -130, -8, -130, -130, -3, -4, -130, 214, -130, - -9, -10, -11, -130, -130, -130, -130, -130, -130, -130, - -23, -130, -27, -130, -130, -130, -130, -130, -130, -130, - -130, -130, -130, -130, -7, -115, -88, -90, -130, -112, - -114, -12, -119, -86, -87, -118, -14, -77, -15, -16, - -130, -20, -24, -28, -31, -34, -37, -43, -130, -46, - -63, -38, -67, -130, -70, -72, -73, -127, -39, -80, - -130, -83, -85, -40, -41, -42, -130, -5, -1, -89, - -116, -91, -130, -130, -13, -120, -121, -122, -74, -130, - -17, -130, -130, -130, -130, -130, -130, -47, -44, -65, - -64, -130, -71, -68, -130, -84, -81, -130, -130, -130, - -96, -130, -130, -78, -130, -21, -25, -29, -32, -35, - -45, -48, -66, -69, -82, -130, -50, -6, -117, -92, - -93, -97, -113, -75, -130, -18, -130, -130, -130, -130, - -130, -130, -130, -96, -95, -86, -112, -101, -130, -130, - -79, -130, -22, -26, -130, -130, -130, -54, -51, -94, - -130, -98, -128, -105, -106, -107, -130, -104, -76, -19, - -30, -123, -125, -126, -33, -36, -49, -52, -55, -86, - -130, -108, -99, -129, -102, -124, -54, -112, -86, -59, - -130, -130, -128, -130, -110, -130, -53, -56, -57, -130, - -130, -62, -130, -100, -109, -112, -130, -60, -111, -103, - -58, -130, -112, -61 ] + -1, -136, -1, -3, -10, -136, -136, -2, -3, -136, + -14, -14, -136, -136, -136, -136, -136, -136, -136, -28, + -29, -34, -35, -36, -136, -136, -136, -136, -136, -136, + -136, -136, -136, -54, -54, -54, -136, -136, -136, -136, + -136, -136, -136, -13, 231, -4, -136, -14, -16, -17, + -20, -131, -100, -101, -130, -18, -23, -89, -24, -25, + -136, -27, -37, -136, -136, -136, -41, -42, -43, -44, + -45, -46, -55, -136, -47, -136, -48, -49, -92, -136, + -95, -97, -98, -50, -51, -52, -53, -136, -136, -11, + -5, -7, -14, -136, -72, -15, -21, -131, -132, -133, + -134, -19, -136, -26, -30, -31, -32, -38, -87, -88, + -135, -39, -40, -136, -56, -58, -60, -136, -83, -85, + -93, -94, -96, -136, -136, -136, -136, -136, -6, -8, + -9, -128, -104, -102, -105, -73, -136, -136, -136, -90, + -33, -59, -57, -61, -80, -86, -84, -99, -136, -66, + -70, -136, -12, -136, -103, -109, -136, -22, -136, -62, + -81, -82, -54, -136, -64, -68, -71, -74, -136, -129, + -106, -107, -127, -91, -136, -67, -70, -72, -100, -72, + -136, -124, -136, -109, -100, -110, -72, -72, -136, -70, + -69, -75, -76, -116, -117, -118, -136, -78, -79, -136, + -70, -108, -136, -111, -72, -54, -115, -63, -136, -100, + -119, -125, -65, -136, -54, -114, -54, -136, -136, -120, + -121, -136, -72, -112, -77, -122, -136, -126, -54, -123, + -113 ] racc_goto_table = [ - 82, 62, 57, 45, 97, 64, 105, 162, 182, 36, - 177, 1, 2, 180, 106, 60, 4, 72, 72, 72, - 72, 130, 185, 46, 48, 49, 185, 185, 68, 73, - 74, 75, 35, 78, 98, 79, 5, 103, 203, 196, - 102, 64, 194, 105, 202, 97, 60, 60, 124, 198, - 33, 108, 206, 10, 159, 170, 174, 175, 72, 72, - 11, 105, 12, 42, 84, 114, 151, 97, 91, 136, - 92, 137, 120, 93, 138, 123, 94, 139, 95, 64, - 140, 102, 56, 61, 99, 60, 121, 60, 125, 176, - 200, 211, 112, 72, 149, 72, 89, 134, 129, 166, - 195, 102, 109, nil, nil, nil, nil, 161, 146, 60, - nil, nil, nil, 72, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, 167, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, 146, 181, nil, - nil, nil, nil, nil, nil, nil, nil, nil, 197, nil, - nil, nil, nil, nil, nil, 187, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, 209, nil, 201, 181, - nil, 204, nil, 213, 187, nil, nil, 181 ] + 73, 118, 136, 54, 48, 49, 164, 96, 91, 120, + 121, 93, 187, 148, 107, 111, 112, 119, 134, 171, + 56, 58, 59, 3, 61, 7, 78, 78, 78, 78, + 62, 63, 64, 65, 115, 74, 76, 192, 1, 129, + 168, 95, 187, 118, 118, 207, 204, 201, 77, 83, + 84, 85, 128, 138, 147, 93, 212, 140, 154, 145, + 146, 101, 130, 116, 42, 127, 103, 208, 78, 78, + 219, 9, 51, 213, 141, 142, 45, 71, 159, 144, + 190, 160, 161, 102, 158, 191, 132, 197, 122, 226, + 170, 177, 220, 199, 203, 205, 221, 186, 153, nil, + nil, nil, nil, 116, 116, nil, 198, nil, nil, nil, + nil, nil, 214, 78, 206, nil, 177, nil, nil, nil, + nil, nil, 210, nil, nil, nil, nil, 186, 210, 174, + 228, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 225, 210, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 215, nil, nil, nil, nil, nil, nil, nil, + nil, 223, nil, 224, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 230 ] racc_goto_check = [ - 41, 46, 32, 34, 33, 40, 53, 42, 59, 54, - 39, 1, 2, 43, 52, 34, 3, 34, 34, 34, - 34, 58, 63, 14, 14, 14, 63, 63, 31, 31, - 31, 31, 4, 5, 32, 54, 6, 46, 59, 39, - 40, 40, 42, 53, 43, 33, 34, 34, 52, 42, - 7, 8, 43, 9, 58, 20, 20, 20, 34, 34, - 10, 53, 11, 12, 13, 15, 16, 33, 17, 18, - 21, 22, 32, 23, 24, 46, 25, 26, 27, 40, - 28, 40, 29, 30, 35, 34, 36, 34, 37, 38, - 44, 45, 48, 34, 49, 34, 50, 51, 57, 60, - 61, 40, 62, nil, nil, nil, nil, 41, 40, 34, - nil, nil, nil, 34, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, 40, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, 40, 40, nil, - nil, nil, nil, nil, nil, nil, nil, nil, 41, nil, - nil, nil, nil, nil, nil, 40, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, 41, nil, 40, 40, - nil, 40, nil, 41, 40, nil, nil, 40 ] + 29, 22, 42, 31, 14, 14, 35, 16, 8, 48, + 48, 13, 40, 34, 24, 24, 24, 45, 52, 54, + 18, 18, 18, 6, 17, 6, 31, 31, 31, 31, + 17, 17, 17, 17, 30, 26, 26, 38, 1, 5, + 34, 14, 40, 22, 22, 35, 38, 54, 27, 27, + 27, 27, 8, 16, 48, 13, 35, 24, 52, 45, + 45, 18, 9, 31, 10, 11, 17, 39, 31, 31, + 38, 7, 15, 39, 30, 30, 7, 25, 32, 33, + 36, 43, 44, 46, 47, 42, 14, 42, 50, 39, + 53, 22, 55, 56, 42, 42, 57, 22, 58, nil, + nil, nil, nil, 31, 31, nil, 22, nil, nil, nil, + nil, nil, 42, 31, 22, nil, 22, nil, nil, nil, + nil, nil, 22, nil, nil, nil, nil, 22, 22, 29, + 42, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 22, 22, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 29, nil, nil, nil, nil, nil, nil, nil, + nil, 29, nil, 29, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 29 ] racc_goto_pointer = [ - nil, 11, 12, 14, 23, -2, 34, 44, -26, 49, - 56, 58, 49, 22, 8, -25, -69, 17, -46, nil, - -99, 18, -45, 20, -43, 22, -41, 23, -39, 56, - 56, 0, -24, -53, -11, 24, -13, -19, -68, -147, - -22, -39, -139, -147, -99, -116, -26, nil, 4, -39, - 49, -16, -56, -63, 0, nil, nil, -12, -89, -154, - -48, -84, 22, -148 ] + nil, 38, nil, nil, nil, -52, 23, 68, -38, -29, + 60, -24, nil, -35, -6, 59, -44, 6, 6, nil, + nil, nil, -74, nil, -49, 44, 1, 12, nil, -33, + -39, -10, -66, -37, -111, -144, -96, nil, -140, -129, + -159, nil, -92, -63, -62, -58, 26, -55, -69, nil, + 8, nil, -75, -65, -136, -118, -88, -115, -33 ] racc_goto_default = [ - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - 38, nil, nil, nil, nil, nil, nil, nil, nil, 22, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, 59, 65, nil, nil, nil, nil, nil, - 172, nil, nil, nil, nil, nil, nil, 66, nil, nil, - nil, nil, 69, 71, nil, 37, 39, nil, nil, nil, - nil, nil, nil, 171 ] + nil, nil, 2, 8, 90, nil, nil, nil, nil, nil, + nil, nil, 10, 11, nil, nil, nil, 55, nil, 21, + 22, 23, 104, 106, nil, nil, nil, nil, 114, 75, + nil, 108, nil, nil, nil, nil, 165, 135, nil, nil, + 179, 167, nil, 109, nil, nil, nil, nil, 81, 80, + 82, 92, nil, nil, nil, nil, nil, nil, nil ] racc_reduce_table = [ 0, 0, :racc_error, - 5, 48, :_reduce_none, - 0, 49, :_reduce_none, - 2, 49, :_reduce_none, - 0, 54, :_reduce_4, - 0, 55, :_reduce_5, - 5, 53, :_reduce_6, - 2, 53, :_reduce_none, - 0, 50, :_reduce_8, - 2, 50, :_reduce_none, - 1, 56, :_reduce_none, - 1, 56, :_reduce_none, - 2, 56, :_reduce_12, - 3, 56, :_reduce_none, - 2, 56, :_reduce_none, - 2, 56, :_reduce_15, - 2, 56, :_reduce_16, - 0, 62, :_reduce_17, - 0, 63, :_reduce_18, - 7, 56, :_reduce_19, - 0, 64, :_reduce_20, - 0, 65, :_reduce_21, - 6, 56, :_reduce_22, - 1, 56, :_reduce_none, - 0, 68, :_reduce_24, - 0, 69, :_reduce_25, - 6, 57, :_reduce_26, - 1, 57, :_reduce_none, - 0, 70, :_reduce_28, - 0, 71, :_reduce_29, - 7, 57, :_reduce_none, - 0, 72, :_reduce_31, - 0, 73, :_reduce_32, - 7, 57, :_reduce_33, - 0, 74, :_reduce_34, - 0, 75, :_reduce_35, - 7, 57, :_reduce_36, - 2, 66, :_reduce_none, - 2, 66, :_reduce_38, - 2, 66, :_reduce_39, - 2, 66, :_reduce_40, - 2, 66, :_reduce_41, - 2, 66, :_reduce_42, - 1, 76, :_reduce_43, - 2, 76, :_reduce_44, - 3, 76, :_reduce_45, - 1, 79, :_reduce_46, - 2, 79, :_reduce_47, - 3, 80, :_reduce_48, - 7, 58, :_reduce_49, - 1, 84, :_reduce_50, - 3, 84, :_reduce_51, - 1, 85, :_reduce_52, - 3, 85, :_reduce_53, - 0, 86, :_reduce_54, - 1, 86, :_reduce_55, - 3, 86, :_reduce_56, - 3, 86, :_reduce_57, - 5, 86, :_reduce_58, - 0, 91, :_reduce_59, - 0, 92, :_reduce_60, - 7, 86, :_reduce_61, - 3, 86, :_reduce_62, - 0, 82, :_reduce_none, - 1, 82, :_reduce_none, - 0, 83, :_reduce_none, - 1, 83, :_reduce_none, - 1, 77, :_reduce_67, - 2, 77, :_reduce_68, - 3, 77, :_reduce_69, - 1, 93, :_reduce_70, - 2, 93, :_reduce_71, - 1, 87, :_reduce_none, + 0, 64, :_reduce_1, + 2, 64, :_reduce_2, + 0, 65, :_reduce_3, + 2, 65, :_reduce_4, + 1, 66, :_reduce_5, + 2, 66, :_reduce_6, + 0, 67, :_reduce_none, + 1, 67, :_reduce_none, + 5, 59, :_reduce_none, + 0, 68, :_reduce_10, + 0, 69, :_reduce_11, + 5, 60, :_reduce_12, + 2, 60, :_reduce_13, + 0, 72, :_reduce_14, + 2, 72, :_reduce_15, + 2, 61, :_reduce_none, + 2, 61, :_reduce_none, + 1, 76, :_reduce_18, + 2, 76, :_reduce_19, + 2, 70, :_reduce_20, + 3, 70, :_reduce_21, + 5, 70, :_reduce_22, + 2, 70, :_reduce_none, + 2, 70, :_reduce_24, + 2, 70, :_reduce_25, + 3, 70, :_reduce_26, + 2, 70, :_reduce_27, + 1, 70, :_reduce_28, + 1, 70, :_reduce_29, + 1, 81, :_reduce_30, + 1, 81, :_reduce_31, + 1, 82, :_reduce_32, + 2, 82, :_reduce_33, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 2, 71, :_reduce_37, + 3, 71, :_reduce_38, + 3, 71, :_reduce_39, + 3, 71, :_reduce_40, + 2, 71, :_reduce_41, + 2, 71, :_reduce_42, + 2, 71, :_reduce_43, + 2, 71, :_reduce_44, + 2, 71, :_reduce_45, + 2, 77, :_reduce_none, + 2, 77, :_reduce_47, + 2, 77, :_reduce_48, + 2, 77, :_reduce_49, + 2, 77, :_reduce_50, + 2, 77, :_reduce_51, + 2, 77, :_reduce_52, + 2, 77, :_reduce_53, + 0, 87, :_reduce_none, 1, 87, :_reduce_none, - 0, 95, :_reduce_74, - 0, 96, :_reduce_75, - 6, 61, :_reduce_76, - 0, 97, :_reduce_77, - 0, 98, :_reduce_78, - 5, 61, :_reduce_79, - 1, 78, :_reduce_80, - 2, 78, :_reduce_81, - 3, 78, :_reduce_82, - 1, 99, :_reduce_83, - 2, 99, :_reduce_84, + 1, 88, :_reduce_56, + 2, 88, :_reduce_57, + 2, 83, :_reduce_58, + 3, 83, :_reduce_59, + 0, 91, :_reduce_none, + 1, 91, :_reduce_none, + 3, 86, :_reduce_62, + 8, 78, :_reduce_63, + 5, 79, :_reduce_64, + 8, 79, :_reduce_65, + 1, 92, :_reduce_66, + 3, 92, :_reduce_67, + 1, 93, :_reduce_68, + 3, 93, :_reduce_69, + 0, 99, :_reduce_none, + 1, 99, :_reduce_none, + 0, 100, :_reduce_none, 1, 100, :_reduce_none, - 1, 81, :_reduce_86, - 1, 81, :_reduce_87, - 1, 51, :_reduce_none, - 2, 51, :_reduce_none, - 1, 101, :_reduce_none, - 2, 101, :_reduce_none, - 4, 102, :_reduce_92, - 1, 104, :_reduce_93, - 3, 104, :_reduce_94, - 2, 104, :_reduce_none, - 0, 105, :_reduce_96, - 1, 105, :_reduce_97, - 3, 105, :_reduce_98, - 4, 105, :_reduce_99, - 6, 105, :_reduce_100, - 0, 107, :_reduce_101, - 0, 108, :_reduce_102, - 7, 105, :_reduce_103, - 3, 105, :_reduce_104, - 1, 89, :_reduce_none, + 1, 94, :_reduce_74, + 3, 94, :_reduce_75, + 3, 94, :_reduce_76, + 6, 94, :_reduce_77, + 3, 94, :_reduce_78, + 3, 94, :_reduce_79, + 0, 102, :_reduce_none, + 1, 102, :_reduce_none, + 1, 90, :_reduce_82, + 1, 103, :_reduce_83, + 2, 103, :_reduce_84, + 2, 84, :_reduce_85, + 3, 84, :_reduce_86, + 1, 80, :_reduce_none, + 1, 80, :_reduce_none, + 0, 104, :_reduce_89, + 0, 105, :_reduce_90, + 5, 75, :_reduce_91, + 1, 106, :_reduce_92, + 2, 106, :_reduce_93, + 2, 107, :_reduce_94, + 1, 108, :_reduce_95, + 2, 108, :_reduce_96, + 1, 85, :_reduce_97, + 1, 85, :_reduce_98, + 3, 85, :_reduce_99, 1, 89, :_reduce_none, 1, 89, :_reduce_none, - 1, 90, :_reduce_108, - 3, 90, :_reduce_109, - 2, 90, :_reduce_110, - 4, 90, :_reduce_111, - 0, 88, :_reduce_none, - 3, 88, :_reduce_113, - 1, 103, :_reduce_none, - 0, 52, :_reduce_none, - 0, 109, :_reduce_116, - 3, 52, :_reduce_117, - 1, 59, :_reduce_none, - 0, 60, :_reduce_none, - 1, 60, :_reduce_none, - 1, 60, :_reduce_none, - 1, 60, :_reduce_none, - 1, 67, :_reduce_123, - 2, 67, :_reduce_124, - 1, 110, :_reduce_none, - 1, 110, :_reduce_none, - 1, 94, :_reduce_127, - 0, 106, :_reduce_none, - 1, 106, :_reduce_none ] - -racc_reduce_n = 130 - -racc_shift_n = 214 + 1, 110, :_reduce_102, + 2, 110, :_reduce_103, + 2, 62, :_reduce_none, + 2, 62, :_reduce_none, + 4, 109, :_reduce_106, + 1, 111, :_reduce_107, + 3, 111, :_reduce_108, + 0, 112, :_reduce_109, + 2, 112, :_reduce_110, + 3, 112, :_reduce_111, + 5, 112, :_reduce_112, + 7, 112, :_reduce_113, + 4, 112, :_reduce_114, + 3, 112, :_reduce_115, + 1, 96, :_reduce_116, + 1, 96, :_reduce_117, + 1, 96, :_reduce_118, + 0, 113, :_reduce_none, + 1, 113, :_reduce_none, + 2, 97, :_reduce_121, + 3, 97, :_reduce_122, + 4, 97, :_reduce_123, + 0, 114, :_reduce_124, + 0, 115, :_reduce_125, + 5, 98, :_reduce_126, + 3, 95, :_reduce_127, + 0, 116, :_reduce_128, + 3, 63, :_reduce_129, + 1, 73, :_reduce_none, + 0, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 101, :_reduce_135 ] + +racc_reduce_n = 136 + +racc_shift_n = 231 racc_token_table = { false => 0, @@ -1045,42 +1076,53 @@ racc_token_table = { "%{" => 10, "%}" => 11, "%require" => 12, - "%expect" => 13, - "%define" => 14, - "%param" => 15, - "%lex-param" => 16, - "%parse-param" => 17, - "%code" => 18, - "{" => 19, - "}" => 20, - "%initial-action" => 21, - ";" => 22, - "%union" => 23, - "%destructor" => 24, - "%printer" => 25, - "%error-token" => 26, - "%token" => 27, - "%type" => 28, - "%left" => 29, - "%right" => 30, - "%precedence" => 31, - "%nonassoc" => 32, - "%rule" => 33, - "(" => 34, - ")" => 35, - ":" => 36, - "," => 37, - "|" => 38, - "%empty" => 39, - "%prec" => 40, - "?" => 41, - "+" => 42, - "*" => 43, - "[" => 44, - "]" => 45, - "{...}" => 46 } - -racc_nt_base = 47 + ";" => 13, + "%expect" => 14, + "%define" => 15, + "{" => 16, + "}" => 17, + "%param" => 18, + "%lex-param" => 19, + "%parse-param" => 20, + "%code" => 21, + "%initial-action" => 22, + "%no-stdlib" => 23, + "%locations" => 24, + "%union" => 25, + "%destructor" => 26, + "%printer" => 27, + "%error-token" => 28, + "%after-shift" => 29, + "%before-reduce" => 30, + "%after-reduce" => 31, + "%after-shift-error-token" => 32, + "%after-pop-stack" => 33, + "-temp-group" => 34, + "%token" => 35, + "%type" => 36, + "%nterm" => 37, + "%left" => 38, + "%right" => 39, + "%precedence" => 40, + "%nonassoc" => 41, + "%start" => 42, + "%rule" => 43, + "(" => 44, + ")" => 45, + ":" => 46, + "%inline" => 47, + "," => 48, + "|" => 49, + "%empty" => 50, + "%prec" => 51, + "?" => 52, + "+" => 53, + "*" => 54, + "[" => 55, + "]" => 56, + "{...}" => 57 } + +racc_nt_base = 58 racc_use_result_var = true @@ -1115,30 +1157,41 @@ Racc_token_to_s_table = [ "\"%{\"", "\"%}\"", "\"%require\"", + "\";\"", "\"%expect\"", "\"%define\"", + "\"{\"", + "\"}\"", "\"%param\"", "\"%lex-param\"", "\"%parse-param\"", "\"%code\"", - "\"{\"", - "\"}\"", "\"%initial-action\"", - "\";\"", + "\"%no-stdlib\"", + "\"%locations\"", "\"%union\"", "\"%destructor\"", "\"%printer\"", "\"%error-token\"", + "\"%after-shift\"", + "\"%before-reduce\"", + "\"%after-reduce\"", + "\"%after-shift-error-token\"", + "\"%after-pop-stack\"", + "\"-temp-group\"", "\"%token\"", "\"%type\"", + "\"%nterm\"", "\"%left\"", "\"%right\"", "\"%precedence\"", "\"%nonassoc\"", + "\"%start\"", "\"%rule\"", "\"(\"", "\")\"", "\":\"", + "\"%inline\"", "\",\"", "\"|\"", "\"%empty\"", @@ -1151,68 +1204,63 @@ Racc_token_to_s_table = [ "\"{...}\"", "$start", "input", - "prologue_declarations", - "bison_declarations", - "grammar", - "epilogue_opt", "prologue_declaration", + "bison_declaration", + "rules_or_grammar_declaration", + "epilogue_declaration", + "\"-many@prologue_declaration\"", + "\"-many@bison_declaration\"", + "\"-many1@rules_or_grammar_declaration\"", + "\"-option@epilogue_declaration\"", "@1", "@2", - "bison_declaration", + "parser_option", "grammar_declaration", - "rule_declaration", + "\"-many@;\"", "variable", "value", - "params", - "@3", - "@4", - "@5", - "@6", + "param", + "\"-many1@param\"", "symbol_declaration", - "generic_symlist", - "@7", - "@8", - "@9", - "@10", - "@11", - "@12", - "@13", - "@14", + "rule_declaration", + "inline_declaration", + "symbol", + "\"-group@symbol|TAG\"", + "\"-many1@-group@symbol|TAG\"", "token_declarations", "symbol_declarations", "token_declarations_for_precedence", - "token_declaration_list", "token_declaration", + "\"-option@TAG\"", + "\"-many1@token_declaration\"", "id", - "int_opt", "alias", + "\"-option@INTEGER\"", "rule_args", "rule_rhs_list", "rule_rhs", - "symbol", - "named_ref_opt", - "parameterizing_suffix", - "parameterizing_args", - "@15", - "@16", - "symbol_declaration_list", + "named_ref", + "parameterized_suffix", + "parameterized_args", + "action", + "\"-option@%empty\"", + "\"-option@named_ref\"", "string_as_id", - "@17", - "@18", - "@19", - "@20", - "token_declaration_list_for_precedence", - "token_declaration_for_precedence", - "rules_or_grammar_declaration", + "\"-option@string_as_id\"", + "\"-many1@symbol\"", + "@3", + "@4", + "\"-many1@id\"", + "\"-group@TAG-\\\"-many1@id\\\"\"", + "\"-many1@-group@TAG-\\\"-many1@id\\\"\"", "rules", - "id_colon", + "\"-many1@;\"", "rhs_list", "rhs", - "tag_opt", - "@21", - "@22", - "@23", - "generic_symlist_item" ] + "\"-option@parameterized_suffix\"", + "@5", + "@6", + "@7" ] Ractor.make_shareable(Racc_token_to_s_table) if defined?(Ractor) Racc_debug_parser = true @@ -1221,126 +1269,138 @@ Racc_debug_parser = true # reduce 0 omitted -# reduce 1 omitted +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_1(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., -# reduce 2 omitted +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_2(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., -# reduce 3 omitted +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_3(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 14) +module_eval(<<'.,.,', 'parser.y', 11) def _reduce_4(val, _values, result) - begin_c_declaration("%}") - @grammar.prologue_first_lineno = @lexer.line - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 19) +module_eval(<<'.,.,', 'parser.y', 11) def _reduce_5(val, _values, result) - end_c_declaration - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 23) +module_eval(<<'.,.,', 'parser.y', 11) def _reduce_6(val, _values, result) - @grammar.prologue = val[2].s_value - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., # reduce 7 omitted -module_eval(<<'.,.,', 'parser.y', 27) - def _reduce_8(val, _values, result) - result = "" - result - end -.,., +# reduce 8 omitted # reduce 9 omitted -# reduce 10 omitted - -# reduce 11 omitted +module_eval(<<'.,.,', 'parser.y', 13) + def _reduce_10(val, _values, result) + begin_c_declaration("%}") -module_eval(<<'.,.,', 'parser.y', 32) - def _reduce_12(val, _values, result) - @grammar.expect = val[1] result end .,., -# reduce 13 omitted +module_eval(<<'.,.,', 'parser.y', 17) + def _reduce_11(val, _values, result) + end_c_declaration -# reduce 14 omitted + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 37) - def _reduce_15(val, _values, result) - val[1].each {|token| - @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value - } +module_eval(<<'.,.,', 'parser.y', 21) + def _reduce_12(val, _values, result) + @grammar.prologue_first_lineno = val[0].first_line + @grammar.prologue = val[2].s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 43) - def _reduce_16(val, _values, result) - val[1].each {|token| - @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value - } +module_eval(<<'.,.,', 'parser.y', 26) + def _reduce_13(val, _values, result) + @grammar.required = true result end .,., -module_eval(<<'.,.,', 'parser.y', 49) - def _reduce_17(val, _values, result) - begin_c_declaration("}") +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_14(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_15(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 53) - def _reduce_18(val, _values, result) - end_c_declaration +# reduce 16 omitted +# reduce 17 omitted + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_18(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 57) +module_eval(<<'.,.,', 'parser.y', 77) def _reduce_19(val, _values, result) - @grammar.add_percent_code(id: val[1], code: val[4]) - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 61) +module_eval(<<'.,.,', 'parser.y', 36) def _reduce_20(val, _values, result) - begin_c_declaration("}") + @grammar.expect = val[1].s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 65) +module_eval(<<'.,.,', 'parser.y', 40) def _reduce_21(val, _values, result) - end_c_declaration + @grammar.define[val[1].s_value] = val[2]&.s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 69) +module_eval(<<'.,.,', 'parser.y', 44) def _reduce_22(val, _values, result) - @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[3]) + @grammar.define[val[1].s_value] = val[3]&.s_value result end @@ -1348,766 +1408,864 @@ module_eval(<<'.,.,', 'parser.y', 69) # reduce 23 omitted -module_eval(<<'.,.,', 'parser.y', 75) +module_eval(<<'.,.,', 'parser.y', 49) def _reduce_24(val, _values, result) - begin_c_declaration("}") + val[1].each {|token| + @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value + } result end .,., -module_eval(<<'.,.,', 'parser.y', 79) +module_eval(<<'.,.,', 'parser.y', 55) def _reduce_25(val, _values, result) - end_c_declaration + val[1].each {|token| + @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value + } result end .,., -module_eval(<<'.,.,', 'parser.y', 83) +module_eval(<<'.,.,', 'parser.y', 61) def _reduce_26(val, _values, result) - @grammar.set_union( - Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[3]), - val[3].line - ) + @grammar.add_percent_code(id: val[1], code: val[2]) result end .,., -# reduce 27 omitted +module_eval(<<'.,.,', 'parser.y', 65) + def _reduce_27(val, _values, result) + @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) -module_eval(<<'.,.,', 'parser.y', 91) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 69) def _reduce_28(val, _values, result) - begin_c_declaration("}") + @grammar.no_stdlib = true result end .,., -module_eval(<<'.,.,', 'parser.y', 95) +module_eval(<<'.,.,', 'parser.y', 73) def _reduce_29(val, _values, result) - end_c_declaration + @grammar.locations = true result end .,., -# reduce 30 omitted +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_30(val, _values, result) + result = val + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 100) +module_eval(<<'.,.,', 'parser.y', 133) def _reduce_31(val, _values, result) - begin_c_declaration("}") - + result = val result end .,., -module_eval(<<'.,.,', 'parser.y', 104) +module_eval(<<'.,.,', 'parser.y', 133) def _reduce_32(val, _values, result) - end_c_declaration - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 108) +module_eval(<<'.,.,', 'parser.y', 133) def _reduce_33(val, _values, result) - @grammar.add_printer( - ident_or_tags: val[6], - token_code: val[3], - lineno: val[3].line - ) - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 116) - def _reduce_34(val, _values, result) - begin_c_declaration("}") - - result - end -.,., +# reduce 34 omitted -module_eval(<<'.,.,', 'parser.y', 120) - def _reduce_35(val, _values, result) - end_c_declaration +# reduce 35 omitted - result - end -.,., +# reduce 36 omitted -module_eval(<<'.,.,', 'parser.y', 124) - def _reduce_36(val, _values, result) - @grammar.add_error_token( - ident_or_tags: val[6], - token_code: val[3], - lineno: val[3].line - ) +module_eval(<<'.,.,', 'parser.y', 82) + def _reduce_37(val, _values, result) + @grammar.set_union( + Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), + val[1].line + ) result end .,., -# reduce 37 omitted - -module_eval(<<'.,.,', 'parser.y', 134) +module_eval(<<'.,.,', 'parser.y', 89) def _reduce_38(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - @grammar.add_type(id: id, tag: hash[:tag]) - } - } + @grammar.add_destructor( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 142) +module_eval(<<'.,.,', 'parser.y', 97) def _reduce_39(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_left(sym, @precedence_number) - } - } - @precedence_number += 1 + @grammar.add_printer( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 152) +module_eval(<<'.,.,', 'parser.y', 105) def _reduce_40(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_right(sym, @precedence_number) - } - } - @precedence_number += 1 + @grammar.add_error_token( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) result end .,., -module_eval(<<'.,.,', 'parser.y', 162) +module_eval(<<'.,.,', 'parser.y', 113) def _reduce_41(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_precedence(sym, @precedence_number) - } - } - @precedence_number += 1 + @grammar.after_shift = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 172) +module_eval(<<'.,.,', 'parser.y', 117) def _reduce_42(val, _values, result) - val[1].each {|hash| - hash[:tokens].each {|id| - sym = @grammar.add_term(id: id) - @grammar.add_nonassoc(sym, @precedence_number) - } - } - @precedence_number += 1 + @grammar.before_reduce = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 183) +module_eval(<<'.,.,', 'parser.y', 121) def _reduce_43(val, _values, result) - val[0].each {|token_declaration| - @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: nil, replace: true) - } + @grammar.after_reduce = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 189) +module_eval(<<'.,.,', 'parser.y', 125) def _reduce_44(val, _values, result) - val[1].each {|token_declaration| - @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[0], replace: true) - } + @grammar.after_shift_error_token = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 195) +module_eval(<<'.,.,', 'parser.y', 129) def _reduce_45(val, _values, result) - val[2].each {|token_declaration| - @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[1], replace: true) - } + @grammar.after_pop_stack = val[1] result end .,., -module_eval(<<'.,.,', 'parser.y', 200) - def _reduce_46(val, _values, result) - result = [val[0]] - result - end -.,., +# reduce 46 omitted -module_eval(<<'.,.,', 'parser.y', 201) +module_eval(<<'.,.,', 'parser.y', 136) def _reduce_47(val, _values, result) - result = val[0].append(val[1]) + val[1].each {|hash| + hash[:tokens].each {|id| + @grammar.add_type(id: id, tag: hash[:tag]) + } + } + result end .,., -module_eval(<<'.,.,', 'parser.y', 203) +module_eval(<<'.,.,', 'parser.y', 144) def _reduce_48(val, _values, result) - result = val + val[1].each {|hash| + hash[:tokens].each {|id| + if @grammar.find_term_by_s_value(id.s_value) + on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) + else + @grammar.add_type(id: id, tag: hash[:tag]) + end + } + } + result end .,., -module_eval(<<'.,.,', 'parser.y', 207) +module_eval(<<'.,.,', 'parser.y', 156) def _reduce_49(val, _values, result) - rule = Grammar::ParameterizingRule::Rule.new(val[1].s_value, val[3], val[6]) - @grammar.add_parameterizing_rule(rule) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_left(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 result end .,., -module_eval(<<'.,.,', 'parser.y', 211) +module_eval(<<'.,.,', 'parser.y', 166) def _reduce_50(val, _values, result) - result = [val[0]] + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_right(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + result end .,., -module_eval(<<'.,.,', 'parser.y', 212) +module_eval(<<'.,.,', 'parser.y', 176) def _reduce_51(val, _values, result) - result = val[0].append(val[2]) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_precedence(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + result end .,., -module_eval(<<'.,.,', 'parser.y', 216) +module_eval(<<'.,.,', 'parser.y', 186) def _reduce_52(val, _values, result) - builder = val[0] - result = [builder] + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_nonassoc(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 result end .,., -module_eval(<<'.,.,', 'parser.y', 221) +module_eval(<<'.,.,', 'parser.y', 196) def _reduce_53(val, _values, result) - builder = val[2] - result = val[0].append(builder) + @grammar.set_start_nterm(val[1]) result end .,., -module_eval(<<'.,.,', 'parser.y', 227) - def _reduce_54(val, _values, result) - reset_precs - result = Grammar::ParameterizingRule::Rhs.new +# reduce 54 omitted +# reduce 55 omitted + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_56(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 232) - def _reduce_55(val, _values, result) - reset_precs - result = Grammar::ParameterizingRule::Rhs.new - +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_57(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 237) - def _reduce_56(val, _values, result) - token = val[1] - token.alias_name = val[2] - builder = val[0] - builder.symbols << token - result = builder +module_eval(<<'.,.,', 'parser.y', 202) + def _reduce_58(val, _values, result) + val[1].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[0], replace: true) + } result end .,., -module_eval(<<'.,.,', 'parser.y', 245) - def _reduce_57(val, _values, result) - builder = val[0] - builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) - result = builder +module_eval(<<'.,.,', 'parser.y', 208) + def _reduce_59(val, _values, result) + val[2].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[1], replace: true) + } result end .,., -module_eval(<<'.,.,', 'parser.y', 251) - def _reduce_58(val, _values, result) - builder = val[0] - builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3]) - result = builder +# reduce 60 omitted + +# reduce 61 omitted +module_eval(<<'.,.,', 'parser.y', 213) + def _reduce_62(val, _values, result) + result = val result end .,., -module_eval(<<'.,.,', 'parser.y', 257) - def _reduce_59(val, _values, result) - if @prec_seen - on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec - @code_after_prec = true - end - begin_c_declaration("}") +module_eval(<<'.,.,', 'parser.y', 218) + def _reduce_63(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 265) - def _reduce_60(val, _values, result) - end_c_declaration +module_eval(<<'.,.,', 'parser.y', 225) + def _reduce_64(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, [], val[4], is_inline: true) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 269) - def _reduce_61(val, _values, result) - user_code = val[3] - user_code.alias_name = val[6] - builder = val[0] - builder.user_code = user_code - result = builder +module_eval(<<'.,.,', 'parser.y', 230) + def _reduce_65(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) + @grammar.add_parameterized_rule(rule) result end .,., -module_eval(<<'.,.,', 'parser.y', 277) - def _reduce_62(val, _values, result) - sym = @grammar.find_symbol_by_id!(val[2]) - @prec_seen = true - builder = val[0] - builder.precedence_sym = sym - result = builder - +module_eval(<<'.,.,', 'parser.y', 235) + def _reduce_66(val, _values, result) + result = [val[0]] result end .,., -# reduce 63 omitted - -# reduce 64 omitted - -# reduce 65 omitted - -# reduce 66 omitted - -module_eval(<<'.,.,', 'parser.y', 292) +module_eval(<<'.,.,', 'parser.y', 236) def _reduce_67(val, _values, result) - result = [{tag: nil, tokens: val[0]}] - + result = val[0].append(val[2]) result end .,., -module_eval(<<'.,.,', 'parser.y', 296) +module_eval(<<'.,.,', 'parser.y', 241) def _reduce_68(val, _values, result) - result = [{tag: val[0], tokens: val[1]}] + builder = val[0] + result = [builder] result end .,., -module_eval(<<'.,.,', 'parser.y', 300) +module_eval(<<'.,.,', 'parser.y', 246) def _reduce_69(val, _values, result) - result = val[0].append({tag: val[1], tokens: val[2]}) + builder = val[2] + result = val[0].append(builder) result end .,., -module_eval(<<'.,.,', 'parser.y', 303) - def _reduce_70(val, _values, result) - result = [val[0]] - result - end -.,., +# reduce 70 omitted -module_eval(<<'.,.,', 'parser.y', 304) - def _reduce_71(val, _values, result) - result = val[0].append(val[1]) - result - end -.,., +# reduce 71 omitted # reduce 72 omitted # reduce 73 omitted -module_eval(<<'.,.,', 'parser.y', 311) +module_eval(<<'.,.,', 'parser.y', 253) def _reduce_74(val, _values, result) - begin_c_declaration("}") + reset_precs + result = Grammar::Parameterized::Rhs.new result end .,., -module_eval(<<'.,.,', 'parser.y', 315) +module_eval(<<'.,.,', 'parser.y', 258) def _reduce_75(val, _values, result) - end_c_declaration + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.symbols << token + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 319) +module_eval(<<'.,.,', 'parser.y', 267) def _reduce_76(val, _values, result) - result = val[0].append(val[3]) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 323) +module_eval(<<'.,.,', 'parser.y', 274) def _reduce_77(val, _values, result) - begin_c_declaration("}") + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 327) +module_eval(<<'.,.,', 'parser.y', 281) def _reduce_78(val, _values, result) - end_c_declaration + user_code = val[1] + user_code.alias_name = val[2] + builder = val[0] + builder.user_code = user_code + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 331) +module_eval(<<'.,.,', 'parser.y', 289) def _reduce_79(val, _values, result) - result = [val[2]] + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder result end .,., -module_eval(<<'.,.,', 'parser.y', 336) - def _reduce_80(val, _values, result) - result = [{tag: nil, tokens: val[0]}] +# reduce 80 omitted + +# reduce 81 omitted +module_eval(<<'.,.,', 'parser.y', 301) + def _reduce_82(val, _values, result) + result = val[0].s_value if val[0] result end .,., -module_eval(<<'.,.,', 'parser.y', 340) - def _reduce_81(val, _values, result) - result = [{tag: val[0], tokens: val[1]}] - +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_83(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 344) - def _reduce_82(val, _values, result) - result = val[0].append({tag: val[1], tokens: val[2]}) - +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_84(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 347) - def _reduce_83(val, _values, result) - result = [val[0]] +module_eval(<<'.,.,', 'parser.y', 306) + def _reduce_85(val, _values, result) + result = if val[0] + [{tag: val[0], tokens: val[1]}] + else + [{tag: nil, tokens: val[1]}] + end + result end .,., -module_eval(<<'.,.,', 'parser.y', 348) - def _reduce_84(val, _values, result) - result = val[0].append(val[1]) +module_eval(<<'.,.,', 'parser.y', 312) + def _reduce_86(val, _values, result) + result = val[0].append({tag: val[1], tokens: val[2]}) result end .,., -# reduce 85 omitted +# reduce 87 omitted + +# reduce 88 omitted + +module_eval(<<'.,.,', 'parser.y', 321) + def _reduce_89(val, _values, result) + begin_c_declaration("}") -module_eval(<<'.,.,', 'parser.y', 352) - def _reduce_86(val, _values, result) - on_action_error("ident after %prec", val[0]) if @prec_seen result end .,., -module_eval(<<'.,.,', 'parser.y', 353) - def _reduce_87(val, _values, result) - on_action_error("char after %prec", val[0]) if @prec_seen +module_eval(<<'.,.,', 'parser.y', 325) + def _reduce_90(val, _values, result) + end_c_declaration + result end .,., -# reduce 88 omitted - -# reduce 89 omitted - -# reduce 90 omitted +module_eval(<<'.,.,', 'parser.y', 329) + def _reduce_91(val, _values, result) + result = val[2] -# reduce 91 omitted + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 363) +module_eval(<<'.,.,', 'parser.y', 338) def _reduce_92(val, _values, result) - lhs = val[0] - lhs.alias_name = val[1] - val[3].each do |builder| - builder.lhs = lhs - builder.complete_input - @grammar.add_rule_builder(builder) - end - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 374) +module_eval(<<'.,.,', 'parser.y', 338) def _reduce_93(val, _values, result) - builder = val[0] - if !builder.line - builder.line = @lexer.line - 1 - end - result = [builder] - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 382) +module_eval(<<'.,.,', 'parser.y', 338) def _reduce_94(val, _values, result) - builder = val[2] - if !builder.line - builder.line = @lexer.line - 1 - end - result = val[0].append(builder) - + result = val result end .,., -# reduce 95 omitted +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_95(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 392) +module_eval(<<'.,.,', 'parser.y', 338) def _reduce_96(val, _values, result) - reset_precs - result = Grammar::RuleBuilder.new(@rule_counter, @midrule_action_counter) - + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 397) +module_eval(<<'.,.,', 'parser.y', 333) def _reduce_97(val, _values, result) - reset_precs - result = Grammar::RuleBuilder.new(@rule_counter, @midrule_action_counter) - + result = [{tag: nil, tokens: val[0]}] result end .,., -module_eval(<<'.,.,', 'parser.y', 402) +module_eval(<<'.,.,', 'parser.y', 334) def _reduce_98(val, _values, result) - token = val[1] - token.alias_name = val[2] - builder = val[0] - builder.add_rhs(token) - result = builder - + result = val[0].map {|tag, ids| {tag: tag, tokens: ids} } result end .,., -module_eval(<<'.,.,', 'parser.y', 410) +module_eval(<<'.,.,', 'parser.y', 335) def _reduce_99(val, _values, result) - token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]], lhs_tag: val[3]) - builder = val[0] - builder.add_rhs(token) - builder.line = val[1].first_line - result = builder - + result = [{tag: nil, tokens: val[0]}, {tag: val[1], tokens: val[2]}] result end .,., -module_eval(<<'.,.,', 'parser.y', 418) - def _reduce_100(val, _values, result) - token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) - builder = val[0] - builder.add_rhs(token) - builder.line = val[1].first_line - result = builder +# reduce 100 omitted +# reduce 101 omitted + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_102(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 426) - def _reduce_101(val, _values, result) - if @prec_seen - on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec - @code_after_prec = true - end - begin_c_declaration("}") - +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_103(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val result end .,., -module_eval(<<'.,.,', 'parser.y', 434) - def _reduce_102(val, _values, result) - end_c_declaration +# reduce 104 omitted + +# reduce 105 omitted + +module_eval(<<'.,.,', 'parser.y', 348) + def _reduce_106(val, _values, result) + lhs = val[0] + lhs.alias_name = val[1] + val[3].each do |builder| + builder.lhs = lhs + builder.complete_input + @grammar.add_rule_builder(builder) + end result end .,., -module_eval(<<'.,.,', 'parser.y', 438) - def _reduce_103(val, _values, result) - user_code = val[3] - user_code.alias_name = val[6] - builder = val[0] - builder.user_code = user_code - result = builder +module_eval(<<'.,.,', 'parser.y', 360) + def _reduce_107(val, _values, result) + if val[0].rhs.count > 1 + empties = val[0].rhs.select { |sym| sym.is_a?(Lrama::Lexer::Token::Empty) } + empties.each do |empty| + on_action_error("%empty on non-empty rule", empty) + end + end + builder = val[0] + if !builder.line + builder.line = @lexer.line - 1 + end + result = [builder] result end .,., -module_eval(<<'.,.,', 'parser.y', 446) - def _reduce_104(val, _values, result) - sym = @grammar.find_symbol_by_id!(val[2]) - @prec_seen = true - builder = val[0] - builder.precedence_sym = sym - result = builder +module_eval(<<'.,.,', 'parser.y', 374) + def _reduce_108(val, _values, result) + builder = val[2] + if !builder.line + builder.line = @lexer.line - 1 + end + result = val[0].append(builder) result end .,., -# reduce 105 omitted +module_eval(<<'.,.,', 'parser.y', 384) + def _reduce_109(val, _values, result) + reset_precs + result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) -# reduce 106 omitted + result + end +.,., -# reduce 107 omitted +module_eval(<<'.,.,', 'parser.y', 389) + def _reduce_110(val, _values, result) + builder = val[0] + builder.add_rhs(Lrama::Lexer::Token::Empty.new(location: @lexer.location)) + result = builder -module_eval(<<'.,.,', 'parser.y', 457) - def _reduce_108(val, _values, result) - result = [val[0]] result end .,., -module_eval(<<'.,.,', 'parser.y', 458) - def _reduce_109(val, _values, result) - result = val[0].append(val[2]) +module_eval(<<'.,.,', 'parser.y', 395) + def _reduce_111(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.add_rhs(token) + result = builder + result end .,., -module_eval(<<'.,.,', 'parser.y', 459) - def _reduce_110(val, _values, result) - result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] +module_eval(<<'.,.,', 'parser.y', 404) + def _reduce_112(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + result end .,., -module_eval(<<'.,.,', 'parser.y', 460) - def _reduce_111(val, _values, result) - result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] +module_eval(<<'.,.,', 'parser.y', 413) + def _reduce_113(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + result end .,., -# reduce 112 omitted +module_eval(<<'.,.,', 'parser.y', 422) + def _reduce_114(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + user_code.tag = val[3] + builder = val[0] + builder.user_code = user_code + result = builder -module_eval(<<'.,.,', 'parser.y', 463) - def _reduce_113(val, _values, result) - result = val[1].s_value result end .,., -# reduce 114 omitted +module_eval(<<'.,.,', 'parser.y', 431) + def _reduce_115(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder -# reduce 115 omitted + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 470) +module_eval(<<'.,.,', 'parser.y', 444) def _reduce_116(val, _values, result) - begin_c_declaration('\Z') - @grammar.epilogue_first_lineno = @lexer.line + 1 - + result = "option" result end .,., -module_eval(<<'.,.,', 'parser.y', 475) +module_eval(<<'.,.,', 'parser.y', 445) def _reduce_117(val, _values, result) - end_c_declaration - @grammar.epilogue = val[2].s_value - + result = "nonempty_list" result end .,., -# reduce 118 omitted +module_eval(<<'.,.,', 'parser.y', 446) + def _reduce_118(val, _values, result) + result = "list" + result + end +.,., # reduce 119 omitted # reduce 120 omitted -# reduce 121 omitted +module_eval(<<'.,.,', 'parser.y', 451) + def _reduce_121(val, _values, result) + result = if val[1] + [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] + else + [val[0]] + end + + result + end +.,., -# reduce 122 omitted +module_eval(<<'.,.,', 'parser.y', 457) + def _reduce_122(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 486) +module_eval(<<'.,.,', 'parser.y', 458) def _reduce_123(val, _values, result) - result = [val[0]] + result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] result end .,., -module_eval(<<'.,.,', 'parser.y', 487) +module_eval(<<'.,.,', 'parser.y', 463) def _reduce_124(val, _values, result) - result = val[0].append(val[1]) + if prec_seen? + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 471) + def _reduce_125(val, _values, result) + end_c_declaration + result end .,., -# reduce 125 omitted +module_eval(<<'.,.,', 'parser.y', 475) + def _reduce_126(val, _values, result) + result = val[2] -# reduce 126 omitted + result + end +.,., -module_eval(<<'.,.,', 'parser.y', 492) +module_eval(<<'.,.,', 'parser.y', 478) def _reduce_127(val, _values, result) - result = Lrama::Lexer::Token::Ident.new(s_value: val[0]) + result = val[1].s_value result end .,., -# reduce 128 omitted +module_eval(<<'.,.,', 'parser.y', 483) + def _reduce_128(val, _values, result) + begin_c_declaration('\Z') -# reduce 129 omitted + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 487) + def _reduce_129(val, _values, result) + end_c_declaration + @grammar.epilogue_first_lineno = val[0].first_line + 1 + @grammar.epilogue = val[2].s_value + + result + end +.,., + +# reduce 130 omitted + +# reduce 131 omitted + +# reduce 132 omitted + +# reduce 133 omitted + +# reduce 134 omitted + +module_eval(<<'.,.,', 'parser.y', 499) + def _reduce_135(val, _values, result) + result = Lrama::Lexer::Token::Ident.new(s_value: val[0].s_value) + result + end +.,., def _reduce_none(val, _values, result) val[0] diff --git a/tool/lrama/lib/lrama/report.rb b/tool/lrama/lib/lrama/report.rb deleted file mode 100644 index 650ac09d52..0000000000 --- a/tool/lrama/lib/lrama/report.rb +++ /dev/null @@ -1,2 +0,0 @@ -require 'lrama/report/duration' -require 'lrama/report/profile' diff --git a/tool/lrama/lib/lrama/report/duration.rb b/tool/lrama/lib/lrama/report/duration.rb deleted file mode 100644 index 7afe284f1a..0000000000 --- a/tool/lrama/lib/lrama/report/duration.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Lrama - class Report - module Duration - def self.enable - @_report_duration_enabled = true - end - - def self.enabled? - !!@_report_duration_enabled - end - - def report_duration(method_name) - time1 = Time.now.to_f - result = yield - time2 = Time.now.to_f - - if Duration.enabled? - puts sprintf("%s %10.5f s", method_name, time2 - time1) - end - - return result - end - end - end -end diff --git a/tool/lrama/lib/lrama/report/profile.rb b/tool/lrama/lib/lrama/report/profile.rb deleted file mode 100644 index 36156800a4..0000000000 --- a/tool/lrama/lib/lrama/report/profile.rb +++ /dev/null @@ -1,14 +0,0 @@ -module Lrama - class Report - module Profile - # See "Profiling Lrama" in README.md for how to use. - def self.report_profile - require "stackprof" - - StackProf.run(mode: :cpu, raw: true, out: 'tmp/stackprof-cpu-myapp.dump') do - yield - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/reporter.rb b/tool/lrama/lib/lrama/reporter.rb new file mode 100644 index 0000000000..ed25cc7f8f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'reporter/conflicts' +require_relative 'reporter/grammar' +require_relative 'reporter/precedences' +require_relative 'reporter/profile' +require_relative 'reporter/rules' +require_relative 'reporter/states' +require_relative 'reporter/terms' + +module Lrama + class Reporter + include Lrama::Tracer::Duration + + # @rbs (**bool options) -> void + def initialize(**options) + @options = options + @rules = Rules.new(**options) + @terms = Terms.new(**options) + @conflicts = Conflicts.new + @precedences = Precedences.new + @grammar = Grammar.new(**options) + @states = States.new(**options) + end + + # @rbs (File io, Lrama::States states) -> void + def report(io, states) + report_duration(:report) do + report_duration(:report_rules) { @rules.report(io, states) } + report_duration(:report_terms) { @terms.report(io, states) } + report_duration(:report_conflicts) { @conflicts.report(io, states) } + report_duration(:report_precedences) { @precedences.report(io, states) } + report_duration(:report_grammar) { @grammar.report(io, states) } + report_duration(:report_states) { @states.report(io, states, ielr: states.ielr_defined?) } + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/conflicts.rb b/tool/lrama/lib/lrama/reporter/conflicts.rb new file mode 100644 index 0000000000..f4d8c604c9 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/conflicts.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Conflicts + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_conflicts(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_conflicts(io, states) + has_conflict = false + + states.states.each do |state| + messages = format_conflict_messages(state.conflicts) + + unless messages.empty? + has_conflict = true + io << "State #{state.id} conflicts: #{messages.join(', ')}\n" + end + end + + io << "\n\n" if has_conflict + end + + # @rbs (Array[(Lrama::State::ShiftReduceConflict | Lrama::State::ReduceReduceConflict)] conflicts) -> Array[String] + def format_conflict_messages(conflicts) + conflict_types = { + shift_reduce: "shift/reduce", + reduce_reduce: "reduce/reduce" + } + + conflict_types.keys.map do |type| + type_conflicts = conflicts.select { |c| c.type == type } + "#{type_conflicts.count} #{conflict_types[type]}" unless type_conflicts.empty? + end.compact + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/grammar.rb b/tool/lrama/lib/lrama/reporter/grammar.rb new file mode 100644 index 0000000000..dc3f3f6bfd --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/grammar.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Grammar + # @rbs (?grammar: bool, **bool _) -> void + def initialize(grammar: false, **_) + @grammar = grammar + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @grammar + + io << "Grammar\n" + last_lhs = nil + + states.rules.each do |rule| + if rule.empty_rule? + r = "ε" + else + r = rule.rhs.map(&:display_name).join(" ") + end + + if rule.lhs == last_lhs + io << sprintf("%5d %s| %s", rule.id, " " * rule.lhs.display_name.length, r) << "\n" + else + io << "\n" + io << sprintf("%5d %s: %s", rule.id, rule.lhs.display_name, r) << "\n" + end + + last_lhs = rule.lhs + end + io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/precedences.rb b/tool/lrama/lib/lrama/reporter/precedences.rb new file mode 100644 index 0000000000..73c0888700 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/precedences.rb @@ -0,0 +1,54 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Precedences + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_precedences(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_precedences(io, states) + used_precedences = states.precedences.select(&:used_by?) + + return if used_precedences.empty? + + io << "Precedences\n\n" + + used_precedences.each do |precedence| + io << " precedence on #{precedence.symbol.display_name} is used to resolve conflict on\n" + + if precedence.used_by_lalr? + io << " LALR\n" + + precedence.used_by_lalr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + + if precedence.used_by_ielr? + io << " IELR\n" + + precedence.used_by_ielr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile.rb b/tool/lrama/lib/lrama/reporter/profile.rb new file mode 100644 index 0000000000..b569b94d4f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'profile/call_stack' +require_relative 'profile/memory' diff --git a/tool/lrama/lib/lrama/reporter/profile/call_stack.rb b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb new file mode 100644 index 0000000000..8a4d44b61c --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module CallStack + # See "Call-stack Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_stackprof + ex = nil #: Exception? + path = 'tmp/stackprof-cpu-myapp.dump' + + StackProf.run(mode: :cpu, raw: true, out: path) do + yield + rescue Exception => e + ex = e + end + + STDERR.puts("Call-stack Profiling result is generated on #{path}") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_stackprof + require "stackprof" + true + rescue LoadError + warn "stackprof is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile/memory.rb b/tool/lrama/lib/lrama/reporter/profile/memory.rb new file mode 100644 index 0000000000..a019581fdf --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/memory.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module Memory + # See "Memory Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_memory_profiler + ex = nil #: Exception? + + report = MemoryProfiler.report do # steep:ignore UnknownConstant + yield + rescue Exception => e + ex = e + end + + report.pretty_print(to_file: "tmp/memory_profiler.txt") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_memory_profiler + require "memory_profiler" + true + rescue LoadError + warn "memory_profiler is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/rules.rb b/tool/lrama/lib/lrama/reporter/rules.rb new file mode 100644 index 0000000000..3e8bf19a0a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/rules.rb @@ -0,0 +1,43 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Rules + # @rbs (?rules: bool, **bool _) -> void + def initialize(rules: false, **_) + @rules = rules + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @rules + + used_rules = states.rules.flat_map(&:rhs) + + unless used_rules.empty? + io << "Rule Usage Frequency\n\n" + frequency_counts = used_rules.each_with_object(Hash.new(0)) { |rule, counts| counts[rule] += 1 } + + frequency_counts + .select { |rule,| !rule.midrule? } + .sort_by { |rule, count| [-count, rule.name] } + .each_with_index { |(rule, count), i| io << sprintf("%5d %s (%d times)", i, rule.name, count) << "\n" } + io << "\n\n" + end + + unused_rules = states.rules.map(&:lhs).select do |rule| + !used_rules.include?(rule) && rule.token_id != 0 + end + + unless unused_rules.empty? + io << "#{unused_rules.count} Unused Rules\n\n" + unused_rules.each_with_index do |rule, index| + io << sprintf("%5d %s", index, rule.display_name) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/states.rb b/tool/lrama/lib/lrama/reporter/states.rb new file mode 100644 index 0000000000..d152d0511a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/states.rb @@ -0,0 +1,387 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class States + # @rbs (?itemsets: bool, ?lookaheads: bool, ?solved: bool, ?counterexamples: bool, ?verbose: bool, **bool _) -> void + def initialize(itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false, **_) + @itemsets = itemsets + @lookaheads = lookaheads + @solved = solved + @counterexamples = counterexamples + @verbose = verbose + end + + # @rbs (IO io, Lrama::States states, ielr: bool) -> void + def report(io, states, ielr: false) + cex = Counterexamples.new(states) if @counterexamples + + states.compute_la_sources_for_conflicted_states + report_split_states(io, states.states) if ielr + + states.states.each do |state| + report_state_header(io, state) + report_items(io, state) + report_conflicts(io, state) + report_shifts(io, state) + report_nonassoc_errors(io, state) + report_reduces(io, state) + report_nterm_transitions(io, state) + report_conflict_resolutions(io, state) if @solved + report_counterexamples(io, state, cex) if @counterexamples && state.has_conflicts? # @type var cex: Lrama::Counterexamples + report_verbose_info(io, state, states) if @verbose + # End of Report State + io << "\n" + end + end + + private + + # @rbs (IO io, Array[Lrama::State] states) -> void + def report_split_states(io, states) + ss = states.select(&:split_state?) + + return if ss.empty? + + io << "Split States\n\n" + + ss.each do |state| + io << " State #{state.id} is split from state #{state.lalr_isocore.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_state_header(io, state) + io << "State #{state.id}\n\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_items(io, state) + last_lhs = nil + list = @itemsets ? state.items : state.kernels + + list.sort_by {|i| [i.rule_id, i.position] }.each do |item| + r = item.empty_rule? ? "ε •" : item.rhs.map(&:display_name).insert(item.position, "•").join(" ") + + l = if item.lhs == last_lhs + " " * item.lhs.id.s_value.length + "|" + else + item.lhs.id.s_value + ":" + end + + la = "" + if @lookaheads && item.end_of_rule? + reduce = state.find_reduce_by_item!(item) + look_ahead = reduce.selected_look_ahead + unless look_ahead.empty? + la = " [#{look_ahead.compact.map(&:display_name).join(", ")}]" + end + end + + last_lhs = item.lhs + io << sprintf("%5i %s %s%s", item.rule_id, l, r, la) << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflicts(io, state) + return if state.conflicts.empty? + + state.conflicts.each do |conflict| + syms = conflict.symbols.map { |sym| sym.display_name } + io << " Conflict on #{syms.join(", ")}. " + + case conflict.type + when :shift_reduce + # @type var conflict: Lrama::State::ShiftReduceConflict + io << "shift/reduce(#{conflict.reduce.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + when :reduce_reduce + # @type var conflict: Lrama::State::ReduceReduceConflict + io << "reduce(#{conflict.reduce1.item.rule.lhs.display_name})/reduce(#{conflict.reduce2.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce1.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + + conflict.reduce2.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + else + raise "Unknown conflict type #{conflict.type}" + end + + io << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_shifts(io, state) + shifts = state.term_transitions.reject(&:not_selected) + + return if shifts.empty? + + next_syms = shifts.map(&:next_sym) + max_len = next_syms.map(&:display_name).map(&:length).max + shifts.each do |shift| + io << " #{shift.next_sym.display_name.ljust(max_len)} shift, and go to state #{shift.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nonassoc_errors(io, state) + error_symbols = state.resolved_conflicts.select { |resolved| resolved.which == :error }.map { |error| error.symbol.display_name } + + return if error_symbols.empty? + + max_len = error_symbols.map(&:length).max + error_symbols.each do |name| + io << " #{name.ljust(max_len)} error (nonassociative)\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_reduces(io, state) + reduce_pairs = [] #: Array[[Lrama::Grammar::Symbol, Lrama::State::Action::Reduce]] + + state.non_default_reduces.each do |reduce| + reduce.look_ahead&.each do |term| + reduce_pairs << [term, reduce] + end + end + + return if reduce_pairs.empty? && !state.default_reduction_rule + + max_len = [ + reduce_pairs.map(&:first).map(&:display_name).map(&:length).max || 0, + state.default_reduction_rule ? "$default".length : 0 + ].max + + reduce_pairs.sort_by { |term, _| term.number }.each do |term, reduce| + rule = reduce.item.rule + io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" + end + + if (r = state.default_reduction_rule) + s = "$default".ljust(max_len) + + if r.initial_rule? + io << " #{s} accept\n" + else + io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nterm_transitions(io, state) + return if state.nterm_transitions.empty? + + goto_transitions = state.nterm_transitions.sort_by do |goto| + goto.next_sym.number + end + + max_len = goto_transitions.map(&:next_sym).map do |nterm| + nterm.id.s_value.length + end.max + goto_transitions.each do |goto| + io << " #{goto.next_sym.id.s_value.ljust(max_len)} go to state #{goto.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflict_resolutions(io, state) + return if state.resolved_conflicts.empty? + + state.resolved_conflicts.each do |resolved| + io << " #{resolved.report_message}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::Counterexamples cex) -> void + def report_counterexamples(io, state, cex) + examples = cex.compute(state) + + examples.each do |example| + is_shift_reduce = example.type == :shift_reduce + label0 = is_shift_reduce ? "shift/reduce" : "reduce/reduce" + label1 = is_shift_reduce ? "Shift derivation" : "First Reduce derivation" + label2 = is_shift_reduce ? "Reduce derivation" : "Second Reduce derivation" + + io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" + io << " #{label1}\n" + + example.derivations1.render_strings_for_report.each do |str| + io << " #{str}\n" + end + + io << " #{label2}\n" + + example.derivations2.render_strings_for_report.each do |str| + io << " #{str}\n" + end + end + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_verbose_info(io, state, states) + report_direct_read_sets(io, state, states) + report_reads_relation(io, state, states) + report_read_sets(io, state, states) + report_includes_relation(io, state, states) + report_lookback_relation(io, state, states) + report_follow_sets(io, state, states) + report_look_ahead_sets(io, state, states) + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_direct_read_sets(io, state, states) + io << " [Direct Read sets]\n" + direct_read_sets = states.direct_read_sets + + state.nterm_transitions.each do |goto| + terms = direct_read_sets[goto] + next unless terms && !terms.empty? + + str = terms.map { |sym| sym.id.s_value }.join(", ") + io << " read #{goto.next_sym.id.s_value} shift #{str}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_reads_relation(io, state, states) + io << " [Reads Relation]\n" + + state.nterm_transitions.each do |goto| + goto2 = states.reads_relation[goto] + next unless goto2 + + goto2.each do |goto2| + io << " (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_read_sets(io, state, states) + io << " [Read sets]\n" + read_sets = states.read_sets + + state.nterm_transitions.each do |goto| + terms = read_sets[goto] + next unless terms && !terms.empty? + + terms.each do |sym| + io << " #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_includes_relation(io, state, states) + io << " [Includes Relation]\n" + + state.nterm_transitions.each do |goto| + gotos = states.includes_relation[goto] + next unless gotos + + gotos.each do |goto2| + io << " (State #{state.id}, #{goto.next_sym.id.s_value}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_lookback_relation(io, state, states) + io << " [Lookback Relation]\n" + + states.rules.each do |rule| + gotos = states.lookback_relation.dig(state.id, rule.id) + next unless gotos + + gotos.each do |goto2| + io << " (Rule: #{rule.display_name}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_follow_sets(io, state, states) + io << " [Follow sets]\n" + follow_sets = states.follow_sets + + state.nterm_transitions.each do |goto| + terms = follow_sets[goto] + next unless terms + + terms.each do |sym| + io << " #{goto.next_sym.id.s_value} -> #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_look_ahead_sets(io, state, states) + io << " [Look-Ahead Sets]\n" + look_ahead_rules = [] #: Array[[Lrama::Grammar::Rule, Array[Lrama::Grammar::Symbol]]] + + states.rules.each do |rule| + syms = states.la.dig(state.id, rule.id) + next unless syms + + look_ahead_rules << [rule, syms] + end + + return if look_ahead_rules.empty? + + max_len = look_ahead_rules.flat_map { |_, syms| syms.map { |s| s.id.s_value.length } }.max + + look_ahead_rules.each do |rule, syms| + syms.each do |sym| + io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/terms.rb b/tool/lrama/lib/lrama/reporter/terms.rb new file mode 100644 index 0000000000..f72d8b1a1a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/terms.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Terms + # @rbs (?terms: bool, **bool _) -> void + def initialize(terms: false, **_) + @terms = terms + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @terms + + look_aheads = states.states.each do |state| + state.reduces.flat_map do |reduce| + reduce.look_ahead unless reduce.look_ahead.nil? + end + end + + next_terms = states.states.flat_map do |state| + state.term_transitions.map {|shift| shift.next_sym } + end + + unused_symbols = states.terms.reject do |term| + (look_aheads + next_terms).include?(term) + end + + io << states.terms.count << " Terms\n\n" + + io << states.nterms.count << " Non-Terminals\n\n" + + unless unused_symbols.empty? + io << "#{unused_symbols.count} Unused Terms\n\n" + unused_symbols.each_with_index do |term, index| + io << sprintf("%5d %s", index, term.id.s_value) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb index 1b40640215..50912e094e 100644 --- a/tool/lrama/lib/lrama/state.rb +++ b/tool/lrama/lib/lrama/state.rb @@ -1,15 +1,62 @@ -require "lrama/state/reduce" -require "lrama/state/reduce_reduce_conflict" -require "lrama/state/resolved_conflict" -require "lrama/state/shift" -require "lrama/state/shift_reduce_conflict" +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "state/action" +require_relative "state/inadequacy_annotation" +require_relative "state/item" +require_relative "state/reduce_reduce_conflict" +require_relative "state/resolved_conflict" +require_relative "state/shift_reduce_conflict" module Lrama class State - attr_reader :id, :accessing_symbol, :kernels, :conflicts, :resolved_conflicts, - :default_reduction_rule, :closure, :items - attr_accessor :shifts, :reduces + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type conflict = State::ShiftReduceConflict | State::ReduceReduceConflict + # type transition = Action::Shift | Action::Goto + # type lookahead_set = Hash[Item, Array[Grammar::Symbol]] + # + # @id: Integer + # @accessing_symbol: Grammar::Symbol + # @kernels: Array[Item] + # @items: Array[Item] + # @items_to_state: Hash[Array[Item], State] + # @conflicts: Array[conflict] + # @resolved_conflicts: Array[ResolvedConflict] + # @default_reduction_rule: Grammar::Rule? + # @closure: Array[Item] + # @nterm_transitions: Array[Action::Goto] + # @term_transitions: Array[Action::Shift] + # @transitions: Array[transition] + # @internal_dependencies: Hash[Action::Goto, Array[Action::Goto]] + # @successor_dependencies: Hash[Action::Goto, Array[Action::Goto]] + + attr_reader :id #: Integer + attr_reader :accessing_symbol #: Grammar::Symbol + attr_reader :kernels #: Array[Item] + attr_reader :conflicts #: Array[conflict] + attr_reader :resolved_conflicts #: Array[ResolvedConflict] + attr_reader :default_reduction_rule #: Grammar::Rule? + attr_reader :closure #: Array[Item] + attr_reader :items #: Array[Item] + attr_reader :annotation_list #: Array[InadequacyAnnotation] + attr_reader :predecessors #: Array[State] + attr_reader :items_to_state #: Hash[Array[Item], State] + attr_reader :lane_items #: Hash[State, Array[[Item, Item]]] + + attr_accessor :_transitions #: Array[[Grammar::Symbol, Array[Item]]] + attr_accessor :reduces #: Array[Action::Reduce] + attr_accessor :ielr_isocores #: Array[State] + attr_accessor :lalr_isocore #: State + attr_accessor :lookaheads_recomputed #: bool + attr_accessor :follow_kernel_items #: Hash[Action::Goto, Hash[Item, bool]] + attr_accessor :always_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor :goto_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void def initialize(id, accessing_symbol, kernels) @id = id @accessing_symbol = accessing_symbol @@ -21,47 +68,77 @@ module Lrama @conflicts = [] @resolved_conflicts = [] @default_reduction_rule = nil + @predecessors = [] + @lalr_isocore = self + @ielr_isocores = [self] + @internal_dependencies = {} + @successor_dependencies = {} + @annotation_list = [] + @lookaheads_recomputed = false + @follow_kernel_items = {} + @always_follows = {} + @goto_follows = {} + @lhs_contributions = {} + @lane_items = {} end + # @rbs (State other) -> bool + def ==(other) + self.id == other.id + end + + # @rbs (Array[Item] closure) -> void def closure=(closure) @closure = closure @items = @kernels + @closure end + # @rbs () -> Array[Action::Reduce] def non_default_reduces - reduces.select do |reduce| - reduce.rule != @default_reduction_rule + reduces.reject do |reduce| + reduce.rule == @default_reduction_rule end end - def compute_shifts_reduces - _shifts = {} + # @rbs () -> void + def compute_transitions_and_reduces + _transitions = {} + @_lane_items ||= {} reduces = [] items.each do |item| # TODO: Consider what should be pushed if item.end_of_rule? - reduces << Reduce.new(item) + reduces << Action::Reduce.new(item) else key = item.next_sym - _shifts[key] ||= [] - _shifts[key] << item.new_by_next_position + _transitions[key] ||= [] + @_lane_items[key] ||= [] + next_item = item.new_by_next_position + _transitions[key] << next_item + @_lane_items[key] << [item, next_item] end end # It seems Bison 3.8.2 iterates transitions order by symbol number - shifts = _shifts.sort_by do |next_sym, new_items| + transitions = _transitions.sort_by do |next_sym, to_items| next_sym.number - end.map do |next_sym, new_items| - Shift.new(next_sym, new_items.flatten) end - self.shifts = shifts.freeze + + self._transitions = transitions.freeze self.reduces = reduces.freeze end + # @rbs (Grammar::Symbol next_sym, State next_state) -> void + def set_lane_items(next_sym, next_state) + @lane_items[next_state] = @_lane_items[next_sym] + end + + # @rbs (Array[Item] items, State next_state) -> void def set_items_to_state(items, next_state) @items_to_state[items] = next_state end + # @rbs (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void def set_look_ahead(rule, look_ahead) reduce = reduces.find do |r| r.rule == rule @@ -70,60 +147,78 @@ module Lrama reduce.look_ahead = look_ahead end - # Returns array of [Shift, next_state] - def nterm_transitions - return @nterm_transitions if @nterm_transitions - - @nterm_transitions = [] - - shifts.each do |shift| - next if shift.next_sym.term? - - @nterm_transitions << [shift, @items_to_state[shift.next_items]] + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void + def set_look_ahead_sources(rule, sources) + reduce = reduces.find do |r| + r.rule == rule end - @nterm_transitions + reduce.look_ahead_sources = sources end - # Returns array of [Shift, next_state] - def term_transitions - return @term_transitions if @term_transitions - - @term_transitions = [] + # @rbs () -> Array[Action::Goto] + def nterm_transitions # steep:ignore + @nterm_transitions ||= transitions.select {|transition| transition.is_a?(Action::Goto) } + end - shifts.each do |shift| - next if shift.next_sym.nterm? + # @rbs () -> Array[Action::Shift] + def term_transitions # steep:ignore + @term_transitions ||= transitions.select {|transition| transition.is_a?(Action::Shift) } + end - @term_transitions << [shift, @items_to_state[shift.next_items]] + # @rbs () -> Array[transition] + def transitions + @transitions ||= _transitions.map do |next_sym, to_items| + if next_sym.term? + Action::Shift.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + else + Action::Goto.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + end end + end - @term_transitions + # @rbs (transition transition, State next_state) -> void + def update_transition(transition, next_state) + set_items_to_state(transition.to_items, next_state) + next_state.append_predecessor(self) + update_transitions_caches(transition) end - def transitions - term_transitions + nterm_transitions + # @rbs () -> void + def update_transitions_caches(transition) + new_transition = + if transition.next_sym.term? + Action::Shift.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + else + Action::Goto.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + end + + @transitions.delete(transition) + @transitions << new_transition + @nterm_transitions = nil + @term_transitions = nil + + @follow_kernel_items[new_transition] = @follow_kernel_items.delete(transition) + @always_follows[new_transition] = @always_follows.delete(transition) end + # @rbs () -> Array[Action::Shift] def selected_term_transitions - term_transitions.select do |shift, next_state| - !shift.not_selected + term_transitions.reject do |shift| + shift.not_selected end end # Move to next state by sym + # + # @rbs (Grammar::Symbol sym) -> State def transition(sym) result = nil if sym.term? - term_transitions.each do |shift, next_state| - term = shift.next_sym - result = next_state if term == sym - end + result = term_transitions.find {|shift| shift.next_sym == sym }.to_state else - nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - result = next_state if nterm == sym - end + result = nterm_transitions.find {|goto| goto.next_sym == sym }.to_state end raise "Can not transit by #{sym} #{self}" if result.nil? @@ -131,12 +226,14 @@ module Lrama result end + # @rbs (Item item) -> Action::Reduce def find_reduce_by_item!(item) reduces.find do |r| r.item == item end || (raise "reduce is not found. #{item}") end + # @rbs (Grammar::Rule default_reduction_rule) -> void def default_reduction_rule=(default_reduction_rule) @default_reduction_rule = default_reduction_rule @@ -147,20 +244,291 @@ module Lrama end end + # @rbs () -> bool def has_conflicts? !@conflicts.empty? end + # @rbs () -> Array[conflict] def sr_conflicts @conflicts.select do |conflict| conflict.type == :shift_reduce end end + # @rbs () -> Array[conflict] def rr_conflicts @conflicts.select do |conflict| conflict.type == :reduce_reduce end end + + # Clear information related to conflicts. + # IELR computation re-calculates conflicts and default reduction of states + # after LALR computation. + # Call this method before IELR computation to avoid duplicated conflicts information + # is stored. + # + # @rbs () -> void + def clear_conflicts + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + + term_transitions.each(&:clear_conflicts) + reduces.each(&:clear_conflicts) + end + + # @rbs () -> bool + def split_state? + @lalr_isocore != self + end + + # Definition 3.40 (propagate_lookaheads) + # + # @rbs (State next_state) -> lookahead_set + def propagate_lookaheads(next_state) + next_state.kernels.map {|next_kernel| + lookahead_sets = + if next_kernel.position > 1 + kernel = kernels.find {|k| k.predecessor_item_of?(next_kernel) } + item_lookahead_set[kernel] + else + goto_follow_set(next_kernel.lhs) + end + + [next_kernel, lookahead_sets & next_state.lookahead_set_filters[next_kernel]] + }.to_h + end + + # Definition 3.43 (is_compatible) + # + # @rbs (lookahead_set filtered_lookahead) -> bool + def is_compatible?(filtered_lookahead) + !lookaheads_recomputed || + @lalr_isocore.annotation_list.all? {|annotation| + a = annotation.dominant_contribution(item_lookahead_set) + b = annotation.dominant_contribution(filtered_lookahead) + a.nil? || b.nil? || a == b + } + end + + # Definition 3.38 (lookahead_set_filters) + # + # @rbs () -> lookahead_set + def lookahead_set_filters + @lookahead_set_filters ||= kernels.map {|kernel| + [kernel, @lalr_isocore.annotation_list.select {|annotation| annotation.contributed?(kernel) }.map(&:token)] + }.to_h + end + + # Definition 3.27 (inadequacy_lists) + # + # @rbs () -> Hash[Grammar::Symbol, Array[Action::Shift | Action::Reduce]] + def inadequacy_list + return @inadequacy_list if @inadequacy_list + + inadequacy_list = {} + + term_transitions.each do |shift| + inadequacy_list[shift.next_sym] ||= [] + inadequacy_list[shift.next_sym] << shift.dup + end + reduces.each do |reduce| + next if reduce.look_ahead.nil? + + reduce.look_ahead.each do |token| + inadequacy_list[token] ||= [] + inadequacy_list[token] << reduce.dup + end + end + + @inadequacy_list = inadequacy_list.select {|token, actions| actions.size > 1 } + end + + # Definition 3.30 (annotate_manifestation) + # + # @rbs () -> void + def annotate_manifestation + inadequacy_list.each {|token, actions| + contribution_matrix = actions.map {|action| + if action.is_a?(Action::Shift) + [action, nil] + else + [action, action.rule.empty_rule? ? lhs_contributions(action.rule.lhs, token) : kernels.map {|k| [k, k.rule == action.item.rule && k.end_of_rule?] }.to_h] + end + }.to_h + @annotation_list << InadequacyAnnotation.new(self, token, actions, contribution_matrix) + } + end + + # Definition 3.32 (annotate_predecessor) + # + # @rbs (State predecessor) -> void + def annotate_predecessor(predecessor) + propagating_list = annotation_list.map {|annotation| + contribution_matrix = annotation.contribution_matrix.map {|action, contributions| + if contributions.nil? + [action, nil] + elsif first_kernels.any? {|kernel| contributions[kernel] && predecessor.lhs_contributions(kernel.lhs, annotation.token).empty? } + [action, nil] + else + cs = predecessor.lane_items[self].map {|pred_kernel, kernel| + c = contributions[kernel] && ( + (kernel.position > 1 && predecessor.item_lookahead_set[pred_kernel].include?(annotation.token)) || + (kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, annotation.token)[pred_kernel]) + ) + [pred_kernel, c] + }.to_h + [action, cs] + end + }.to_h + + # Observation 3.33 (Simple Split-Stable Dominance) + # + # If all of contributions in the contribution_matrix are + # always contribution or never contribution, we can stop annotate propagations + # to the predecessor state. + next nil if contribution_matrix.all? {|_, contributions| contributions.nil? || contributions.all? {|_, contributed| !contributed } } + + InadequacyAnnotation.new(annotation.state, annotation.token, annotation.actions, contribution_matrix) + }.compact + predecessor.append_annotation_list(propagating_list) + end + + # @rbs () -> Array[Item] + def first_kernels + @first_kernels ||= kernels.select {|kernel| kernel.position == 1 } + end + + # @rbs (Array[InadequacyAnnotation] propagating_list) -> void + def append_annotation_list(propagating_list) + annotation_list.each do |annotation| + merging_list = propagating_list.select {|a| a.state == annotation.state && a.token == annotation.token && a.actions == annotation.actions } + annotation.merge_matrix(merging_list.map(&:contribution_matrix)) + propagating_list -= merging_list + end + + @annotation_list += propagating_list + end + + # Definition 3.31 (compute_lhs_contributions) + # + # @rbs (Grammar::Symbol sym, Grammar::Symbol token) -> (nil | Hash[Item, bool]) + def lhs_contributions(sym, token) + return @lhs_contributions[sym][token] unless @lhs_contributions.dig(sym, token).nil? + + transition = nterm_transitions.find {|goto| goto.next_sym == sym } + @lhs_contributions[sym] ||= {} + @lhs_contributions[sym][token] = + if always_follows[transition].include?(token) + {} + else + kernels.map {|kernel| [kernel, follow_kernel_items[transition][kernel] && item_lookahead_set[kernel].include?(token)] }.to_h + end + end + + # Definition 3.26 (item_lookahead_sets) + # + # @rbs () -> lookahead_set + def item_lookahead_set + return @item_lookahead_set if @item_lookahead_set + + @item_lookahead_set = kernels.map {|k| [k, []] }.to_h + @item_lookahead_set = kernels.map {|kernel| + value = + if kernel.lhs.accept_symbol? + [] + elsif kernel.position > 1 + prev_items = predecessors_with_item(kernel) + prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms } + elsif kernel.position == 1 + prev_state = @predecessors.find {|p| p.transitions.any? {|transition| transition.next_sym == kernel.lhs } } + goto = prev_state.nterm_transitions.find {|goto| goto.next_sym == kernel.lhs } + prev_state.goto_follows[goto] + end + [kernel, value] + }.to_h + end + + # @rbs (lookahead_set k) -> void + def item_lookahead_set=(k) + @item_lookahead_set = k + end + + # @rbs (Item item) -> Array[[State, Item]] + def predecessors_with_item(item) + result = [] + @predecessors.each do |pre| + pre.items.each do |i| + result << [pre, i] if i.predecessor_item_of?(item) + end + end + result + end + + # @rbs (State prev_state) -> void + def append_predecessor(prev_state) + @predecessors << prev_state + @predecessors.uniq! + end + + # Definition 3.39 (compute_goto_follow_set) + # + # @rbs (Grammar::Symbol nterm_token) -> Array[Grammar::Symbol] + def goto_follow_set(nterm_token) + return [] if nterm_token.accept_symbol? + goto = @lalr_isocore.nterm_transitions.find {|g| g.next_sym == nterm_token } + + @kernels + .select {|kernel| @lalr_isocore.follow_kernel_items[goto][kernel] } + .map {|kernel| item_lookahead_set[kernel] } + .reduce(@lalr_isocore.always_follows[goto]) {|result, terms| result |= terms } + end + + # Definition 3.8 (Goto Follows Internal Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def internal_dependencies(goto) + return @internal_dependencies[goto] if @internal_dependencies[goto] + + syms = @items.select {|i| + i.next_sym == goto.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0 + }.map(&:lhs).uniq + @internal_dependencies[goto] = nterm_transitions.select {|goto2| syms.include?(goto2.next_sym) } + end + + # Definition 3.5 (Goto Follows Successor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def successor_dependencies(goto) + return @successor_dependencies[goto] if @successor_dependencies[goto] + + @successor_dependencies[goto] = goto.to_state.nterm_transitions.select {|next_goto| next_goto.next_sym.nullable } + end + + # Definition 3.9 (Goto Follows Predecessor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def predecessor_dependencies(goto) + state_items = [] + @kernels.select {|kernel| + kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) + }.each do |item| + queue = predecessors_with_item(item) + until queue.empty? + st, i = queue.pop + if i.position == 0 + state_items << [st, i] + else + st.predecessors_with_item(i).each {|v| queue << v } + end + end + end + + state_items.map {|state, item| + state.nterm_transitions.find {|goto2| goto2.next_sym == item.lhs } + } + end end end diff --git a/tool/lrama/lib/lrama/state/action.rb b/tool/lrama/lib/lrama/state/action.rb new file mode 100644 index 0000000000..791685fc23 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "action/goto" +require_relative "action/reduce" +require_relative "action/shift" diff --git a/tool/lrama/lib/lrama/state/action/goto.rb b/tool/lrama/lib/lrama/state/action/goto.rb new file mode 100644 index 0000000000..4c2c82afdc --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/goto.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Goto + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/reduce.rb b/tool/lrama/lib/lrama/state/action/reduce.rb new file mode 100644 index 0000000000..9678ab0a98 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/reduce.rb @@ -0,0 +1,71 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Reduce + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @item: Item + # @look_ahead: Array[Grammar::Symbol]? + # @look_ahead_sources: Hash[Grammar::Symbol, Array[Action::Goto]]? + # @not_selected_symbols: Array[Grammar::Symbol] + + attr_reader :item #: Item + attr_reader :look_ahead #: Array[Grammar::Symbol]? + attr_reader :look_ahead_sources #: Hash[Grammar::Symbol, Array[Action::Goto]]? + attr_reader :not_selected_symbols #: Array[Grammar::Symbol] + + # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html + attr_accessor :default_reduction #: bool + + # @rbs (Item item) -> void + def initialize(item) + @item = item + @look_ahead = nil + @look_ahead_sources = nil + @not_selected_symbols = [] + end + + # @rbs () -> Grammar::Rule + def rule + @item.rule + end + + # @rbs (Array[Grammar::Symbol] look_ahead) -> Array[Grammar::Symbol] + def look_ahead=(look_ahead) + @look_ahead = look_ahead.freeze + end + + # @rbs (Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> Hash[Grammar::Symbol, Array[Action::Goto]] + def look_ahead_sources=(sources) + @look_ahead_sources = sources.freeze + end + + # @rbs (Grammar::Symbol sym) -> Array[Grammar::Symbol] + def add_not_selected_symbol(sym) + @not_selected_symbols << sym + end + + # @rbs () -> (::Array[Grammar::Symbol?]) + def selected_look_ahead + if look_ahead + look_ahead - @not_selected_symbols + else + [] + end + end + + # @rbs () -> void + def clear_conflicts + @not_selected_symbols = [] + @default_reduction = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/shift.rb b/tool/lrama/lib/lrama/state/action/shift.rb new file mode 100644 index 0000000000..52d9f8c4f0 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/shift.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Shift + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + attr_accessor :not_selected #: bool + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + + # @rbs () -> void + def clear_conflicts + @not_selected = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/inadequacy_annotation.rb b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb new file mode 100644 index 0000000000..3654fa4607 --- /dev/null +++ b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb @@ -0,0 +1,140 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class InadequacyAnnotation + # @rbs! + # type action = Action::Shift | Action::Reduce + + attr_accessor :state #: State + attr_accessor :token #: Grammar::Symbol + attr_accessor :actions #: Array[action] + attr_accessor :contribution_matrix #: Hash[action, Hash[Item, bool]] + + # @rbs (State state, Grammar::Symbol token, Array[action] actions, Hash[action, Hash[Item, bool]] contribution_matrix) -> void + def initialize(state, token, actions, contribution_matrix) + @state = state + @token = token + @actions = actions + @contribution_matrix = contribution_matrix + end + + # @rbs (Item item) -> bool + def contributed?(item) + @contribution_matrix.any? {|action, contributions| !contributions.nil? && contributions[item] } + end + + # @rbs (Array[Hash[action, Hash[Item, bool]]] another_matrixes) -> void + def merge_matrix(another_matrixes) + another_matrixes.each do |another_matrix| + @contribution_matrix.merge!(another_matrix) {|action, contributions, another_contributions| + next contributions if another_contributions.nil? + next another_contributions if contributions.nil? + + contributions.merge!(another_contributions) {|_, contributed, another_contributed| contributed || another_contributed } + } + end + end + + # Definition 3.42 (dominant_contribution) + # + # @rbs (State::lookahead_set lookaheads) -> Array[action]? + def dominant_contribution(lookaheads) + actions = @actions.select {|action| + contribution_matrix[action].nil? || contribution_matrix[action].any? {|item, contributed| contributed && lookaheads[item].include?(@token) } + } + return nil if actions.empty? + + resolve_conflict(actions) + end + + # @rbs (Array[action] actions) -> Array[action] + def resolve_conflict(actions) + # @type var shifts: Array[Action::Shift] + # @type var reduces: Array[Action::Reduce] + shifts = actions.select {|action| action.is_a?(Action::Shift)} + reduces = actions.select {|action| action.is_a?(Action::Reduce) } + + shifts.each do |shift| + reduces.each do |reduce| + sym = shift.next_sym + + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + actions.delete(shift) + next + when shift_prec > reduce_prec + # Shift is selected + actions.delete(reduce) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence&.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + next + when :right + # Shift is selected + actions.delete(reduce) + next + when :left + # Reduce is selected + actions.delete(shift) + next + when :nonassoc + # Can not resolve + # + # nonassoc creates "run-time" error, precedence creates "compile-time" error. + # Then omit both the shift and reduce. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + actions.delete(shift) + actions.delete(reduce) + else + raise "Unknown precedence type. #{sym}" + end + end + end + + actions + end + + # @rbs () -> String + def to_s + "State: #{@state.id}, Token: #{@token.id.s_value}, Actions: #{actions_to_s}, Contributions: #{contribution_matrix_to_s}" + end + + private + + # @rbs () -> String + def actions_to_s + '[' + @actions.map {|action| + if action.is_a?(Action::Shift) || action.is_a?(Action::Goto) + action.class.name + elsif action.is_a?(Action::Reduce) + "#{action.class.name}: (#{action.item})" + end + }.join(', ') + ']' + end + + # @rbs () -> String + def contribution_matrix_to_s + '[' + @contribution_matrix.map {|action, contributions| + "#{(action.is_a?(Action::Shift) || action.is_a?(Action::Goto)) ? action.class.name : "#{action.class.name}: (#{action.item})"}: " + contributions&.transform_keys(&:to_s).to_s + }.join(', ') + ']' + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/item.rb b/tool/lrama/lib/lrama/state/item.rb new file mode 100644 index 0000000000..3ecdd70b76 --- /dev/null +++ b/tool/lrama/lib/lrama/state/item.rb @@ -0,0 +1,120 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# TODO: Validate position is not over rule rhs + +require "forwardable" + +module Lrama + class State + class Item < Struct.new(:rule, :position, keyword_init: true) + # @rbs! + # include Grammar::Rule::_DelegatedMethods + # + # attr_accessor rule: Grammar::Rule + # attr_accessor position: Integer + # + # def initialize: (?rule: Grammar::Rule, ?position: Integer) -> void + + extend Forwardable + + def_delegators "rule", :lhs, :rhs + + # Optimization for States#setup_state + # + # @rbs () -> Integer + def hash + [rule_id, position].hash + end + + # @rbs () -> Integer + def rule_id + rule.id + end + + # @rbs () -> bool + def empty_rule? + rule.empty_rule? + end + + # @rbs () -> Integer + def number_of_rest_symbols + @number_of_rest_symbols ||= rhs.count - position + end + + # @rbs () -> Grammar::Symbol + def next_sym + rhs[position] + end + + # @rbs () -> Grammar::Symbol + def next_next_sym + @next_next_sym ||= rhs[position + 1] + end + + # @rbs () -> Grammar::Symbol + def previous_sym + rhs[position - 1] + end + + # @rbs () -> bool + def end_of_rule? + rhs.count == position + end + + # @rbs () -> bool + def beginning_of_rule? + position == 0 + end + + # @rbs () -> bool + def start_item? + rule.initial_rule? && beginning_of_rule? + end + + # @rbs () -> State::Item + def new_by_next_position + Item.new(rule: rule, position: position + 1) + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_before_dot # steep:ignore + rhs[0...position] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_dot # steep:ignore + rhs[position..-1] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_transition # steep:ignore + rhs[position+1..-1] + end + + # @rbs () -> ::String + def to_s + "#{lhs.id.s_value}: #{display_name}" + end + + # @rbs () -> ::String + def display_name + r = rhs.map(&:display_name).insert(position, "•").join(" ") + "#{r} (rule #{rule_id})" + end + + # Right after position + # + # @rbs () -> ::String + def display_rest + r = symbols_after_dot.map(&:display_name).join(" ") + ". #{r} (rule #{rule_id})" + end + + # @rbs (State::Item other_item) -> bool + def predecessor_item_of?(other_item) + rule == other_item.rule && position == other_item.position - 1 + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/reduce.rb b/tool/lrama/lib/lrama/state/reduce.rb deleted file mode 100644 index 8ba51f45f2..0000000000 --- a/tool/lrama/lib/lrama/state/reduce.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Lrama - class State - class Reduce - # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html - attr_reader :item, :look_ahead, :not_selected_symbols - attr_accessor :default_reduction - - def initialize(item) - @item = item - @look_ahead = nil - @not_selected_symbols = [] - end - - def rule - @item.rule - end - - def look_ahead=(look_ahead) - @look_ahead = look_ahead.freeze - end - - def add_not_selected_symbol(sym) - @not_selected_symbols << sym - end - - def selected_look_ahead - if @look_ahead - @look_ahead - @not_selected_symbols - else - [] - end - end - end - end -end diff --git a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb index 0a0e4dc20a..55ecad40bd 100644 --- a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb +++ b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb @@ -1,6 +1,21 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class State - class ReduceReduceConflict < Struct.new(:symbols, :reduce1, :reduce2, keyword_init: true) + class ReduceReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :reduce1 #: State::Action::Reduce + attr_reader :reduce2 #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], reduce1: State::Action::Reduce, reduce2: State::Action::Reduce) -> void + def initialize(symbols:, reduce1:, reduce2:) + @symbols = symbols + @reduce1 = reduce1 + @reduce2 = reduce2 + end + + # @rbs () -> :reduce_reduce def type :reduce_reduce end diff --git a/tool/lrama/lib/lrama/state/resolved_conflict.rb b/tool/lrama/lib/lrama/state/resolved_conflict.rb index 02ea892147..014533c233 100644 --- a/tool/lrama/lib/lrama/state/resolved_conflict.rb +++ b/tool/lrama/lib/lrama/state/resolved_conflict.rb @@ -1,18 +1,54 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class State + # * state: A state on which the conflct is resolved # * symbol: A symbol under discussion # * reduce: A reduce under discussion # * which: For which a conflict is resolved. :shift, :reduce or :error (for nonassociative) - class ResolvedConflict < Struct.new(:symbol, :reduce, :which, :same_prec, keyword_init: true) + # * resolved_by_precedence: If the conflict is resolved by precedence definition or not + class ResolvedConflict + # @rbs! + # type which_enum = :reduce | :shift | :error + + attr_reader :state #: State + attr_reader :symbol #: Grammar::Symbol + attr_reader :reduce #: State::Action::Reduce + attr_reader :which #: which_enum + attr_reader :resolved_by_precedence #: bool + + # @rbs (state: State, symbol: Grammar::Symbol, reduce: State::Action::Reduce, which: which_enum, resolved_by_precedence: bool) -> void + def initialize(state:, symbol:, reduce:, which:, resolved_by_precedence:) + @state = state + @symbol = symbol + @reduce = reduce + @which = which + @resolved_by_precedence = resolved_by_precedence + end + + # @rbs () -> (::String | bot) def report_message + "Conflict between rule #{reduce.rule.id} and token #{symbol.display_name} #{how_resolved}." + end + + # @rbs () -> (::String | bot) + def report_precedences_message + "Conflict between reduce by \"#{reduce.rule.display_name}\" and shift #{symbol.display_name} #{how_resolved}." + end + + private + + # @rbs () -> (::String | bot) + def how_resolved s = symbol.display_name - r = reduce.rule.precedence_sym.display_name + r = reduce.rule.precedence_sym&.display_name case - when which == :shift && same_prec + when which == :shift && resolved_by_precedence msg = "resolved as #{which} (%right #{s})" when which == :shift msg = "resolved as #{which} (#{r} < #{s})" - when which == :reduce && same_prec + when which == :reduce && resolved_by_precedence msg = "resolved as #{which} (%left #{s})" when which == :reduce msg = "resolved as #{which} (#{s} < #{r})" @@ -22,7 +58,7 @@ module Lrama raise "Unknown direction. #{self}" end - "Conflict between rule #{reduce.rule.id} and token #{s} #{msg}." + msg end end end diff --git a/tool/lrama/lib/lrama/state/shift.rb b/tool/lrama/lib/lrama/state/shift.rb deleted file mode 100644 index 2021eb61f6..0000000000 --- a/tool/lrama/lib/lrama/state/shift.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Lrama - class State - class Shift - attr_reader :next_sym, :next_items - attr_accessor :not_selected - - def initialize(next_sym, next_items) - @next_sym = next_sym - @next_items = next_items - end - end - end -end diff --git a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb index f80bd5f352..548f2de614 100644 --- a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb +++ b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb @@ -1,6 +1,21 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama class State - class ShiftReduceConflict < Struct.new(:symbols, :shift, :reduce, keyword_init: true) + class ShiftReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :shift #: State::Action::Shift + attr_reader :reduce #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], shift: State::Action::Shift, reduce: State::Action::Reduce) -> void + def initialize(symbols:, shift:, reduce:) + @symbols = symbols + @shift = shift + @reduce = reduce + end + + # @rbs () -> :shift_reduce def type :shift_reduce end diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb index 290e996b82..ddce627df4 100644 --- a/tool/lrama/lib/lrama/states.rb +++ b/tool/lrama/lib/lrama/states.rb @@ -1,6 +1,9 @@ +# rbs_inline: enabled +# frozen_string_literal: true + require "forwardable" -require "lrama/report/duration" -require "lrama/states/item" +require_relative "tracer/duration" +require_relative "state/item" module Lrama # States is passed to a template file @@ -8,18 +11,42 @@ module Lrama # "Efficient Computation of LALR(1) Look-Ahead Sets" # https://dl.acm.org/doi/pdf/10.1145/69622.357187 class States + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type state_id = Integer + # type rule_id = Integer + # + # include Grammar::_DelegatedMethods + # + # @grammar: Grammar + # @tracer: Tracer + # @states: Array[State] + # @direct_read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @reads_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @includes_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @lookback_relation: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + # @follow_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @la: Hash[state_id, Hash[rule_id, Bitmap::bitmap]] + extend Forwardable - include Lrama::Report::Duration + include Lrama::Tracer::Duration - def_delegators "@grammar", :symbols, :terms, :nterms, :rules, - :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value! + def_delegators "@grammar", :symbols, :terms, :nterms, :rules, :precedences, + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined? - attr_reader :states, :reads_relation, :includes_relation, :lookback_relation + attr_reader :states #: Array[State] + attr_reader :reads_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :includes_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :lookback_relation #: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] - def initialize(grammar, warning, trace_state: false) + # @rbs (Grammar grammar, Tracer tracer) -> void + def initialize(grammar, tracer) @grammar = grammar - @warning = warning - @trace_state = trace_state + @tracer = tracer @states = [] @@ -27,7 +54,7 @@ module Lrama # where p is state, A is nterm, t is term. # # `@direct_read_sets` is a hash whose - # key is [state.id, nterm.token_id], + # key is goto, # value is bitmap of term. @direct_read_sets = {} @@ -36,12 +63,14 @@ module Lrama # where p, r are state, A, C are nterm. # # `@reads_relation` is a hash whose - # key is [state.id, nterm.token_id], - # value is array of [state.id, nterm.token_id]. + # key is goto, + # value is array of goto. @reads_relation = {} + # `Read(p, A) =s DR(p, A) ∪ ∪{Read(r, C) | (p, A) reads (r, C)}` + # # `@read_sets` is a hash whose - # key is [state.id, nterm.token_id], + # key is goto, # value is bitmap of term. @read_sets = {} @@ -49,98 +78,163 @@ module Lrama # where p, p' are state, A, B are nterm, β, γ is sequence of symbol. # # `@includes_relation` is a hash whose - # key is [state.id, nterm.token_id], - # value is array of [state.id, nterm.token_id]. + # key is goto, + # value is array of goto. @includes_relation = {} # `(q, A -> ω) lookback (p, A) iff p -(ω)-> q` # where p, q are state, A -> ω is rule, A is nterm, ω is sequence of symbol. # - # `@lookback_relation` is a hash whose - # key is [state.id, rule.id], - # value is array of [state.id, nterm.token_id]. + # `@lookback_relation` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is array of goto. @lookback_relation = {} + # `Follow(p, A) =s Read(p, A) ∪ ∪{Follow(p', B) | (p, A) includes (p', B)}` + # # `@follow_sets` is a hash whose - # key is [state.id, rule.id], + # key is goto, # value is bitmap of term. @follow_sets = {} # `LA(q, A -> ω) = ∪{Follow(p, A) | (q, A -> ω) lookback (p, A)` # - # `@la` is a hash whose - # key is [state.id, rule.id], + # `@la` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, # value is bitmap of term. @la = {} end + # @rbs () -> void def compute - # Look Ahead Sets report_duration(:compute_lr0_states) { compute_lr0_states } - report_duration(:compute_direct_read_sets) { compute_direct_read_sets } - report_duration(:compute_reads_relation) { compute_reads_relation } - report_duration(:compute_read_sets) { compute_read_sets } - report_duration(:compute_includes_relation) { compute_includes_relation } - report_duration(:compute_lookback_relation) { compute_lookback_relation } - report_duration(:compute_follow_sets) { compute_follow_sets } + + # Look Ahead Sets report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } # Conflicts - report_duration(:compute_conflicts) { compute_conflicts } + report_duration(:compute_conflicts) { compute_conflicts(:lalr) } report_duration(:compute_default_reduction) { compute_default_reduction } - - check_conflicts end - def reporter - StatesReporter.new(self) + # @rbs () -> void + def compute_ielr + # Preparation + report_duration(:clear_conflicts) { clear_conflicts } + # Phase 1 + report_duration(:compute_predecessors) { compute_predecessors } + report_duration(:compute_follow_kernel_items) { compute_follow_kernel_items } + report_duration(:compute_always_follows) { compute_always_follows } + report_duration(:compute_goto_follows) { compute_goto_follows } + # Phase 2 + report_duration(:compute_inadequacy_annotations) { compute_inadequacy_annotations } + # Phase 3 + report_duration(:split_states) { split_states } + # Phase 4 + report_duration(:clear_look_ahead_sets) { clear_look_ahead_sets } + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + # Phase 5 + report_duration(:compute_conflicts) { compute_conflicts(:ielr) } + report_duration(:compute_default_reduction) { compute_default_reduction } end + # @rbs () -> Integer def states_count @states.count end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def direct_read_sets - @direct_read_sets.transform_values do |v| + @_direct_read_sets ||= @direct_read_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def read_sets - @read_sets.transform_values do |v| + @_read_sets ||= @read_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] def follow_sets - @follow_sets.transform_values do |v| + @_follow_sets ||= @follow_sets.transform_values do |v| bitmap_to_terms(v) end end + # @rbs () -> Hash[state_id, Hash[rule_id, Array[Grammar::Symbol]]] def la - @la.transform_values do |v| - bitmap_to_terms(v) + @_la ||= @la.transform_values do |second_hash| + second_hash.transform_values do |v| + bitmap_to_terms(v) + end end end - private + # @rbs () -> Integer + def sr_conflicts_count + @sr_conflicts_count ||= @states.flat_map(&:sr_conflicts).count + end - def sr_conflicts - @states.flat_map(&:sr_conflicts) + # @rbs () -> Integer + def rr_conflicts_count + @rr_conflicts_count ||= @states.flat_map(&:rr_conflicts).count end - def rr_conflicts - @states.flat_map(&:rr_conflicts) + # @rbs (Logger logger) -> void + def validate!(logger) + validate_conflicts_within_threshold!(logger) end - def trace_state - if @trace_state - yield STDERR + def compute_la_sources_for_conflicted_states + reflexive = {} + @states.each do |state| + state.nterm_transitions.each do |goto| + reflexive[goto] = [goto] + end + end + + # compute_read_sets + read_sets = Digraph.new(nterm_transitions, @reads_relation, reflexive).compute + # compute_follow_sets + follow_sets = Digraph.new(nterm_transitions, @includes_relation, read_sets).compute + + @states.select(&:has_conflicts?).each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + sources = {} + + ary.each do |goto| + source = follow_sets[goto] + + next unless source + + source.each do |goto2| + tokens = direct_read_sets[goto2] + tokens.each do |token| + sources[token] ||= [] + sources[token] |= [goto2] + end + end + end + + state.set_look_ahead_sources(rule, sources) + end end end + private + + # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] def create_state(accessing_symbol, kernels, states_created) # A item can appear in some states, # so need to use `kernels` (not `kernels.first`) as a key. @@ -187,27 +281,25 @@ module Lrama return [state, true] end + # @rbs (State state) -> void def setup_state(state) # closure closure = [] - visited = {} queued = {} items = state.kernels.dup items.each do |item| - queued[item] = true + queued[item.rule_id] = true if item.position == 0 end while (item = items.shift) do - visited[item] = true - if (sym = item.next_sym) && sym.nterm? @grammar.find_rules_by_symbol!(sym).each do |rule| - i = Item.new(rule: rule, position: 0) - next if queued[i] + next if queued[rule.id] + i = State::Item.new(rule: rule, position: 0) closure << i items << i - queued[i] = true + queued[i.rule_id] = true end end end @@ -215,116 +307,107 @@ module Lrama state.closure = closure.sort_by {|i| i.rule.id } # Trace - trace_state do |out| - out << "Closure: input\n" - state.kernels.each do |item| - out << " #{item.display_rest}\n" - end - out << "\n\n" - out << "Closure: output\n" - state.items.each do |item| - out << " #{item.display_rest}\n" - end - out << "\n\n" - end + @tracer.trace_closure(state) # shift & reduce - state.compute_shifts_reduces + state.compute_transitions_and_reduces end + # @rbs (Array[State] states, State state) -> void def enqueue_state(states, state) # Trace - previous = state.kernels.first.previous_sym - trace_state do |out| - out << sprintf("state_list_append (state = %d, symbol = %d (%s))", - @states.count, previous.number, previous.display_name) - end + @tracer.trace_state_list_append(@states.count, state) states << state end + # @rbs () -> void def compute_lr0_states # State queue states = [] states_created = {} - state, _ = create_state(symbols.first, [Item.new(rule: @grammar.rules.first, position: 0)], states_created) + state, _ = create_state(symbols.first, [State::Item.new(rule: @grammar.rules.first, position: 0)], states_created) enqueue_state(states, state) while (state = states.shift) do # Trace - # - # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but - # I think it is not correct... - previous = state.kernels.first.previous_sym - trace_state do |out| - out << "Processing state #{state.id} (reached by #{previous.display_name})\n" - end + @tracer.trace_state(state) setup_state(state) - state.shifts.each do |shift| - new_state, created = create_state(shift.next_sym, shift.next_items, states_created) - state.set_items_to_state(shift.next_items, new_state) + # `State#transitions` can not be used here + # because `items_to_state` of the `state` is not set yet. + state._transitions.each do |next_sym, to_items| + new_state, created = create_state(next_sym, to_items, states_created) + state.set_items_to_state(to_items, new_state) + state.set_lane_items(next_sym, new_state) enqueue_state(states, new_state) if created end end end + # @rbs () -> Array[State::Action::Goto] def nterm_transitions a = [] @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - a << [state, nterm, next_state] + state.nterm_transitions.each do |goto| + a << goto end end a end + # @rbs () -> void + def compute_look_ahead_sets + report_duration(:compute_direct_read_sets) { compute_direct_read_sets } + report_duration(:compute_reads_relation) { compute_reads_relation } + report_duration(:compute_read_sets) { compute_read_sets } + report_duration(:compute_includes_relation) { compute_includes_relation } + report_duration(:compute_lookback_relation) { compute_lookback_relation } + report_duration(:compute_follow_sets) { compute_follow_sets } + report_duration(:compute_la) { compute_la } + end + + # @rbs () -> void def compute_direct_read_sets @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - - ary = next_state.term_transitions.map do |shift, _| + state.nterm_transitions.each do |goto| + ary = goto.to_state.term_transitions.map do |shift| shift.next_sym.number end - key = [state.id, nterm.token_id] - @direct_read_sets[key] = Bitmap.from_array(ary) + @direct_read_sets[goto] = Bitmap.from_array(ary) end end end + # @rbs () -> void def compute_reads_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - next_state.nterm_transitions.each do |shift2, _next_state2| - nterm2 = shift2.next_sym + state.nterm_transitions.each do |goto| + goto.to_state.nterm_transitions.each do |goto2| + nterm2 = goto2.next_sym if nterm2.nullable - key = [state.id, nterm.token_id] - @reads_relation[key] ||= [] - @reads_relation[key] << [next_state.id, nterm2.token_id] + @reads_relation[goto] ||= [] + @reads_relation[goto] << goto2 end end end end end + # @rbs () -> void def compute_read_sets - sets = nterm_transitions.map do |state, nterm, next_state| - [state.id, nterm.token_id] - end - - @read_sets = Digraph.new(sets, @reads_relation, @direct_read_sets).compute + @read_sets = Digraph.new(nterm_transitions, @reads_relation, @direct_read_sets).compute end # Execute transition of state by symbols # then return final state. + # + # @rbs (State state, Array[Grammar::Symbol] symbols) -> State def transition(state, symbols) symbols.each do |sym| state = state.transition(sym) @@ -333,10 +416,11 @@ module Lrama state end + # @rbs () -> void def compute_includes_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym + state.nterm_transitions.each do |goto| + nterm = goto.next_sym @grammar.find_rules_by_symbol!(nterm).each do |rule| i = rule.rhs.count - 1 @@ -346,11 +430,13 @@ module Lrama break if sym.term? state2 = transition(state, rule.rhs[0...i]) # p' = state, B = nterm, p = state2, A = sym - key = [state2.id, sym.token_id] + key = state2.nterm_transitions.find do |goto2| + goto2.next_sym.token_id == sym.token_id + end || (raise "Goto by #{sym.name} on state #{state2.id} is not found") # TODO: need to omit if state == state2 ? @includes_relation[key] ||= [] - @includes_relation[key] << [state.id, nterm.token_id] - break if !sym.nullable + @includes_relation[key] << goto + break unless sym.nullable i -= 1 end end @@ -358,45 +444,46 @@ module Lrama end end + # @rbs () -> void def compute_lookback_relation @states.each do |state| - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym + state.nterm_transitions.each do |goto| + nterm = goto.next_sym @grammar.find_rules_by_symbol!(nterm).each do |rule| state2 = transition(state, rule.rhs) # p = state, A = nterm, q = state2, A -> ω = rule - key = [state2.id, rule.id] - @lookback_relation[key] ||= [] - @lookback_relation[key] << [state.id, nterm.token_id] + @lookback_relation[state2.id] ||= {} + @lookback_relation[state2.id][rule.id] ||= [] + @lookback_relation[state2.id][rule.id] << goto end end end end + # @rbs () -> void def compute_follow_sets - sets = nterm_transitions.map do |state, nterm, next_state| - [state.id, nterm.token_id] - end - - @follow_sets = Digraph.new(sets, @includes_relation, @read_sets).compute + @follow_sets = Digraph.new(nterm_transitions, @includes_relation, @read_sets).compute end - def compute_look_ahead_sets + # @rbs () -> void + def compute_la @states.each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state rules.each do |rule| - ary = @lookback_relation[[state.id, rule.id]] - next if !ary + ary = lookback_relation_on_state[rule.id] + next unless ary - ary.each do |state2_id, nterm_token_id| + ary.each do |goto| # q = state, A -> ω = rule, p = state2, A = nterm - follows = @follow_sets[[state2_id, nterm_token_id]] + follows = @follow_sets[goto] next if follows == 0 - key = [state.id, rule.id] - @la[key] ||= 0 - look_ahead = @la[key] | follows - @la[key] |= look_ahead + @la[state.id] ||= {} + @la[state.id][rule.id] ||= 0 + look_ahead = @la[state.id][rule.id] | follows + @la[state.id][rule.id] |= look_ahead # No risk of conflict when # * the state only has single reduce @@ -409,6 +496,7 @@ module Lrama end end + # @rbs (Bitmap::bitmap bit) -> Array[Grammar::Symbol] def bitmap_to_terms(bit) ary = Bitmap.to_array(bit) ary.map do |i| @@ -416,19 +504,21 @@ module Lrama end end - def compute_conflicts - compute_shift_reduce_conflicts + # @rbs () -> void + def compute_conflicts(lr_type) + compute_shift_reduce_conflicts(lr_type) compute_reduce_reduce_conflicts end - def compute_shift_reduce_conflicts + # @rbs () -> void + def compute_shift_reduce_conflicts(lr_type) states.each do |state| - state.shifts.each do |shift| + state.term_transitions.each do |shift| state.reduces.each do |reduce| sym = shift.next_sym next unless reduce.look_ahead - next if !reduce.look_ahead.include?(sym) + next unless reduce.look_ahead.include?(sym) # Shift/Reduce conflict shift_prec = sym.precedence @@ -443,43 +533,57 @@ module Lrama case when shift_prec < reduce_prec # Reduce is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :reduce) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when shift_prec > reduce_prec # Shift is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next end # shift_prec == reduce_prec, then check associativity case sym.precedence.type when :precedence + # Can not resolve the conflict + # # %precedence only specifies precedence and not specify associativity # then a conflict is unresolved if precedence is same. state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) next when :right # Shift is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift, same_prec: true) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when :left # Reduce is selected - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :reduce, same_prec: true) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) next when :nonassoc - # Can not resolve + # The conflict is resolved # - # nonassoc creates "run-time" error, precedence creates "compile-time" error. - # Then omit both the shift and reduce. + # %nonassoc creates "run-time" error by removing both shift and reduce from + # the state. This makes the state to get syntax error if the conflicted token appears. + # On the other hand, %precedence creates "compile-time" error by keeping both + # shift and reduce on the state. This makes the state to be conflicted on the token. # # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html - state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :error) + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :error, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict shift.not_selected = true reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) else raise "Unknown precedence type. #{sym}" end @@ -488,35 +592,41 @@ module Lrama end end + # @rbs (Grammar::Precedence shift_prec, Grammar::Precedence reduce_prec, State::ResolvedConflict resolved_conflict) -> void + def mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + case lr_type + when :lalr + shift_prec.mark_used_by_lalr(resolved_conflict) + reduce_prec.mark_used_by_lalr(resolved_conflict) + when :ielr + shift_prec.mark_used_by_ielr(resolved_conflict) + reduce_prec.mark_used_by_ielr(resolved_conflict) + end + end + + # @rbs () -> void def compute_reduce_reduce_conflicts states.each do |state| - count = state.reduces.count - - for i in 0...count do - reduce1 = state.reduces[i] - next if reduce1.look_ahead.nil? + state.reduces.combination(2) do |reduce1, reduce2| + next if reduce1.look_ahead.nil? || reduce2.look_ahead.nil? - for j in (i+1)...count do - reduce2 = state.reduces[j] - next if reduce2.look_ahead.nil? + intersection = reduce1.look_ahead & reduce2.look_ahead - intersection = reduce1.look_ahead & reduce2.look_ahead - - if !intersection.empty? - state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) - end + unless intersection.empty? + state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) end end end end + # @rbs () -> void def compute_default_reduction states.each do |state| next if state.reduces.empty? # Do not set, if conflict exist - next if !state.conflicts.empty? + next unless state.conflicts.empty? # Do not set, if shift with `error` exists. - next if state.shifts.map(&:next_sym).include?(@grammar.error_symbol) + next if state.term_transitions.map {|shift| shift.next_sym }.include?(@grammar.error_symbol) state.default_reduction_rule = state.reduces.map do |r| [r.rule, r.rule.id, (r.look_ahead || []).count] @@ -526,31 +636,232 @@ module Lrama end end - def check_conflicts - sr_count = sr_conflicts.count - rr_count = rr_conflicts.count + # @rbs () -> void + def clear_conflicts + states.each(&:clear_conflicts) + end + + # Definition 3.15 (Predecessors) + # + # @rbs () -> void + def compute_predecessors + @states.each do |state| + state.transitions.each do |transition| + transition.to_state.append_predecessor(state) + end + end + end + + # Definition 3.16 (follow_kernel_items) + # + # @rbs () -> void + def compute_follow_kernel_items + set = nterm_transitions + relation = compute_goto_internal_relation + base_function = compute_goto_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, follow_kernel_items| + state = goto.from_state + state.follow_kernel_items[goto] = state.kernels.map {|kernel| + [kernel, Bitmap.to_bool_array(follow_kernel_items, state.kernels.count)] + }.to_h + end + end - if @grammar.expect + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_relation + relations = {} - expected_sr_conflicts = @grammar.expect - expected_rr_conflicts = 0 + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_goto_bitmaps + nterm_transitions.map {|goto| + bools = goto.from_state.kernels.map.with_index {|kernel, i| i if kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) }.compact + [goto, Bitmap.from_array(bools)] + }.to_h + end + + # Definition 3.20 (always_follows, one closure) + # + # @rbs () -> void + def compute_always_follows + set = nterm_transitions + relation = compute_goto_successor_or_internal_relation + base_function = compute_transition_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, always_follows_bitmap| + goto.from_state.always_follows[goto] = bitmap_to_terms(always_follows_bitmap) + end + end - if expected_sr_conflicts != sr_count - @warning.error("shift/reduce conflicts: #{sr_count} found, #{expected_sr_conflicts} expected") + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_successor_or_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.successor_dependencies(goto) + state.internal_dependencies(goto) end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_transition_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.to_state.term_transitions.map {|shift| shift.next_sym.number })] + }.to_h + end + + # Definition 3.24 (goto_follows, via always_follows) + # + # @rbs () -> void + def compute_goto_follows + set = nterm_transitions + relation = compute_goto_internal_or_predecessor_dependencies + base_function = compute_always_follows_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, goto_follows_bitmap| + goto.from_state.goto_follows[goto] = bitmap_to_terms(goto_follows_bitmap) + end + end - if expected_rr_conflicts != rr_count - @warning.error("reduce/reduce conflicts: #{rr_count} found, #{expected_rr_conflicts} expected") + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_or_predecessor_dependencies + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + state.predecessor_dependencies(goto) end - else - if sr_count != 0 - @warning.warn("shift/reduce conflicts: #{sr_count} found") + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_always_follows_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.from_state.always_follows[goto].map(&:number))] + }.to_h + end + + # @rbs () -> void + def split_states + @states.each do |state| + state.transitions.each do |transition| + compute_state(state, transition, transition.to_state) end + end + end + + # @rbs () -> void + def compute_inadequacy_annotations + @states.each do |state| + state.annotate_manifestation + end - if rr_count != 0 - @warning.warn("reduce/reduce conflicts: #{rr_count} found") + queue = @states.reject {|state| state.annotation_list.empty? } + + while (curr = queue.shift) do + curr.predecessors.each do |pred| + cache = pred.annotation_list.dup + curr.annotate_predecessor(pred) + queue << pred if cache != pred.annotation_list && !queue.include?(pred) end end end + + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> void + def merge_lookaheads(state, filtered_lookaheads) + return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? } + + state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 } + state.transitions.each do |transition| + next if transition.to_state.lookaheads_recomputed + compute_state(state, transition, transition.to_state) + end + end + + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void + def compute_state(state, transition, next_state) + propagating_lookaheads = state.propagate_lookaheads(next_state) + s = next_state.ielr_isocores.find {|st| st.is_compatible?(propagating_lookaheads) } + + if s.nil? + s = next_state.lalr_isocore + new_state = State.new(@states.count, s.accessing_symbol, s.kernels) + new_state.closure = s.closure + new_state.compute_transitions_and_reduces + s.transitions.each do |transition| + new_state.set_items_to_state(transition.to_items, transition.to_state) + end + @states << new_state + new_state.lalr_isocore = s + s.ielr_isocores << new_state + s.ielr_isocores.each do |st| + st.ielr_isocores = s.ielr_isocores + end + new_state.lookaheads_recomputed = true + new_state.item_lookahead_set = propagating_lookaheads + state.update_transition(transition, new_state) + elsif(!s.lookaheads_recomputed) + s.lookaheads_recomputed = true + s.item_lookahead_set = propagating_lookaheads + else + merge_lookaheads(s, propagating_lookaheads) + state.update_transition(transition, s) if state.items_to_state[transition.to_items].id != s.id + end + end + + # @rbs (Logger logger) -> void + def validate_conflicts_within_threshold!(logger) + exit false unless conflicts_within_threshold?(logger) + end + + # @rbs (Logger logger) -> bool + def conflicts_within_threshold?(logger) + return true unless @grammar.expect + + [sr_conflicts_within_threshold?(logger), rr_conflicts_within_threshold?(logger)].all? + end + + # @rbs (Logger logger) -> bool + def sr_conflicts_within_threshold?(logger) + return true if @grammar.expect == sr_conflicts_count + + logger.error("shift/reduce conflicts: #{sr_conflicts_count} found, #{@grammar.expect} expected") + false + end + + # @rbs (Logger logger) -> bool + def rr_conflicts_within_threshold?(logger, expected: 0) + return true if expected == rr_conflicts_count + + logger.error("reduce/reduce conflicts: #{rr_conflicts_count} found, #{expected} expected") + false + end + + # @rbs () -> void + def clear_look_ahead_sets + @direct_read_sets.clear + @reads_relation.clear + @read_sets.clear + @includes_relation.clear + @lookback_relation.clear + @follow_sets.clear + @la.clear + + @_direct_read_sets = nil + @_read_sets = nil + @_follow_sets = nil + @_la = nil + end end end diff --git a/tool/lrama/lib/lrama/states/item.rb b/tool/lrama/lib/lrama/states/item.rb deleted file mode 100644 index 823ccc72e1..0000000000 --- a/tool/lrama/lib/lrama/states/item.rb +++ /dev/null @@ -1,79 +0,0 @@ -# TODO: Validate position is not over rule rhs - -module Lrama - class States - class Item < Struct.new(:rule, :position, keyword_init: true) - # Optimization for States#setup_state - def hash - [rule.id, position].hash - end - - def rule_id - rule.id - end - - def empty_rule? - rule.empty_rule? - end - - def number_of_rest_symbols - rule.rhs.count - position - end - - def lhs - rule.lhs - end - - def next_sym - rule.rhs[position] - end - - def next_next_sym - rule.rhs[position + 1] - end - - def previous_sym - rule.rhs[position - 1] - end - - def end_of_rule? - rule.rhs.count == position - end - - def beginning_of_rule? - position == 0 - end - - def start_item? - rule.id == 0 && position == 0 - end - - def new_by_next_position - Item.new(rule: rule, position: position + 1) - end - - def symbols_before_dot - rule.rhs[0...position] - end - - def symbols_after_dot - rule.rhs[position..-1] - end - - def to_s - "#{lhs.id.s_value}: #{display_name}" - end - - def display_name - r = rule.rhs.map(&:display_name).insert(position, "•").join(" ") - "#{r} (rule #{rule.id})" - end - - # Right after position - def display_rest - r = rule.rhs[position..-1].map(&:display_name).join(" ") - ". #{r} (rule #{rule.id})" - end - end - end -end diff --git a/tool/lrama/lib/lrama/states_reporter.rb b/tool/lrama/lib/lrama/states_reporter.rb deleted file mode 100644 index 19830a63bb..0000000000 --- a/tool/lrama/lib/lrama/states_reporter.rb +++ /dev/null @@ -1,323 +0,0 @@ -module Lrama - class StatesReporter - include Lrama::Report::Duration - - def initialize(states) - @states = states - end - - def report(io, **options) - report_duration(:report) do - _report(io, **options) - end - end - - private - - def _report(io, grammar: false, states: false, itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false) - # TODO: Unused terms - # TODO: Unused rules - - report_conflicts(io) - report_grammar(io) if grammar - report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) - end - - def report_conflicts(io) - has_conflict = false - - @states.states.each do |state| - messages = [] - cs = state.conflicts.group_by(&:type) - if cs[:shift_reduce] - messages << "#{cs[:shift_reduce].count} shift/reduce" - end - - if cs[:reduce_reduce] - messages << "#{cs[:reduce_reduce].count} reduce/reduce" - end - - if !messages.empty? - has_conflict = true - io << "State #{state.id} conflicts: #{messages.join(', ')}\n" - end - end - - if has_conflict - io << "\n\n" - end - end - - def report_grammar(io) - io << "Grammar\n" - last_lhs = nil - - @states.rules.each do |rule| - if rule.rhs.empty? - r = "ε" - else - r = rule.rhs.map(&:display_name).join(" ") - end - - if rule.lhs == last_lhs - io << sprintf("%5d %s| %s\n", rule.id, " " * rule.lhs.display_name.length, r) - else - io << "\n" - io << sprintf("%5d %s: %s\n", rule.id, rule.lhs.display_name, r) - end - - last_lhs = rule.lhs - end - io << "\n\n" - end - - def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) - if counterexamples - cex = Counterexamples.new(@states) - end - - @states.states.each do |state| - # Report State - io << "State #{state.id}\n\n" - - # Report item - last_lhs = nil - list = itemsets ? state.items : state.kernels - list.sort_by {|i| [i.rule_id, i.position] }.each do |item| - rule = item.rule - position = item.position - if rule.rhs.empty? - r = "ε •" - else - r = rule.rhs.map(&:display_name).insert(position, "•").join(" ") - end - if rule.lhs == last_lhs - l = " " * rule.lhs.id.s_value.length + "|" - else - l = rule.lhs.id.s_value + ":" - end - la = "" - if lookaheads && item.end_of_rule? - reduce = state.find_reduce_by_item!(item) - look_ahead = reduce.selected_look_ahead - if !look_ahead.empty? - la = " [#{look_ahead.map(&:display_name).join(", ")}]" - end - end - last_lhs = rule.lhs - - io << sprintf("%5i %s %s%s\n", rule.id, l, r, la) - end - io << "\n" - - # Report shifts - tmp = state.term_transitions.select do |shift, _| - !shift.not_selected - end.map do |shift, next_state| - [shift.next_sym, next_state.id] - end - max_len = tmp.map(&:first).map(&:display_name).map(&:length).max - tmp.each do |term, state_id| - io << " #{term.display_name.ljust(max_len)} shift, and go to state #{state_id}\n" - end - io << "\n" if !tmp.empty? - - # Report error caused by %nonassoc - nl = false - tmp = state.resolved_conflicts.select do |resolved| - resolved.which == :error - end.map do |error| - error.symbol.display_name - end - max_len = tmp.map(&:length).max - tmp.each do |name| - nl = true - io << " #{name.ljust(max_len)} error (nonassociative)\n" - end - io << "\n" if !tmp.empty? - - # Report reduces - nl = false - max_len = state.non_default_reduces.flat_map(&:look_ahead).compact.map(&:display_name).map(&:length).max || 0 - max_len = [max_len, "$default".length].max if state.default_reduction_rule - ary = [] - - state.non_default_reduces.each do |reduce| - reduce.look_ahead.each do |term| - ary << [term, reduce] - end - end - - ary.sort_by do |term, reduce| - term.number - end.each do |term, reduce| - rule = reduce.item.rule - io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" - nl = true - end - - if (r = state.default_reduction_rule) - nl = true - s = "$default".ljust(max_len) - - if r.initial_rule? - io << " #{s} accept\n" - else - io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" - end - end - io << "\n" if nl - - # Report nonterminal transitions - tmp = [] - max_len = 0 - state.nterm_transitions.each do |shift, next_state| - nterm = shift.next_sym - tmp << [nterm, next_state.id] - max_len = [max_len, nterm.id.s_value.length].max - end - tmp.uniq! - tmp.sort_by! do |nterm, state_id| - nterm.number - end - tmp.each do |nterm, state_id| - io << " #{nterm.id.s_value.ljust(max_len)} go to state #{state_id}\n" - end - io << "\n" if !tmp.empty? - - if solved - # Report conflict resolutions - state.resolved_conflicts.each do |resolved| - io << " #{resolved.report_message}\n" - end - io << "\n" if !state.resolved_conflicts.empty? - end - - if counterexamples && state.has_conflicts? - # Report counterexamples - examples = cex.compute(state) - examples.each do |example| - label0 = example.type == :shift_reduce ? "shift/reduce" : "reduce/reduce" - label1 = example.type == :shift_reduce ? "Shift derivation" : "First Reduce derivation" - label2 = example.type == :shift_reduce ? "Reduce derivation" : "Second Reduce derivation" - - io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" - io << " #{example.path1_item}\n" - io << " #{example.path2_item}\n" - io << " #{label1}\n" - example.derivations1.render_strings_for_report.each do |str| - io << " #{str}\n" - end - io << " #{label2}\n" - example.derivations2.render_strings_for_report.each do |str| - io << " #{str}\n" - end - end - end - - if verbose - # Report direct_read_sets - io << " [Direct Read sets]\n" - direct_read_sets = @states.direct_read_sets - @states.nterms.each do |nterm| - terms = direct_read_sets[[state.id, nterm.token_id]] - next if !terms - next if terms.empty? - - str = terms.map {|sym| sym.id.s_value }.join(", ") - io << " read #{nterm.id.s_value} shift #{str}\n" - end - io << "\n" - - # Report reads_relation - io << " [Reads Relation]\n" - @states.nterms.each do |nterm| - a = @states.reads_relation[[state.id, nterm.token_id]] - next if !a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report read_sets - io << " [Read sets]\n" - read_sets = @states.read_sets - @states.nterms.each do |nterm| - terms = read_sets[[state.id, nterm.token_id]] - next if !terms - next if terms.empty? - - terms.each do |sym| - io << " #{sym.id.s_value}\n" - end - end - io << "\n" - - # Report includes_relation - io << " [Includes Relation]\n" - @states.nterms.each do |nterm| - a = @states.includes_relation[[state.id, nterm.token_id]] - next if !a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (State #{state.id}, #{nterm.id.s_value}) -> (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report lookback_relation - io << " [Lookback Relation]\n" - @states.rules.each do |rule| - a = @states.lookback_relation[[state.id, rule.id]] - next if !a - - a.each do |state_id2, nterm_id2| - n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (Rule: #{rule}) -> (State #{state_id2}, #{n.id.s_value})\n" - end - end - io << "\n" - - # Report follow_sets - io << " [Follow sets]\n" - follow_sets = @states.follow_sets - @states.nterms.each do |nterm| - terms = follow_sets[[state.id, nterm.token_id]] - - next if !terms - - terms.each do |sym| - io << " #{nterm.id.s_value} -> #{sym.id.s_value}\n" - end - end - io << "\n" - - # Report LA - io << " [Look-Ahead Sets]\n" - tmp = [] - max_len = 0 - @states.rules.each do |rule| - syms = @states.la[[state.id, rule.id]] - next if !syms - - tmp << [rule, syms] - max_len = ([max_len] + syms.map {|s| s.id.s_value.length }).max - end - tmp.each do |rule, syms| - syms.each do |sym| - io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" - end - end - io << "\n" if !tmp.empty? - end - - # End of Report State - io << "\n" - end - end - end -end diff --git a/tool/lrama/lib/lrama/tracer.rb b/tool/lrama/lib/lrama/tracer.rb new file mode 100644 index 0000000000..fda699a665 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer.rb @@ -0,0 +1,51 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "tracer/actions" +require_relative "tracer/closure" +require_relative "tracer/duration" +require_relative "tracer/only_explicit_rules" +require_relative "tracer/rules" +require_relative "tracer/state" + +module Lrama + class Tracer + # @rbs (IO io, **bool options) -> void + def initialize(io, **options) + @io = io + @options = options + @only_explicit_rules = OnlyExplicitRules.new(io, **options) + @rules = Rules.new(io, **options) + @actions = Actions.new(io, **options) + @closure = Closure.new(io, **options) + @state = State.new(io, **options) + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + @only_explicit_rules.trace(grammar) + @rules.trace(grammar) + @actions.trace(grammar) + end + + # @rbs (Lrama::State state) -> void + def trace_closure(state) + @closure.trace(state) + end + + # @rbs (Lrama::State state) -> void + def trace_state(state) + @state.trace(state) + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_state_list_append(state_count, state) + @state.trace_list_append(state_count, state) + end + + # @rbs () -> void + def enable_duration + Duration.enable if @options[:time] + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/actions.rb b/tool/lrama/lib/lrama/tracer/actions.rb new file mode 100644 index 0000000000..7b9c9b9f53 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/actions.rb @@ -0,0 +1,22 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Actions + # @rbs (IO io, ?actions: bool, **bool options) -> void + def initialize(io, actions: false, **options) + @io = io + @actions = actions + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @actions + + @io << "Grammar rules with actions:" << "\n" + grammar.rules.each { |rule| @io << rule.with_actions << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/closure.rb b/tool/lrama/lib/lrama/tracer/closure.rb new file mode 100644 index 0000000000..5b2f0b27e6 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/closure.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Closure + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @closure = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @closure + + @io << "Closure: input" << "\n" + state.kernels.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + @io << "Closure: output" << "\n" + state.items.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/duration.rb b/tool/lrama/lib/lrama/tracer/duration.rb new file mode 100644 index 0000000000..91c49625b2 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/duration.rb @@ -0,0 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + module Duration + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @_report_duration_enabled: bool + + # @rbs () -> void + def self.enable + @_report_duration_enabled = true + end + + # @rbs () -> bool + def self.enabled? + !!@_report_duration_enabled + end + + # @rbs [T] (_ToS message) { -> T } -> T + def report_duration(message) + time1 = Time.now.to_f + result = yield + time2 = Time.now.to_f + + if Duration.enabled? + STDERR.puts sprintf("%s %10.5f s", message, time2 - time1) + end + + return result + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb new file mode 100644 index 0000000000..4f64e7d2f4 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class OnlyExplicitRules + # @rbs (IO io, ?only_explicit: bool, **bool) -> void + def initialize(io, only_explicit: false, **_) + @io = io + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each do |rule| + @io << rule.display_name_without_action << "\n" if rule.lhs.first_set.any? + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/rules.rb b/tool/lrama/lib/lrama/tracer/rules.rb new file mode 100644 index 0000000000..d6e85b8432 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Rules + # @rbs (IO io, ?rules: bool, ?only_explicit: bool, **bool) -> void + def initialize(io, rules: false, only_explicit: false, **_) + @io = io + @rules = rules + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return if !@rules || @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each { |rule| @io << rule.display_name << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/state.rb b/tool/lrama/lib/lrama/tracer/state.rb new file mode 100644 index 0000000000..21c0047f8e --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/state.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class State + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @state = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @state + + # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but + # I think it is not correct... + previous = state.kernels.first.previous_sym + @io << "Processing state #{state.id} (reached by #{previous.display_name})" << "\n" + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_list_append(state_count, state) + return unless @state + + previous = state.kernels.first.previous_sym + @io << sprintf("state_list_append (state = %d, symbol = %d (%s))", + state_count, previous.number, previous.display_name) << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb index 406939918c..d649b74939 100644 --- a/tool/lrama/lib/lrama/version.rb +++ b/tool/lrama/lib/lrama/version.rb @@ -1,3 +1,6 @@ +# rbs_inline: enabled +# frozen_string_literal: true + module Lrama - VERSION = "0.6.1".freeze + VERSION = "0.7.1".freeze #: String end diff --git a/tool/lrama/lib/lrama/warning.rb b/tool/lrama/lib/lrama/warning.rb deleted file mode 100644 index 3c99791ebf..0000000000 --- a/tool/lrama/lib/lrama/warning.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Lrama - class Warning - attr_reader :errors, :warns - - def initialize(out = STDERR) - @out = out - @errors = [] - @warns = [] - end - - def error(message) - @out << message << "\n" - @errors << message - end - - def warn(message) - @out << message << "\n" - @warns << message - end - - def has_error? - !@errors.empty? - end - end -end diff --git a/tool/lrama/lib/lrama/warnings.rb b/tool/lrama/lib/lrama/warnings.rb new file mode 100644 index 0000000000..52f09144ef --- /dev/null +++ b/tool/lrama/lib/lrama/warnings.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'warnings/conflicts' +require_relative 'warnings/implicit_empty' +require_relative 'warnings/name_conflicts' +require_relative 'warnings/redefined_rules' +require_relative 'warnings/required' +require_relative 'warnings/useless_precedence' + +module Lrama + class Warnings + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @conflicts = Conflicts.new(logger, warnings) + @implicit_empty = ImplicitEmpty.new(logger, warnings) + @name_conflicts = NameConflicts.new(logger, warnings) + @redefined_rules = RedefinedRules.new(logger, warnings) + @required = Required.new(logger, warnings) + @useless_precedence = UselessPrecedence.new(logger, warnings) + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + @conflicts.warn(states) + @implicit_empty.warn(grammar) + @name_conflicts.warn(grammar) + @redefined_rules.warn(grammar) + @required.warn(grammar) + @useless_precedence.warn(grammar, states) + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/conflicts.rb b/tool/lrama/lib/lrama/warnings/conflicts.rb new file mode 100644 index 0000000000..6ba0de6f9c --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/conflicts.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Conflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::States states) -> void + def warn(states) + return unless @warnings + + if states.sr_conflicts_count != 0 + @logger.warn("shift/reduce conflicts: #{states.sr_conflicts_count} found") + end + + if states.rr_conflicts_count != 0 + @logger.warn("reduce/reduce conflicts: #{states.rr_conflicts_count} found") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/implicit_empty.rb b/tool/lrama/lib/lrama/warnings/implicit_empty.rb new file mode 100644 index 0000000000..ba81adca01 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/implicit_empty.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Empty rules are easily overlooked and ambiguous + # - Empty alternatives like `rule: | "token";` can be missed during code reading + # - Difficult to distinguish between intentional empty rules vs. omissions + # - Explicit marking with %empty directive comment improves clarity + class ImplicitEmpty + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.rule_builders.each do |builder| + if builder.rhs.empty? + @logger.warn("warning: empty rule without %empty") + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/name_conflicts.rb b/tool/lrama/lib/lrama/warnings/name_conflicts.rb new file mode 100644 index 0000000000..c0754ab551 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/name_conflicts.rb @@ -0,0 +1,63 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Parameterized rule names conflicting with symbol names + # - When a %rule name is identical to a terminal or non-terminal symbol name, + # it reduces grammar readability and may cause unintended behavior + # - Detecting these conflicts helps improve grammar definition quality + class NameConflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return if grammar.parameterized_rules.empty? + + symbol_names = collect_symbol_names(grammar) + check_conflicts(grammar.parameterized_rules, symbol_names) + end + + private + + # @rbs (Lrama::Grammar grammar) -> Set[String] + def collect_symbol_names(grammar) + symbol_names = Set.new + + collect_term_names(grammar.terms, symbol_names) + collect_nterm_names(grammar.nterms, symbol_names) + + symbol_names + end + + # @rbs (Array[untyped] terms, Set[String] symbol_names) -> void + def collect_term_names(terms, symbol_names) + terms.each do |term| + symbol_names.add(term.id.s_value) + symbol_names.add(term.alias_name) if term.alias_name + end + end + + # @rbs (Array[untyped] nterms, Set[String] symbol_names) -> void + def collect_nterm_names(nterms, symbol_names) + nterms.each do |nterm| + symbol_names.add(nterm.id.s_value) + end + end + + # @rbs (Array[untyped] parameterized_rules, Set[String] symbol_names) -> void + def check_conflicts(parameterized_rules, symbol_names) + parameterized_rules.each do |param_rule| + next unless symbol_names.include?(param_rule.name) + + @logger.warn("warning: parameterized rule name \"#{param_rule.name}\" conflicts with symbol name") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/redefined_rules.rb b/tool/lrama/lib/lrama/warnings/redefined_rules.rb new file mode 100644 index 0000000000..8ac2f1f103 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/redefined_rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class RedefinedRules + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.parameterized_resolver.redefined_rules.each do |rule| + @logger.warn("parameterized rule redefined: #{rule}") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/required.rb b/tool/lrama/lib/lrama/warnings/required.rb new file mode 100644 index 0000000000..4ab1ed787e --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/required.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Required + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings = false, **_) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + if grammar.required + @logger.warn("currently, %require is simply valid as a grammar but does nothing") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/useless_precedence.rb b/tool/lrama/lib/lrama/warnings/useless_precedence.rb new file mode 100644 index 0000000000..2913d6d7e5 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/useless_precedence.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class UselessPrecedence + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + return unless @warnings + + grammar.precedences.each do |precedence| + unless precedence.used_by? + @logger.warn("Precedence #{precedence.s_value} (line: #{precedence.lineno}) is defined but not used in any rule.") + end + end + end + end + end +end diff --git a/tool/lrama/template/bison/_yacc.h b/tool/lrama/template/bison/_yacc.h index 34ed6d81f5..3e270c9171 100644 --- a/tool/lrama/template/bison/_yacc.h +++ b/tool/lrama/template/bison/_yacc.h @@ -28,6 +28,7 @@ extern int yydebug; <%-# b4_declare_yylstype -%> <%-# b4_value_type_define -%> /* Value type. */ +<% if output.grammar.union %> #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED union YYSTYPE { @@ -40,6 +41,13 @@ typedef union YYSTYPE YYSTYPE; # define YYSTYPE_IS_TRIVIAL 1 # define YYSTYPE_IS_DECLARED 1 #endif +<% else %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef int YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% end %> <%-# b4_location_type_define -%> /* Location type. */ diff --git a/tool/lrama/template/bison/yacc.c b/tool/lrama/template/bison/yacc.c index f72d346178..6edd59a0d5 100644 --- a/tool/lrama/template/bison/yacc.c +++ b/tool/lrama/template/bison/yacc.c @@ -1145,7 +1145,12 @@ yydestruct (const char *yymsg, YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp<%= output.user_args %>); YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN - YY_USE (yykind); + switch (yykind) + { +<%= output.symbol_actions_for_destructor -%> + default: + break; + } YY_IGNORE_MAYBE_UNINITIALIZED_END } @@ -1161,9 +1166,9 @@ yydestruct (const char *yymsg, #endif enum yy_repair_type { - insert, - delete, - shift, + inserting, + deleting, + shifting, }; struct yy_repair { @@ -1396,27 +1401,27 @@ yyrecover(yy_state_t *yyss, yy_state_t *yyssp, int yychar<%= output.user_formals if (current->repair_length + 1 > YYMAXREPAIR(<%= output.parse_param_name %>)) continue; - yy_repairs *new = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); - new->id = count; - new->next = 0; - new->stack_length = stack_length; - new->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); - new->state = new->states + (current->state - current->states); - YYCOPY (new->states, current->states, current->state - current->states + 1); - new->repair_length = current->repair_length + 1; - new->prev_repair = current; - new->repair.type = insert; - new->repair.term = (yysymbol_kind_t) yyx; + yy_repairs *reps = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + reps->id = count; + reps->next = 0; + reps->stack_length = stack_length; + reps->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + reps->state = reps->states + (current->state - current->states); + YYCOPY (reps->states, current->states, current->state - current->states + 1); + reps->repair_length = current->repair_length + 1; + reps->prev_repair = current; + reps->repair.type = inserting; + reps->repair.term = (yysymbol_kind_t) yyx; /* Process PDA assuming next token is yyx */ - if (! yy_process_repairs (new, yyx)) + if (! yy_process_repairs (reps, (yysymbol_kind_t)yyx)) { - YYFREE (new); + YYFREE (reps); continue; } - tail->next = new; - tail = new; + tail->next = reps; + tail = reps; count++; if (yyx == yytoken) @@ -1432,7 +1437,7 @@ yyrecover(yy_state_t *yyss, yy_state_t *yyssp, int yychar<%= output.user_formals YYDPRINTF ((stderr, "New repairs is enqueued. count: %d, yystate: %d, yyx: %d\n", count, yystate, yyx)); - yy_print_repairs (new<%= output.user_args %>); + yy_print_repairs (reps<%= output.user_args %>); } } } @@ -1470,7 +1475,12 @@ int yychar; /* The semantic value of the lookahead symbol. */ /* Default value used for initialization, for pacifying older GCCs or non-GCC compilers. */ +#ifdef __cplusplus +static const YYSTYPE yyval_default = {}; +(void) yyval_default; +#else YY_INITIAL_VALUE (static const YYSTYPE yyval_default;) +#endif YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); /* Location data for the lookahead symbol. */ @@ -1752,6 +1762,7 @@ yybackup: *++yyvsp = yylval; YY_IGNORE_MAYBE_UNINITIALIZED_END *++yylsp = yylloc; +<%= output.after_shift_function("/* %after-shift code. */") %> /* Discard the shifted token. */ yychar = YYEMPTY; @@ -1784,6 +1795,7 @@ yyreduce: unconditionally makes the parser a bit smaller, and it avoids a GCC warning that YYVAL may be used uninitialized. */ yyval = yyvsp[1-yylen]; +<%= output.before_reduce_function("/* %before-reduce function. */") %> /* Default location. */ YYLLOC_DEFAULT (yyloc, (yylsp - yylen), yylen); @@ -1809,6 +1821,7 @@ yyreduce: YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc<%= output.user_args %>); YYPOPSTACK (yylen); +<%= output.after_reduce_function("/* %after-reduce function. */") %> yylen = 0; *++yyvsp = yyval; @@ -1910,6 +1923,7 @@ yyerrorlab: /* Do not reclaim the symbols of the rule whose action triggered this YYERROR. */ YYPOPSTACK (yylen); +<%= output.after_pop_stack_function("yylen", "/* %after-pop-stack function. */") %> yylen = 0; YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); yystate = *yyssp; @@ -1969,6 +1983,7 @@ yyerrlab1: yydestruct ("Error: popping", YY_ACCESSING_SYMBOL (yystate), yyvsp, yylsp<%= output.user_args %>); YYPOPSTACK (1); +<%= output.after_pop_stack_function(1, "/* %after-pop-stack function. */") %> yystate = *yyssp; YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); } @@ -1983,6 +1998,7 @@ yyerrlab1: /* Shift the error token. */ YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp<%= output.user_args %>); +<%= output.after_shift_error_token_function("/* %after-shift-error-token code. */") %> yystate = yyn; goto yynewstate; diff --git a/tool/lrama/template/diagram/diagram.html b/tool/lrama/template/diagram/diagram.html new file mode 100644 index 0000000000..3e87e6e519 --- /dev/null +++ b/tool/lrama/template/diagram/diagram.html @@ -0,0 +1,102 @@ +<!DOCTYPE html> +<html> +<head> + <title>Lrama syntax diagrams</title> + + <style> + <%= output.default_style %> + .diagram-header { + display: inline-block; + font-weight: bold; + font-size: 18px; + margin-bottom: -8px; + text-align: center; + } + + svg { + width: 100%; + } + + svg.railroad-diagram g.non-terminal text { + cursor: pointer; + } + + h2.hover-header { + background-color: #90ee90; + } + + svg.railroad-diagram g.non-terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + + svg.railroad-diagram g.terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + </style> +</head> + +<body align="center"> + <%= output.diagrams %> + <script> + document.addEventListener("DOMContentLoaded", () => { + function addHoverEffect(selector, hoverClass, relatedSelector, relatedHoverClass, getTextElements) { + document.querySelectorAll(selector).forEach(element => { + element.addEventListener("mouseenter", () => { + element.classList.add(hoverClass); + getTextElements(element).forEach(textEl => { + if (!relatedSelector) return; + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.add(relatedHoverClass); + }); + }); + }); + + element.addEventListener("mouseleave", () => { + element.classList.remove(hoverClass); + if (!relatedSelector) return; + getTextElements(element).forEach(textEl => { + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.remove(relatedHoverClass); + }); + }); + }); + }); + } + + function getElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => el.textContent.trim() === text.trim()); + } + + function getParentElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => + [...el.querySelectorAll("text")].some(textEl => textEl.textContent.trim() === text.trim()) + ); + } + + function scrollToMatchingHeader() { + document.querySelectorAll("g.non-terminal").forEach(element => { + element.addEventListener("click", () => { + const textElements = [...element.querySelectorAll("text")]; + for (const textEl of textElements) { + const targetHeader = getElementsByText("h2", textEl.textContent)[0]; + if (targetHeader) { + targetHeader.scrollIntoView({ behavior: "smooth", block: "start" }); + break; + } + } + }); + }); + } + + addHoverEffect("h2", "hover-header", "g.non-terminal", "hover-g", element => [element]); + addHoverEffect("g.non-terminal", "hover-g", "h2", "hover-header", + element => [...element.querySelectorAll("text")] + ); + addHoverEffect("g.terminal", "hover-g", "", "", element => [element]); + scrollToMatchingHeader(); + }); + </script> +</body> +</html> diff --git a/tool/m4/ruby_append_option.m4 b/tool/m4/ruby_append_option.m4 index ff828d2162..8cd2741ae8 100644 --- a/tool/m4/ruby_append_option.m4 +++ b/tool/m4/ruby_append_option.m4 @@ -3,3 +3,7 @@ AC_DEFUN([RUBY_APPEND_OPTION], [# RUBY_APPEND_OPTION($1) AS_CASE([" [$]{$1-} "], [*" $2 "*], [], [' '], [ $1="$2"], [ $1="[$]$1 $2"])])dnl +AC_DEFUN([RUBY_PREPEND_OPTION], + [# RUBY_PREPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_check_builtin_overflow.m4 b/tool/m4/ruby_check_builtin_overflow.m4 new file mode 100644 index 0000000000..8568d2c6d9 --- /dev/null +++ b/tool/m4/ruby_check_builtin_overflow.m4 @@ -0,0 +1,28 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_OVERFLOW], [dnl +{ # $0($1) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow, [int x;__builtin_[$1]_overflow(0,0,&x)]) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow_p, [__builtin_[$1]_overflow_p(0,0,(int)0)]) + + AS_IF([test "$rb_cv_builtin___builtin_[$1]_overflow" != no], [ + AC_CACHE_CHECK(for __builtin_[$1]_overflow with long long arguments, rb_cv_use___builtin_[$1]_overflow_long_long, [ + AC_LINK_IFELSE([AC_LANG_SOURCE([[ +@%:@pragma clang optimize off + +int +main(void) +{ + long long x = 0, y; + __builtin_$1_overflow(x, x, &y); + + return 0; +} +]])], + rb_cv_use___builtin_[$1]_overflow_long_long=yes, + rb_cv_use___builtin_[$1]_overflow_long_long=no)]) + ]) + AS_IF([test "$rb_cv_use___builtin_[$1]_overflow_long_long" = yes], [ + AC_DEFINE(USE___BUILTIN_[]AS_TR_CPP($1)_OVERFLOW_LONG_LONG, 1) + ]) +} +])dnl diff --git a/tool/m4/ruby_check_builtin_setjmp.m4 b/tool/m4/ruby_check_builtin_setjmp.m4 index 05118e2243..1e5d9b3028 100644 --- a/tool/m4/ruby_check_builtin_setjmp.m4 +++ b/tool/m4/ruby_check_builtin_setjmp.m4 @@ -20,7 +20,7 @@ AC_CACHE_CHECK(for __builtin_setjmp, ac_cv_func___builtin_setjmp, void (*volatile f)(void) = t; if (!jump()) printf("%d\n", f != 0); ]])], - [ac_cv_func___builtin_setjmp="yes with cast ($cast)"]) + [ac_cv_func___builtin_setjmp="yes${cast:+ with cast ($cast)}"]) ]) test "$ac_cv_func___builtin_setjmp" = no || break done]) diff --git a/tool/m4/ruby_check_header.m4 b/tool/m4/ruby_check_header.m4 new file mode 100644 index 0000000000..6fec9d16c5 --- /dev/null +++ b/tool/m4/ruby_check_header.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_HEADER], + [# RUBY_CHECK_HEADER($@) + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS m4_if([$5], [], [$INCFLAGS], [$5])" + AC_CHECK_HEADERS([$1], [$2], [$3], [$4]) + CPPFLAGS="$save_CPPFLAGS" + unset save_CPPFLAGS]) diff --git a/tool/m4/ruby_defint.m4 b/tool/m4/ruby_defint.m4 index e9ed68e5b8..7f262a73fc 100644 --- a/tool/m4/ruby_defint.m4 +++ b/tool/m4/ruby_defint.m4 @@ -17,7 +17,8 @@ typedef $1 t; int s = sizeof(t) == 42;])], ["${ac_cv_sizeof___int128@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int128"], [ rb_cv_type_$1=no])])]) AS_IF([test "${rb_cv_type_$1}" != no], [ - type="${rb_cv_type_$1@%:@@%:@unsigned }" + type="${rb_cv_type_$1@%:@@%:@*signed }" + AS_IF([test "$type" = "long long"], [type=long_long]) AS_IF([test "$type" != yes && eval 'test -n "${ac_cv_sizeof_'$type'+set}"'], [ eval cond='"${ac_cv_sizeof_'$type'}"' AS_CASE([$cond], [*:*], [ diff --git a/tool/m4/ruby_modular_gc.m4 b/tool/m4/ruby_modular_gc.m4 new file mode 100644 index 0000000000..661fce2e60 --- /dev/null +++ b/tool/m4/ruby_modular_gc.m4 @@ -0,0 +1,41 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_MODULAR_GC],[ +AC_ARG_WITH(modular-gc, + AS_HELP_STRING([--with-modular-gc=DIR], + [Enable replacement of Ruby's GC from a modular library in the specified directory.]), + [modular_gc_dir=$withval], [unset modular_gc_dir] +) + +AS_IF([test "$modular_gc_dir" = yes], [ + AC_MSG_ERROR(you must specify a directory when using --with-modular-gc) +]) + +AC_MSG_CHECKING([if building with modular GC support]) +AS_IF([test x"$modular_gc_dir" != x], [ + AC_MSG_RESULT([yes]) + + # Ensure that modular_gc_dir is always an absolute path so that Ruby + # never loads a modular GC from a relative path + AS_CASE(["$modular_gc_dir"], + [/*], [], + [test "$load_relative" = yes || modular_gc_dir="$prefix/$modular_gc_dir"] + ) + + # Ensure that modular_gc_dir always terminates with a / + AS_CASE(["$modular_gc_dir"], + [*/], [], + [modular_gc_dir="$modular_gc_dir/"] + ) + + AC_DEFINE([USE_MODULAR_GC], [1]) + + modular_gc_summary="yes (in $modular_gc_dir)" +], [ + AC_MSG_RESULT([no]) + AC_DEFINE([USE_MODULAR_GC], [0]) + + modular_gc_summary="no" +]) + +AC_SUBST(modular_gc_dir, "${modular_gc_dir}") +])dnl diff --git a/tool/m4/ruby_setjmp_type.m4 b/tool/m4/ruby_setjmp_type.m4 index 4ae26fe5cd..d26af34ea0 100644 --- a/tool/m4/ruby_setjmp_type.m4 +++ b/tool/m4/ruby_setjmp_type.m4 @@ -3,18 +3,15 @@ AC_DEFUN([RUBY_SETJMP_TYPE], [ RUBY_CHECK_BUILTIN_SETJMP RUBY_CHECK_SETJMP(_setjmpex, [], [@%:@include <setjmpex.h>]) RUBY_CHECK_SETJMP(_setjmp) -RUBY_CHECK_SETJMP(sigsetjmp, [sigjmp_buf]) AC_MSG_CHECKING(for setjmp type) setjmp_suffix= -unset setjmp_sigmask AC_ARG_WITH(setjmp-type, - AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), + AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), [ AS_CASE([$withval], [__builtin_setjmp], [setjmp=__builtin_setjmp], [_setjmp], [ setjmp_prefix=_], - [sigsetjmp,*], [ setjmp_prefix=sig setjmp_sigmask=`expr "$withval" : 'sigsetjmp\(,.*\)'`], - [sigsetjmp], [ setjmp_prefix=sig], + [sigsetjmp*], [ AC_MSG_WARN(No longer use sigsetjmp; use setjmp instead); setjmp_prefix=], [setjmp], [ setjmp_prefix=], [setjmpex], [ setjmp_prefix= setjmp_suffix=ex], [''], [ unset setjmp_prefix], @@ -34,19 +31,13 @@ AS_IF([test ${setjmp_prefix+set}], [ ], [test "$ac_cv_func__setjmp" = yes], [ setjmp_prefix=_ setjmp_suffix= -], [test "$ac_cv_func_sigsetjmp" = yes], [ - AS_CASE([$target_os],[solaris*|cygwin*],[setjmp_prefix=],[setjmp_prefix=sig]) - setjmp_suffix= ], [ setjmp_prefix= setjmp_suffix= ]) -AS_IF([test x$setjmp_prefix:$setjmp_sigmask = xsig:], [ - setjmp_sigmask=,0 -]) -AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}${setjmp_sigmask}) -AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env)${setjmp_sigmask})]) +AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}) +AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env))]) AC_DEFINE_UNQUOTED([RUBY_LONGJMP(env,val)], [${setjmp_prefix}longjmp($setjmp_cast(env),val)]) -AS_IF([test "(" "$GCC" != yes ")" -o x$setjmp_prefix != x__builtin_], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, ${setjmp_sigmask+${setjmp_prefix}}jmp_buf)) +AS_CASE(["$GCC:$setjmp_prefix"], [yes:__builtin_], [], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, jmp_buf)) AS_IF([test x$setjmp_suffix = xex], [AC_DEFINE_UNQUOTED(RUBY_USE_SETJMPEX, 1)]) ])dnl diff --git a/tool/m4/ruby_try_cflags.m4 b/tool/m4/ruby_try_cflags.m4 index b74718fe5e..b397642aad 100644 --- a/tool/m4/ruby_try_cflags.m4 +++ b/tool/m4/ruby_try_cflags.m4 @@ -22,3 +22,20 @@ AC_DEFUN([RUBY_TRY_CFLAGS], [ AC_MSG_RESULT(no)], [$4], [$5]) ])dnl + +AC_DEFUN([_RUBY_TRY_CFLAGS_PREPEND], [ + RUBY_WERROR_FLAG([ + CFLAGS="$1 [$]CFLAGS" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS_PREPEND], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS_PREPEND([$1], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)], + [$4], [$5]) +])dnl diff --git a/tool/m4/ruby_wasm_tools.m4 b/tool/m4/ruby_wasm_tools.m4 index a6d8c34ebc..efc017e771 100644 --- a/tool/m4/ruby_wasm_tools.m4 +++ b/tool/m4/ruby_wasm_tools.m4 @@ -9,7 +9,7 @@ AC_DEFUN([RUBY_WASM_TOOLS], AC_SUBST(wasmoptflags) : ${wasmoptflags=-O3} - AC_MSG_CHECKING([wheather \$WASI_SDK_PATH is set]) + AC_MSG_CHECKING([whether \$WASI_SDK_PATH is set]) AS_IF([test x"${WASI_SDK_PATH}" = x], [ AC_MSG_RESULT([no]) AC_MSG_ERROR([WASI_SDK_PATH environment variable is required]) @@ -19,6 +19,7 @@ AC_DEFUN([RUBY_WASM_TOOLS], LD="${LD:-${WASI_SDK_PATH}/bin/clang}" AR="${AR:-${WASI_SDK_PATH}/bin/llvm-ar}" RANLIB="${RANLIB:-${WASI_SDK_PATH}/bin/llvm-ranlib}" + OBJCOPY="${OBJCOPY:-${WASI_SDK_PATH}/bin/llvm-objcopy}" ]) ]) ])dnl diff --git a/tool/make-snapshot b/tool/make-snapshot index 7446f18578..4af6a855eb 100755 --- a/tool/make-snapshot +++ b/tool/make-snapshot @@ -10,6 +10,7 @@ require 'fileutils' require 'shellwords' require 'tmpdir' require 'pathname' +require 'date' require 'yaml' require 'json' require File.expand_path("../lib/vcs", __FILE__) @@ -56,7 +57,7 @@ PACKAGES = { DEFAULT_PACKAGES = PACKAGES.keys - ["tar"] if !$no7z and system("7z", out: IO::NULL) PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" - PACKAGES["zip"] = %w".zip 7z a -tzip -l -mx -mtc=off" << {out: IO::NULL} + PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL} elsif gzip = ENV.delete("GZIP") PACKAGES["gzip"].concat(gzip.shellsplit) end @@ -71,16 +72,6 @@ ENV["LC_ALL"] = ENV["LANG"] = "C" GITURL = URI.parse("https://github.com/ruby/ruby.git") RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ -ENV["VPATH"] ||= "include/ruby" -YACC = ENV["YACC"] ||= "#{$tooldir}/lrama/exe/lrama" -ENV["BASERUBY"] ||= "ruby" -ENV["RUBY"] ||= "ruby" -ENV["MV"] ||= "mv" -ENV["RM"] ||= "rm -f" -ENV["MINIRUBY"] ||= "ruby" -ENV["PROGRAM"] ||= "ruby" -ENV["AUTOCONF"] ||= "autoconf" -ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" ENV["TZ"] = "UTC" class String @@ -117,22 +108,32 @@ $digests &&= $digests.split(/[, ]+/).tap {|dig| $digests ||= DIGESTS $patch_file &&= File.expand_path($patch_file) -path = ENV["PATH"].split(File::PATH_SEPARATOR) -%w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| - cmd, = ENV[var].shellsplit - unless path.any? {|dir| +PATH = ENV["PATH"].split(File::PATH_SEPARATOR) +def PATH.executable_env(var, command = nil) + command = ENV[var] ||= (command or return) + cmd, = command.shellsplit + unless any? {|dir| file = File.expand_path(cmd, dir) File.file?(file) and File.executable?(file) } abort "#{File.basename $0}: #{var} command not found - #{cmd}" end + command end +PATH.executable_env("MV", "mv") +PATH.executable_env("RM", "rm -f") +PATH.executable_env("AUTOCONF", "autoconf") + %w[BASERUBY RUBY MINIRUBY].each do |var| - %x[#{ENV[var]} --disable-gem -e1 2>&1] - if $?.success? - ENV[var] += ' --disable-gem' + cmd = PATH.executable_env(var, "ruby") + help = IO.popen("#{cmd} --help", err: %i[child out], &:read) + unless $?.success? and /ruby/ =~ help + abort "#{File.basename $0}: #{var} ruby not found - #{cmd}" end + IO.popen("#{cmd} --disable-gem -eexit", err: %i[child out], &:read) + cmd += ' --disable-gem' if $?.success? + ENV[var] = cmd end if defined?($help) or defined?($_help) @@ -252,7 +253,6 @@ end def package(vcs, rev, destdir, tmp = nil) pwd = Dir.pwd - patchlevel = false prerelease = false if rev and revision = rev[/@(\h+)\z/, 1] rev = $` @@ -269,22 +269,23 @@ def package(vcs, rev, destdir, tmp = nil) when /\Astable\z/ vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} url &&= vcs.branch(url) - when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + when /\A(\d+)\.(\d+)\.(\d+)-(preview|rc)(\d+)/ prerelease = true tag = "#{$4}#{$5}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") - when /\A(.*)\.(.*)\.(.*)-p(\d+)/ - patchlevel = true - tag = "p#{$4}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") - when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ - if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") - patchlevel = true - tag = "" - url = vcs.tag("v#{$1}_#{$2}_#{$3}") + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") else - url = vcs.branch("ruby_#{rev.tr('.', '_')}") + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") + end + when /\A(\d+)\.(\d+)\.(\d+)\z/ + tag = "" + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") + else + url = vcs.tag("v#{$1}_#{$2}_#{$3}") end + when /\A(\d+)\.(\d+)\z/ + url = vcs.branch("ruby_#{rev.tr('.', '_')}") else warn "#{$0}: unknown version - #{rev}" return @@ -334,7 +335,7 @@ def package(vcs, rev, destdir, tmp = nil) FileUtils.rm(file, verbose: $VERBOSE) end - status = IO.read(File.dirname(__FILE__) + "/prereq.status") + status = File.read(File.dirname(__FILE__) + "/prereq.status") Dir.chdir(tmp) if tmp if !File.directory?(v) @@ -346,26 +347,20 @@ def package(vcs, rev, destdir, tmp = nil) File.open("#{v}/revision.h", "wb") {|f| f.puts vcs.revision_header(revision, modified) } - version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version ||= (versionhdr = File.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] version ||= begin - include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h") + include_ruby_versionhdr = File.read("#{v}/include/ruby/version.h") api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] [api_major_version, api_minor_version, version_teeny].join('.') end version or return - if patchlevel - unless tag.empty? - versionhdr ||= IO.read("#{v}/version.h") - patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] - tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision)) - end - elsif prerelease - versionhdr ||= IO.read("#{v}/version.h") + if prerelease + versionhdr ||= File.read("#{v}/version.h") versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace" - IO.write("#{v}/version.h", versionhdr) + File.write("#{v}/version.h", versionhdr) else tag ||= vcs.revision_name(revision) end @@ -430,7 +425,7 @@ def package(vcs, rev, destdir, tmp = nil) puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE unless File.exist?("configure") print "creating configure..." - unless system([ENV["AUTOCONF"]]*2) + unless system(File.exist?(gen = "./autogen.sh") ? gen : [ENV["AUTOCONF"]]*2) puts $colorize.fail(" failed") return end @@ -439,11 +434,11 @@ def package(vcs, rev, destdir, tmp = nil) clean.add("autom4te.cache") clean.add("enc/unicode/data") print "creating prerequisites..." - if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + if File.file?("common.mk") && /^prereq/ =~ commonmk = File.read("common.mk") puts extout = clean.add('tmp') begin - status = IO.read("tool/prereq.status") + status = File.read("tool/prereq.status") rescue Errno::ENOENT # use fallback file end @@ -456,7 +451,7 @@ def package(vcs, rev, destdir, tmp = nil) File.binwrite("#{defaults}/ruby.rb", "") miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] - mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). + mk = (File.read("template/Makefile.in") rescue File.read("Makefile.in")). gsub(/^@.*\n/, '') vars = { "EXTOUT"=>extout, @@ -469,6 +464,9 @@ def package(vcs, rev, destdir, tmp = nil) "MAJOR"=>api_major_version, "MINOR"=>api_minor_version, "TEENY"=>version_teeny, + "VPATH"=>(ENV["VPATH"] || "include/ruby"), + "PROGRAM"=>(ENV["PROGRAM"] || "ruby"), + "BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"), } status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do vars[$2] ||= $3 @@ -477,6 +475,13 @@ def package(vcs, rev, destdir, tmp = nil) vars["UNICODE_VERSION"] = $unicode_version if $unicode_version args = vars.dup mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} + commonmk.gsub!(/^!(?:include \$\(srcdir\)\/(.*))?/) do + if inc = $1 and File.exist?(inc) + File.binread(inc).gsub(/^!/, '# !') + else + "#" + end + end mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') mk << <<-'APPEND' @@ -514,7 +519,7 @@ touch-unicode-files: end print "prerequisites" else - system(*%W"#{YACC} -o parse.c parse.y") + system(*%W[#{PATH.executable_env("YACC", "bison")} -o parse.c parse.y]) end vcs.after_export(".") if exported clean.concat(Dir.glob("ext/**/autom4te.cache")) @@ -631,7 +636,7 @@ revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| key = basename[/\A(.*)\.(?:tar|zip)/, 1] info[key] ||= Hash.new{|h,k|h[k]={}} info[key]['version'] = version if version - info[key]['date'] = release_date.strftime('%Y-%m-%d') + info[key]['date'] = release_date.to_date if version info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" diff --git a/tool/merger.rb b/tool/merger.rb index d181a77f84..4c096087fc 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -2,9 +2,8 @@ # -*- ruby -*- exec "${RUBY-ruby}" "-x" "$0" "$@" && [ ] if false #!ruby -# This needs ruby 2.0, Subversion and Git. -# As a Ruby committer, run this in an SVN repository -# to commit a change. +# This needs ruby 2.0 and Git. +# As a Ruby committer, run this in a git repository to commit a change. require 'tempfile' require 'net/http' @@ -15,20 +14,11 @@ ENV['LC_ALL'] = 'C' ORIGIN = 'git@git.ruby-lang.org:ruby.git' GITHUB = 'git@github.com:ruby/ruby.git' -module Merger - REPOS = 'svn+ssh://svn@ci.ruby-lang.org/ruby/' -end - -class << Merger - include Merger - +class << Merger = Object.new def help puts <<-HELP \e[1msimple backport\e[0m - ruby #$0 1234 - -\e[1mbackport from other branch\e[0m - ruby #$0 17502 mvm + ruby #$0 1234abc \e[1mrevision increment\e[0m ruby #$0 revisionup @@ -37,16 +27,16 @@ class << Merger ruby #$0 teenyup \e[1mtagging major release\e[0m - ruby #$0 tag 2.2.0 + ruby #$0 tag 3.2.0 -\e[1mtagging patch release\e[0m (about 2.1.0 or later, it means X.Y.Z (Z > 0) release) +\e[1mtagging patch release\e[0m (for 2.1.0 or later, it means X.Y.Z (Z > 0) release) ruby #$0 tag \e[1mtagging preview/RC\e[0m - ruby #$0 tag 2.2.0-preview1 + ruby #$0 tag 3.2.0-preview1 \e[1mremove tag\e[0m - ruby #$0 removetag 2.2.9 + ruby #$0 removetag 3.2.9 \e[33;1m* all operations shall be applied to the working directory.\e[0m HELP @@ -57,11 +47,11 @@ class << Merger yield if block_given? STDERR.puts "\e[1;33m#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})\e[0m" case STDIN.gets - when /\Aa/i then exit + when /\Aa/i then exit 1 when /\Ar/i then redo when /\Ay/i then break when /\Ae/i then system(ENV['EDITOR'], editfile) - else exit + else exit 1 end end end @@ -69,17 +59,14 @@ class << Merger def version_up(teeny: false) now = Time.now now = now.localtime(9*60*60) # server is Japan Standard Time +09:00 - if svn_mode? - system('svn', 'revert', 'version.h') - else - system('git', 'checkout', 'HEAD', 'version.h') - end + system('git', 'checkout', 'HEAD', 'version.h') v, pl = version if teeny v[2].succ! end - if pl != '-1' # trunk does not have patchlevel + # We stopped bumping RUBY_PATCHLEVEL at Ruby 4.0.0. + if Integer(v[0]) <= 3 pl.succ! end @@ -127,33 +114,21 @@ class << Merger abort 'no relname is given and not in a release branch even if this is patch release' end end - tagname = "v#{v.join('_')}#{("_#{pl}" if v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl)}" - - if svn_mode? - if relname - branch_url = `svn info`[/URL: (.*)/, 1] - else - branch_url = "#{REPOS}branches/ruby_" - if v[0] < '2' || (v[0] == '2' && v[1] < '1') - abort 'patchlevel must be greater than 0 for patch release' if pl == '0' - branch_url << v.join('_') - else - abort 'teeny must be greater than 0 for patch release' if v[2] == '0' - branch_url << v.join('_').sub(/_\d+\z/, '') - end - end - tag_url = "#{REPOS}tags/#{tagname}" - system('svn', 'info', tag_url, out: IO::NULL, err: IO::NULL) - if $?.success? - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('svn', 'cp', '-m', "add tag #{tagname}", branch_url, tag_url, interactive: true) + if /^(?:preview|rc)/ =~ pl + tagname = "v#{v.join('.')}-#{pl}" + elsif Integer(v[0]) >= 4 + tagname = "v#{v.join('.')}" else - unless execute('git', 'tag', tagname) - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('git', 'push', ORIGIN, tagname, interactive: true) + tagname = "v#{v.join('_')}" end + + unless execute('git', 'diff', '--exit-code') + abort 'uncommitted changes' + end + unless execute('git', 'tag', tagname) + abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' + end + execute('git', 'push', ORIGIN, tagname, interactive: true) end def remove_tag(relname) @@ -167,70 +142,52 @@ class << Merger unless relname raise ArgumentError, 'relname is not specified' end - if /^v/ !~ relname - tagname = "v#{relname.gsub(/[.-]/, '_')}" - else + if relname.start_with?('v') tagname = relname - end - - if svn_mode? - tag_url = "#{REPOS}tags/#{tagname}" - execute('svn', 'rm', '-m', "remove tag #{tagname}", tag_url, interactive: true) + elsif Integer(relname.split('.', 2).first) >= 4 + tagname = "v#{relname}" else - execute('git', 'tag', '-d', tagname) - execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) - execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) + tagname = "v#{relname.gsub(/[.-]/, '_')}" end + + execute('git', 'tag', '-d', tagname) + execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) + execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) end def update_revision_h - if svn_mode? - execute('svn', 'up') - end execute('ruby tool/file2lastrev.rb --revision.h . > revision.tmp') execute('tool/ifchange', '--timestamp=.revision.time', 'revision.h', 'revision.tmp') execute('rm', '-f', 'revision.tmp') end def stat - if svn_mode? - `svn stat` - else - `git status --short` - end + `git status --short` end def diff(file = nil) - if svn_mode? - command = %w[svn diff --diff-cmd=diff -x -upw] - else - command = %w[git diff --color HEAD] - end + command = %w[git diff --color HEAD] IO.popen(command + [file].compact, &:read) end def commit(file) - if svn_mode? - begin - execute('svn', 'ci', '-F', file) - execute('svn', 'update') # svn ci doesn't update revision info on working copy - ensure - execute('rm', '-f', 'subversion.commitlog') - end - else - current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip - execute('git', 'add', '.') && - execute('git', 'commit', '-F', file) - end + current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip + execute('git', 'add', '.') && execute('git', 'commit', '-F', file) end - private - - def svn_mode? - return @svn_mode if defined?(@svn_mode) - @svn_mode = system("svn info", %i(out err) => IO::NULL) + def has_conflicts? + changes = IO.popen(%w[git status --porcelain -z]) { |io| io.readlines("\0", chomp: true) } + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + !conflict.empty? end + private + # Prints the version of Ruby found in version.h def version v = p = nil @@ -289,10 +246,11 @@ else case ARGV[0] when /--ticket=(.*)/ - tickets = $1.split(/,/).map{|num| " [Backport ##{num}]"}.join + tickets = $1.split(/,/) ARGV.shift else - tickets = '' + tickets = [] + detect_ticket = true end revstr = ARGV[0].gsub(%r!https://github\.com/ruby/ruby/commit/|https://bugs\.ruby-lang\.org/projects/ruby-master/repository/git/revisions/!, '') @@ -303,8 +261,6 @@ else revs.each do |rev| git_rev = nil case rev - when /\A\d{1,6}\z/ - svn_rev = rev when /\A\h{7,40}\z/ git_rev = rev when nil then @@ -315,28 +271,25 @@ else exit end - # Merge revision from Git patch or SVN - if git_rev - git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" - resp = Net::HTTP.get_response(URI(git_uri)) - if resp.code != '200' - abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" - end - patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - - message = "\n\n#{(patch[/^Subject: (.*)\n\ndiff --git/m, 1] || "Message not found for revision: #{git_rev}\n")}" - puts '+ git apply' - IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } - else - default_merge_branch = (%r{^URL: .*/branches/ruby_1_8_} =~ `svn info` ? 'branches/ruby_1_8' : 'trunk') - svn_src = "#{Merger::REPOS}#{ARGV[1] || default_merge_branch}" - message = IO.popen(['svn', 'log', '-c', svn_rev, svn_src], &:read) + # Merge revision from Git patch + git_uri = "https://github.com/ruby/ruby/commit/#{git_rev}.patch" + resp = Net::HTTP.get_response(URI(git_uri)) + if resp.code != '200' + abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" + end + patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - cmd = ['svn', 'merge', '--accept=postpone', '-c', svn_rev, svn_src] - puts "+ #{cmd.join(' ')}" - system(*cmd) + if detect_ticket + tickets += patch.scan(/\[(?:Bug|Feature|Misc) #(\d+)\]/i).map(&:first) end + message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" + message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") + message = "\n\n#{message}" + + puts '+ git apply' + IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } + commit_message << message.sub(/\A-+\nr.*/, '').sub(/\n-+\n\z/, '').gsub(/^./, "\t\\&") end @@ -346,20 +299,22 @@ else Merger.version_up f = Tempfile.new 'merger.rb' - f.printf "merge revision(s) %s:%s", revstr, tickets + f.printf "merge revision(s) %s:%s", revs.join(', '), tickets.map{|num| " [Backport ##{num}]"}.join f.write commit_message f.flush f.close - Merger.interactive('conflicts resolved?', f.path) do - IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| - g << Merger.stat - g << "\n\n" - f.open - g << f.read - f.close - g << "\n\n" - g << Merger.diff + if Merger.has_conflicts? + Merger.interactive('conflicts resolved?', f.path) do + IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| + g << Merger.stat + g << "\n\n" + f.open + g << f.read + f.close + g << "\n\n" + g << Merger.diff + end end end diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat index d6d66a4d86..d39568fe86 100755 --- a/tool/missing-baseruby.bat +++ b/tool/missing-baseruby.bat @@ -1,5 +1,30 @@ -: " -@echo off -: " -echo executable host ruby is required. use --with-baseruby option. -exit 1 +:"" == " +@echo off || ( + :warn + echo>&2.%~1 + goto :eof + :abort + exit /b 1 +)||( +:)"||( + # necessary libraries + require 'erb' + require 'fileutils' + require 'tempfile' + s = %^# +) +: ; call() { local call=${1#:}; shift; $call "$@"; } +: ; warn() { echo "$1" >&2; } +: ; abort () { exit 1; } + +call :warn "executable host ruby is required. use --with-baseruby option." +call :warn "Note that BASERUBY must be Ruby 3.1.0 or later." +call :abort +(goto :eof ^;) +verbose = true if ARGV[0] == "--verbose" +case +when !defined?(RubyVM::InstructionSequence) + abort(*(["BASERUBY must be CRuby"] if verbose)) +when RUBY_VERSION < s[%r[warn .*\KBASERUBY .*Ruby ([\d.]+)(?:\.0)?.*(?=\")],1] + abort(*(["#{$&}. Found: #{RUBY_VERSION}"] if verbose)) +end diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index 95600e6a3b..5aa07962f9 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -6,7 +6,23 @@ require_relative 'ruby_vm/helpers/c_escape' SUBLIBS = {} REQUIRED = {} -BUILTIN_ATTRS = %w[leaf no_gc] +BUILTIN_ATTRS = %w[leaf inline_block use_block c_trace] + +module CompileWarning + @@warnings = 0 + + def warn(message) + @@warnings += 1 + super + end + + def self.reset + w, @@warnings = @@warnings, 0 + w.nonzero? + end +end + +Warning.extend CompileWarning def string_literal(lit, str = []) while lit @@ -166,7 +182,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil when 'cstmt' text = inline_text argc, args.first - func_name = "_bi#{inlines.size}" + func_name = "_bi#{lineno}" cfunc_name = make_cfunc_name(inlines, name, lineno) inlines[cfunc_name] = [lineno, text, locals, func_name] argc -= 1 @@ -174,7 +190,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil text = inline_text argc, args.first code = "return #{text};" - func_name = "_bi#{inlines.size}" + func_name = "_bi#{lineno}" cfunc_name = make_cfunc_name(inlines, name, lineno) locals = [] if $1 == 'cconst' @@ -266,16 +282,22 @@ def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_nam # Avoid generating fetches of lvars we don't need. This is imperfect as it # will match text inside strings or other false positives. - local_candidates = text.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + local_ptrs = [] + local_candidates = text.gsub(/\bLOCAL_PTR\(\K[a-zA-Z_][a-zA-Z0-9_]*(?=\))/) { + local_ptrs << $&; '' + }.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) f.puts '{' lineno += 1 # locals is nil outside methods locals&.reverse_each&.with_index{|param, i| next unless Symbol === param - next unless local_candidates.include?(param.to_s) - f.puts "MAYBE_UNUSED(const VALUE) #{param} = rb_vm_lvar(ec, #{-3 - i});" - lineno += 1 + param = param.to_s + lvar = local_candidates.include?(param) + next unless lvar or local_ptrs.include?(param) + f.puts "VALUE *const #{param}__ptr = (VALUE *)&ec->cfp->ep[#{-3 - i}];" + f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" if lvar + lineno += lvar ? 2 : 1 } f.puts "#line #{body_lineno} \"#{line_file}\"" lineno += 1 @@ -299,16 +321,18 @@ def mk_builtin_header file # bs = { func_name => argc } code = File.read(file) - collect_iseq RubyVM::InstructionSequence.compile(code).to_a - collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) - begin - f = File.open(ofile, 'w') - rescue SystemCallError # EACCES, EPERM, EROFS, etc. - # Fall back to the current directory - f = File.open(File.basename(ofile), 'w') + verbose, $VERBOSE = $VERBOSE, true + collect_iseq RubyVM::InstructionSequence.compile(code, base).to_a + ensure + $VERBOSE = verbose end - begin + if warnings = CompileWarning.reset + raise "#{warnings} warnings in #{file}" + end + collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) + + StringIO.open do |f| if File::ALT_SEPARATOR file = file.tr(File::ALT_SEPARATOR, File::SEPARATOR) ofile = ofile.tr(File::ALT_SEPARATOR, File::SEPARATOR) @@ -389,8 +413,13 @@ def mk_builtin_header file f.puts " rb_load_with_builtin_functions(#{base.dump}, #{table});" f.puts "}" - ensure - f.close + + begin + File.write(ofile, f.string) + rescue SystemCallError # EACCES, EPERM, EROFS, etc. + # Fall back to the current directory + File.write(File.basename(ofile), f.string) + end end end diff --git a/tool/mk_rbbin.rb b/tool/mk_rbbin.rb new file mode 100755 index 0000000000..991230f094 --- /dev/null +++ b/tool/mk_rbbin.rb @@ -0,0 +1,48 @@ +#!ruby -s + +OPTIMIZATION = { + inline_const_cache: true, + peephole_optimization: true, + tailcall_optimization: false, + specialized_instruction: true, + operands_unification: true, + instructions_unification: true, + frozen_string_literal: true, + debug_frozen_string_literal: false, + coverage_enabled: false, + debug_level: 0, +} + +file = File.basename(ARGV[0], ".rb") +name = "<internal:#{file}>" +iseq = RubyVM::InstructionSequence.compile(ARGF.read, name, name, **OPTIMIZATION) +puts <<C +/* -*- C -*- */ + +static const char #{file}_builtin[] = { +C +iseq.to_binary.bytes.each_slice(8) do |b| + print " ", b.map {|c| "0x%.2x," % c}.join(" ") + if $comment + print " /* ", b.pack("C*").gsub(/([[ -~]&&[^\\]])|(?m:.)/) { + (c = $1) ? "#{c} " : (c = $&.dump).size == 2 ? c : ". " + }, "*/" + end + puts +end +puts <<C +}; + +#include "ruby/ruby.h" +#include "vm_core.h" + +void +Init_#{file}(void) +{ + const char *builtin = #{file}_builtin; + size_t size = sizeof(#{file}_builtin); + VALUE code = rb_str_new_static(builtin, (long)size); + VALUE iseq = rb_funcallv(rb_cISeq, rb_intern_const("load_from_binary"), 1, &code); + rb_funcallv(iseq, rb_intern_const("eval"), 0, 0); +} +C diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb index 55e781a28e..db74115730 100755 --- a/tool/mkconfig.rb +++ b/tool/mkconfig.rb @@ -1,4 +1,5 @@ #!./miniruby -s +# frozen-string-literal: true # This script, which is run when ruby is built, generates rbconfig.rb by # parsing information from config.status. rbconfig.rb contains build @@ -63,8 +64,6 @@ File.foreach "config.status" do |line| when /^(?:X|(?:MINI|RUN|(?:HAVE_)?BASE|BOOTSTRAP|BTEST)RUBY(?:_COMMAND)?$)/; next when /^INSTALLDOC|TARGET$/; next when /^DTRACE/; next - when /^RJIT_(CC|SUPPORT)$/; # pass - when /^RJIT_/; next when /^(?:MAJOR|MINOR|TEENY)$/; vars[name] = val; next when /^LIBRUBY_D?LD/; next when /^RUBY_INSTALL_NAME$/; next vars[name] = (install_name = val).dup if $install_name @@ -169,8 +168,8 @@ def vars.expand(val, config = self) val.replace(newval) unless newval == val val end -prefix = vars.expand(vars["prefix"] ||= "") -rubyarchdir = vars.expand(vars["rubyarchdir"] ||= "") +prefix = vars.expand(vars["prefix"] ||= +"") +rubyarchdir = vars.expand(vars["rubyarchdir"] ||= +"") relative_archdir = rubyarchdir.rindex(prefix, 0) ? rubyarchdir[prefix.size..-1] : rubyarchdir puts %[\ @@ -257,14 +256,14 @@ end v_others.compact! if $install_name - if install_name and vars.expand("$(RUBY_INSTALL_NAME)") == $install_name + if install_name and vars.expand(+"$(RUBY_INSTALL_NAME)") == $install_name $install_name = install_name end v_fast << " CONFIG[\"ruby_install_name\"] = \"" + $install_name + "\"\n" v_fast << " CONFIG[\"RUBY_INSTALL_NAME\"] = \"" + $install_name + "\"\n" end if $so_name - if so_name and vars.expand("$(RUBY_SO_NAME)") == $so_name + if so_name and vars.expand(+"$(RUBY_SO_NAME)") == $so_name $so_name = so_name end v_fast << " CONFIG[\"RUBY_SO_NAME\"] = \"" + $so_name + "\"\n" @@ -395,6 +394,7 @@ print <<EOS ) end end +# Non-nil if configured for cross compiling. CROSS_COMPILING = nil unless defined? CROSS_COMPILING EOS diff --git a/tool/mkrunnable.rb b/tool/mkrunnable.rb index 8bfb4fe6a4..ef358e2425 100755 --- a/tool/mkrunnable.rb +++ b/tool/mkrunnable.rb @@ -6,6 +6,7 @@ require './rbconfig' require 'fileutils' +require_relative 'lib/path' case ARGV[0] when "-n" @@ -18,93 +19,7 @@ else include FileUtils end -module Mswin - def ln_safe(src, dest, *opt) - cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] - cmd[1, 0] = opt - return if system("cmd", "/c", *cmd) - # TODO: use RUNAS or something - puts cmd.join(" ") - end - - def ln_dir_safe(src, dest) - ln_safe(src, dest, "/d") - end -end - -def clean_link(src, dest) - begin - link = File.readlink(dest) - rescue - else - return if link == src - File.unlink(dest) - end - yield src, dest -end - -def ln_safe(src, dest) - ln_sf(src, dest) -rescue Errno::ENOENT - # Windows disallows to create broken symboic links, probably because - # it is a kind of reparse points. - raise if File.exist?(src) -end - -alias ln_dir_safe ln_safe - -case RUBY_PLATFORM -when /linux|darwin|solaris/ - def ln_exe(src, dest) - ln(src, dest, force: true) - end -else - alias ln_exe ln_safe -end - -if !File.respond_to?(:symlink) && /mingw|mswin/ =~ (CROSS_COMPILING || RUBY_PLATFORM) - extend Mswin -end - -def clean_path(path) - path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') - nil while path.sub!(/[^\/]+\/\.\.\//, '') - path -end - -def relative_path_from(path, base) - path = clean_path(path) - base = clean_path(base) - path, base = [path, base].map{|s|s.split("/")} - until path.empty? or base.empty? or path[0] != base[0] - path.shift - base.shift - end - path, base = [path, base].map{|s|s.join("/")} - if /(\A|\/)\.\.\// =~ base - File.expand_path(path) - else - base.gsub!(/[^\/]+/, '..') - File.join(base, path) - end -end - -def ln_relative(src, dest, executable = false) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - if executable - return (ln_exe(src, dest) if File.exist?(src)) - end - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_safe(s, d)} -end - -def ln_dir_relative(src, dest) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_dir_safe(s, d)} -end +include Path config = RbConfig::MAKEFILE_CONFIG.merge("prefix" => ".", "exec_prefix" => ".") config.each_value {|s| RbConfig.expand(s, config)} @@ -119,18 +34,25 @@ vendordir = config["vendordir"] rubylibdir = config["rubylibdir"] rubyarchdir = config["rubyarchdir"] archdir = "#{extout}/#{arch}" -[bindir, libdir, archdir].uniq.each do |dir| +exedir = bindir +if libdirname == "archlibdir" + exedir = exedir.sub(%r[/\K(?=[^/]+\z)]) {extout+"/"} +end +[exedir, libdir, archdir].uniq.each do |dir| File.directory?(dir) or mkdir_p(dir) end +unless exedir == bindir + ln_dir_relative(exedir, bindir) +end exeext = config["EXEEXT"] ruby_install_name = config["ruby_install_name"] rubyw_install_name = config["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name -[ruby_install_name, rubyw_install_name, goruby_install_name].map do |ruby| +[ruby_install_name, rubyw_install_name, goruby_install_name].each do |ruby| if ruby and !ruby.empty? ruby += exeext - ln_relative(ruby, "#{bindir}/#{ruby}", true) + ln_relative(ruby, "#{exedir}/#{ruby}", true) end end so = config["LIBRUBY_SO"] diff --git a/tool/notes-github-pr.rb b/tool/notes-github-pr.rb new file mode 100644 index 0000000000..d69d479cdf --- /dev/null +++ b/tool/notes-github-pr.rb @@ -0,0 +1,138 @@ +#!/usr/bin/env ruby +# Add GitHub pull request reference / author info to git notes. + +require 'net/http' +require 'uri' +require 'tmpdir' +require 'json' +require 'yaml' + +# Conversion for people whose GitHub account name and SVN_ACCOUNT_NAME are different. +GITHUB_TO_SVN = { + 'amatsuda' => 'a_matsuda', + 'matzbot' => 'git', + 'jeremyevans' => 'jeremy', + 'znz' => 'kazu', + 'k-tsj' => 'ktsj', + 'nurse' => 'naruse', + 'ioquatix' => 'samuel', + 'suketa' => 'suke', + 'unak' => 'usa', +} + +EMAIL_YML_URL = 'https://raw.githubusercontent.com/ruby/git.ruby-lang.org/refs/heads/master/config/email.yml' +SVN_TO_EMAILS = YAML.safe_load(Net::HTTP.get_response(URI(EMAIL_YML_URL)).tap(&:value).body) + +class GitHub + ENDPOINT = URI.parse('https://api.github.com') + + def initialize(access_token) + @access_token = access_token + end + + # https://developer.github.com/changes/2019-04-11-pulls-branches-for-commit/ + def pulls(owner:, repo:, commit_sha:) + resp = get("/repos/#{owner}/#{repo}/commits/#{commit_sha}/pulls", accept: 'application/vnd.github.groot-preview+json') + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/pulls/#get-a-single-pull-request + def pull_request(owner:, repo:, number:) + resp = get("/repos/#{owner}/#{repo}/pulls/#{number}") + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/users/#get-a-single-user + def user(username:) + resp = get("/users/#{username}") + JSON.parse(resp.body) + end + + private + + def get(path, accept: 'application/vnd.github.v3+json') + Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: ENDPOINT.scheme == 'https') do |http| + headers = { 'Accept': accept, 'Authorization': "bearer #{@access_token}" } + http.get(path, headers).tap(&:value) + end + end +end + +module Git + class << self + def abbrev_ref(refname, repo_path:) + git('rev-parse', '--symbolic', '--abbrev-ref', refname, repo_path: repo_path).strip + end + + def rev_list(arg, first_parent: false, repo_path: nil) + git('rev-list', *[('--first-parent' if first_parent)].compact, arg, repo_path: repo_path).lines.map(&:chomp) + end + + def commit_message(sha) + git('log', '-1', '--pretty=format:%B', sha) + end + + def notes_message(sha) + git('log', '-1', '--pretty=format:%N', sha) + end + + def committer_name(sha) + git('log', '-1', '--pretty=format:%cn', sha) + end + + def committer_email(sha) + git('log', '-1', '--pretty=format:%cE', sha) + end + + private + + def git(*cmd, repo_path: nil) + env = {} + if repo_path + env['GIT_DIR'] = repo_path + end + out = IO.popen(env, ['git', *cmd], &:read) + unless $?.success? + abort "Failed to execute: git #{cmd.join(' ')}\n#{out}" + end + out + end + end +end + +github = GitHub.new(ENV.fetch('GITHUB_TOKEN')) + +repo_path, *rest = ARGV +rest.each_slice(3).map do |oldrev, newrev, _refname| + system('git', 'fetch', 'origin', 'refs/notes/commits:refs/notes/commits', exception: true) + + updated = false + Git.rev_list("#{oldrev}..#{newrev}", first_parent: true).each do |sha| + github.pulls(owner: 'ruby', repo: 'ruby', commit_sha: sha).each do |pull| + number = pull.fetch('number') + url = pull.fetch('html_url') + next unless url.start_with?('https://github.com/ruby/ruby/pull/') + + # "Merged" notes for "Squash and merge" + message = Git.commit_message(sha) + notes = Git.notes_message(sha) + if !message.include?(url) && !message.match(/[ (]##{number}[) ]/) && !notes.include?(url) + system('git', 'notes', 'append', '-m', "Merged: #{url}", sha, exception: true) + updated = true + end + + # "Merged-By" notes for "Rebase and merge" + if Git.committer_name(sha) == 'GitHub' && Git.committer_email(sha) == 'noreply@github.com' + username = github.pull_request(owner: 'ruby', repo: 'ruby', number: number).fetch('merged_by').fetch('login') + email = github.user(username: username).fetch('email') + email ||= SVN_TO_EMAILS[GITHUB_TO_SVN.fetch(username, username)]&.first + system('git', 'notes', 'append', '-m', "Merged-By: #{username}#{(" <#{email}>" if email)}", sha, exception: true) + updated = true + end + end + end + + if updated + system('git', 'push', 'origin', 'refs/notes/commits', exception: true) + end +end diff --git a/tool/notify-slack-commits.rb b/tool/notify-slack-commits.rb new file mode 100644 index 0000000000..73e22b9a03 --- /dev/null +++ b/tool/notify-slack-commits.rb @@ -0,0 +1,87 @@ +#!/usr/bin/env ruby + +require "net/https" +require "open3" +require "json" +require "digest/md5" + +SLACK_WEBHOOK_URLS = [ + ENV.fetch("SLACK_WEBHOOK_URL_ALERTS").chomp, # ruby-lang#alerts + ENV.fetch("SLACK_WEBHOOK_URL_COMMITS").chomp, # ruby-lang#commits + ENV.fetch("SLACK_WEBHOOK_URL_RUBY_JP").chomp, # ruby-jp#ruby-commits +] +GRAVATAR_OVERRIDES = { + "nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars0.githubusercontent.com/u/21976", + "noreply@github.com" => "https://avatars1.githubusercontent.com/u/9919", + "nurse@users.noreply.github.com" => "https://avatars1.githubusercontent.com/u/13423", + "svn-admin@ruby-lang.org" => "https://avatars1.githubusercontent.com/u/29403229", + "svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars1.githubusercontent.com/u/29403229", + "usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars2.githubusercontent.com/u/17790", + "usa@ruby-lang.org" => "https://avatars2.githubusercontent.com/u/17790", + "yui-knk@ruby-lang.org" => "https://avatars0.githubusercontent.com/u/5356517", + "znz@users.noreply.github.com" => "https://avatars3.githubusercontent.com/u/11857", +} + +def escape(s) + s.gsub(/[&<>]/, "&" => "&", "<" => "<", ">" => ">") +end + +ARGV.each_slice(3) do |oldrev, newrev, refname| + out, = Open3.capture2("git", "rev-parse", "--symbolic", "--abbrev-ref", refname) + branch = out.strip + + out, = Open3.capture2("git", "log", "--pretty=format:%H\n%h\n%cn\n%ce\n%ct\n%B", "--abbrev=10", "-z", "#{oldrev}..#{newrev}") + + attachments = [] + out.split("\0").reverse_each do |s| + sha, sha_abbr, committer, committeremail, committertime, body = s.split("\n", 6) + subject, body = body.split("\n", 2) + + # Append notes content to `body` if it's notes + if refname.match(%r[\Arefs/notes/\w+\z]) + # `--diff-filter=AM -M` to exclude rename by git's directory optimization + object = IO.popen(["git", "diff", "--diff-filter=AM", "-M", "--name-only", "#{sha}^..#{sha}"], &:read).chomp + if md = object.match(/\A(?<prefix>\h{2})\/?(?<rest>\h{38})\z/) + body = [body, IO.popen(["git", "notes", "show", md[:prefix] + md[:rest]], &:read)].join + end + end + + gravatar = GRAVATAR_OVERRIDES.fetch(committeremail) do + "https://www.gravatar.com/avatar/#{ Digest::MD5.hexdigest(committeremail.downcase) }" + end + + attachments << { + title: "#{ sha_abbr } (#{ branch }): #{ escape(subject) }", + title_link: "https://github.com/ruby/ruby/commit/#{ sha }", + text: escape((body || "").strip), + footer: committer, + footer_icon: gravatar, + ts: committertime.to_i, + color: '#24282D', + } + end + + # 100 attachments cannot be exceeded. 20 is recommended. https://api.slack.com/docs/message-attachments + attachments.each_slice(20).each do |attachments_group| + payload = { attachments: attachments_group } + + #Net::HTTP.post( + # URI.parse(SLACK_WEBHOOK_URL), + # JSON.generate(payload), + # "Content-Type" => "application/json" + #) + responses = SLACK_WEBHOOK_URLS.map do |url| + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.start do + req = Net::HTTP::Post.new(uri.path) + req.set_form_data(payload: payload.to_json) + http.request(req) + end + end + + results = responses.map { |resp| "#{resp.code} (#{resp.body})" }.join(', ') + puts "#{results} -- #{payload.to_json}" + end +end diff --git a/tool/outdate-bundled-gems.rb b/tool/outdate-bundled-gems.rb index 68adb3d7dc..47ee80bc89 100755 --- a/tool/outdate-bundled-gems.rb +++ b/tool/outdate-bundled-gems.rb @@ -3,34 +3,52 @@ require 'fileutils' require 'rubygems' fu = FileUtils::Verbose + until ARGV.empty? case ARGV.first when '--' ARGV.shift break - when '-n', '--dryrun' + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Don't remove fu = FileUtils::DryRun when /\A--make=/ # just to run when `make -n` when /\A--mflags=(.*)/ fu = FileUtils::DryRun if /\A-\S*n/ =~ $1 when /\A--gem[-_]platform=(.*)/im + ## --gem-platform=PLATFORM Platform in RubyGems style gem_platform = $1 ruby_platform = nil when /\A--ruby[-_]platform=(.*)/im + ## --ruby-platform=PLATFORM Platform in Ruby style ruby_platform = $1 gem_platform = nil when /\A--ruby[-_]version=(.*)/im + ## --ruby-version=VERSION Ruby version to keep ruby_version = $1 when /\A--only=(?:(curdir|srcdir)|all)\z/im + ## --only=(curdir|srcdir|all) Specify directory to remove gems from only = $1&.downcase when /\A--all\z/im + ## --all Remove all gems not only bundled gems all = true + when /\A--help\z/im + ## --help Print this message + puts "Usage: #$0 [options] [srcdir]" + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ raise "#{$0}: unknown option: #{ARGV.first}" else break end + ## ARGV.shift end @@ -97,7 +115,7 @@ srcdir = Removal.new(ARGV.shift) curdir = !srcdir.base || File.identical?(srcdir.base, ".") ? srcdir : Removal.new bundled = File.readlines("#{srcdir.base}gems/bundled_gems"). - grep(/^(\w\S+)\s+\S+(?:\s+\S+\s+(\S+))?/) {$~.captures}.to_h rescue nil + grep(/^(\w[^\#\s]+)\s+[^\#\s]+(?:\s+[^\#\s]+\s+([^\#\s]+))?/) {$~.captures}.to_h rescue nil srcdir.glob(".bundle/gems/*/") do |dir| base = File.basename(dir) diff --git a/tool/prereq.status b/tool/prereq.status index 6de00c8a92..6aca615e90 100644 --- a/tool/prereq.status +++ b/tool/prereq.status @@ -41,4 +41,5 @@ s,@rubylibprefix@,,g s,@srcdir@,.,g s/@[A-Za-z][A-Za-z0-9_]*@//g -s/{\$([A-Za-z]*)}//g +s/{\$([^(){}]*)}//g +s/^!/#!/ diff --git a/tool/prism_btests b/tool/prism_btests deleted file mode 100644 index e14f0d3cf9..0000000000 --- a/tool/prism_btests +++ /dev/null @@ -1,35 +0,0 @@ -../src/bootstraptest/test_attr.rb -../src/bootstraptest/test_autoload.rb -../src/bootstraptest/test_class.rb -../src/bootstraptest/test_constant_cache.rb -../src/bootstraptest/test_env.rb -../src/bootstraptest/test_eval.rb -../src/bootstraptest/test_fiber.rb -../src/bootstraptest/test_finalizer.rb -../src/bootstraptest/test_flip.rb -../src/bootstraptest/test_fork.rb -../src/bootstraptest/test_gc.rb -../src/bootstraptest/test_jump.rb -../src/bootstraptest/test_literal.rb -../src/bootstraptest/test_literal_suffix.rb -../src/bootstraptest/test_load.rb -../src/bootstraptest/test_marshal.rb -../src/bootstraptest/test_objectspace.rb -../src/bootstraptest/test_proc.rb -../src/bootstraptest/test_rjit.rb -../src/bootstraptest/test_string.rb -../src/bootstraptest/test_struct.rb -../src/bootstraptest/test_thread.rb -../src/bootstraptest/test_block.rb -# ../src/bootstraptest/test_exception.rb -# ../src/bootstraptest/test_flow.rb -# ../src/bootstraptest/test_insns.rb -# ../src/bootstraptest/test_io.rb -# ../src/bootstraptest/test_massign.rb -# ../src/bootstraptest/test_method.rb -# ../src/bootstraptest/test_ractor.rb -# ../src/bootstraptest/test_syntax.rb -# ../src/bootstraptest/test_yjit.rb -../src/bootstraptest/test_yjit_30k_ifelse.rb -../src/bootstraptest/test_yjit_30k_methods.rb -# ../src/bootstraptest/test_yjit_rust_port.rb diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 90697bfe92..874c3ef1d9 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -28,6 +28,7 @@ begin rescue LoadError $" << "zlib.rb" end +require_relative 'lib/path' INDENT = " "*36 STDOUT.sync = true @@ -96,6 +97,20 @@ def parse_args(argv = ARGV) opt.on('--gnumake') {gnumake = true} opt.on('--debug-symbols=SUFFIX', /\w+/) {|name| $debug_symbols = ".#{name}"} + unless $install_procs.empty? + w = (w = ENV["COLUMNS"] and (w = w.to_i) > 80) ? w - 30 : 50 + opt.on("\n""Types for --install and --exclude:") + mesg = +" " + $install_procs.each_key do |t| + if mesg.size + t.size > w + opt.on(mesg) + mesg = +" " + end + mesg << " " << t.to_s + end + opt.on(mesg) + end + opt.order!(argv) do |v| case v when /\AINSTALL[-_]([-\w]+)=(.*)/ @@ -135,6 +150,7 @@ def parse_args(argv = ARGV) end $destdir ||= $mflags.defined?("DESTDIR") + $destdir = File.expand_path($destdir) unless $destdir.empty? if $extout ||= $mflags.defined?("EXTOUT") RbConfig.expand($extout) end @@ -157,6 +173,16 @@ def parse_args(argv = ARGV) end end +Compressors = {".gz"=>"gzip", ".bz2"=>"bzip2"} +def Compressors.for(type) + ext = File.extname(type) + if compress = fetch(ext, nil) + [type.chomp(ext), ext, compress] + else + [type, *find {|_, z| system(z, in: IO::NULL, out: IO::NULL)}] + end +end + $install_procs = Hash.new {[]} def install?(*types, &block) unless types.delete(:nodefault) @@ -205,15 +231,20 @@ def ln_sf(src, dest) end $made_dirs = {} + +def dir_creating(dir) + $made_dirs.fetch(dir) do + $made_dirs[dir] = true + $installed_list.puts(File.join(dir, "")) if $installed_list + yield if defined?(yield) + end +end + def makedirs(dirs) dirs = fu_list(dirs) dirs.collect! do |dir| realdir = with_destdir(dir) - realdir unless $made_dirs.fetch(dir) do - $made_dirs[dir] = true - $installed_list.puts(File.join(dir, "")) if $installed_list - File.directory?(realdir) - end + realdir unless dir_creating(dir) {File.directory?(realdir)} end.compact! super(dirs, :mode => $dir_mode) unless dirs.empty? end @@ -346,6 +377,9 @@ rubyw_install_name = CONFIG["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name bindir = CONFIG["bindir", true] +if CONFIG["libdirname"] == "archlibdir" + archbindir = bindir.sub(%r[/\K(?=[^/]+\z)]) {CONFIG["config_target"] + "/"} +end libdir = CONFIG[CONFIG.fetch("libdirname", "libdir"), true] rubyhdrdir = CONFIG["rubyhdrdir", true] archhdrdir = CONFIG["rubyarchhdrdir"] || (rubyhdrdir + "/" + CONFIG['arch']) @@ -369,113 +403,13 @@ load_relative = CONFIG["LIBRUBY_RELATIVE"] == 'yes' rdoc_noinst = %w[created.rid] -install?(:local, :arch, :bin, :'bin-arch') do - prepare "binary commands", bindir - - install ruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - if rubyw_install_name and !rubyw_install_name.empty? - install rubyw_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support - # to load and execute the second .wasm file. Both are required to execute ruby - if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" - install ruby_install_name+".wasm", bindir, :mode => $prog_mode, :strip => $strip - end - if File.exist? goruby_install_name+exeext - install goruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - if enable_shared and dll != lib - install dll, bindir, :mode => $prog_mode, :strip => $strip - end -end - -install?(:local, :arch, :lib, :'lib-arch') do - prepare "base libraries", libdir - - install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc - install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" - if dll == lib and dll != arc - for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] - ln_sf(dll, File.join(libdir, link)) - end - end - - prepare "arch files", archlibdir - install "rbconfig.rb", archlibdir, :mode => $data_mode - if CONFIG["ARCHFILE"] - for file in CONFIG["ARCHFILE"].split - install file, archlibdir, :mode => $data_mode - end - end -end - -install?(:local, :arch, :data) do - pc = CONFIG["ruby_pc"] - if pc and File.file?(pc) and File.size?(pc) - prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") - install pc, pkgconfigdir, :mode => $data_mode - end -end - -install?(:ext, :arch, :'ext-arch') do - prepare "extension objects", archlibdir - noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split - install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) - prepare "extension objects", sitearchlibdir - prepare "extension objects", vendorarchlibdir - if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and - !(extso = extso.gsub(/\\\n/, '').split).empty? - libpathenv = CONFIG["LIBPATHENV"] - dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] - prepare "external libraries", dest - for file in extso - install file, dest, :mode => $prog_mode - end - end -end -install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do - prepare "extension headers", archhdrdir - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_rjit_header-*.obj", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_rjit_header-*.pch", :mode => $data_mode) -end -install?(:ext, :comm, :'ext-comm') do - prepare "extension scripts", rubylibdir - install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) - prepare "extension scripts", sitelibdir - prepare "extension scripts", vendorlibdir -end -install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do - hdrdir = rubyhdrdir + "/ruby" - prepare "extension headers", hdrdir - install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) -end - -install?(:doc, :rdoc) do - if $rdocdir - ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") - prepare "rdoc", ridatadir - install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :html) do - if $htmldir - prepare "html-docs", docdir - install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :capi) do - prepare "capi-docs", docdir - install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode -end - prolog_script = <<EOS bindir="#{load_relative ? '${0%/*}' : bindir.gsub(/\"/, '\\\\"')}" EOS -if CONFIG["LIBRUBY_RELATIVE"] != 'yes' and libpathenv = CONFIG["LIBPATHENV"] +if !load_relative and libpathenv = CONFIG["LIBPATHENV"] pathsep = File::PATH_SEPARATOR prolog_script << <<EOS -libdir="#{load_relative ? '$\{bindir%/bin\}/lib' : libdir.gsub(/\"/, '\\\\"')}" +libdir="#{libdir.gsub(/\"/, '\\\\"')}" export #{libpathenv}="$libdir${#{libpathenv}:+#{pathsep}$#{libpathenv}}" EOS end @@ -584,129 +518,6 @@ $script_installer = Class.new(installer) do break new(ruby_shebang, ruby_bin, ruby_install_name, nil, trans) end -install?(:local, :comm, :bin, :'bin-comm') do - prepare "command scripts", bindir - - install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| - $script_installer.install(src, cmd) - end -end - -install?(:local, :comm, :lib) do - prepare "library scripts", rubylibdir - noinst = %w[*.txt *.rdoc *.gemspec] - install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) -end - -install?(:local, :comm, :hdr, :'comm-hdr') do - prepare "common headers", rubyhdrdir - - noinst = [] - unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ - noinst << "win32.h" - end - noinst = nil if noinst.empty? - install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) -end - -install?(:local, :comm, :man) do - mdocs = Dir["#{srcdir}/man/*.[1-9]"] - prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} - - case $mantype - when /\.(?:(gz)|bz2)\z/ - compress = $1 ? "gzip" : "bzip2" - suffix = $& - end - mandir = File.join(mandir, "man") - has_goruby = File.exist?(goruby_install_name+exeext) - require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype - mdocs.each do |mdoc| - next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' - base = File.basename(mdoc) - if base == "goruby.1" - next unless has_goruby - end - - destdir = mandir + (section = mdoc[/\d+$/]) - destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) - destfile = File.join(destdir, "#{destname}.#{section}") - - if /\Adoc\b/ =~ $mantype - if compress - begin - w = IO.popen(compress, "rb", in: mdoc, &:read) - rescue - else - destfile << suffix - end - end - if w - open_for_install(destfile, $data_mode) {w} - else - install mdoc, destfile, :mode => $data_mode - end - else - class << (w = []) - alias print push - end - if File.basename(mdoc).start_with?('bundle') || - File.basename(mdoc).start_with?('gemfile') - w = File.read(mdoc) - else - File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} - w = w.join("") - end - if compress - begin - w = IO.popen(compress, "r+b") do |f| - Thread.start {f.write w; f.close_write} - f.read - end - rescue - else - destfile << suffix - end - end - open_for_install(destfile, $data_mode) {w} - end - end -end - -install?(:dbg, :nodefault) do - prepare "debugger commands", bindir - prepare "debugger scripts", rubylibdir - conf = RbConfig::MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) - Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| - cmd = $script_installer.transform(File.basename(src, ".in")) - open_for_install(File.join(bindir, cmd), $script_mode) { - RbConfig.expand(File.read(src), conf) - } - end - Dir.glob(File.join(srcdir, "misc/lldb_*")) do |src| - if File.directory?(src) - install_recursive src, File.join(rubylibdir, File.basename(src)) - else - install src, rubylibdir - end - end - install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") - if $debug_symbols - { - ruby_install_name => bindir, - rubyw_install_name => bindir, - goruby_install_name => bindir, - dll => libdir, - }.each do |src, dest| - next if src.empty? - src += $debug_symbols - if File.directory?(src) - install_recursive src, File.join(dest, src) - end - end - end -end - module RbInstall def self.no_write(options = nil) u = File.umask(0022) @@ -716,6 +527,8 @@ module RbInstall const_set(:FileUtils, fu::NoWrite) fu end + # RubyGems 3.0.0 or later supports `dir_mode`, but it uses + # `File` method to apply it, not `FileUtils`. dir_mode = options.delete(:dir_mode) if options end yield @@ -745,47 +558,119 @@ module RbInstall end def collect - ruby_libraries.sort + requirable_features.sort + end + + private + + def features_from_makefile(makefile_path) + makefile = File.read(makefile_path) + + name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + return [] if name.nil? || name.empty? + + feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = feature.sub("$(TARGET)", name) + + target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? + + Array(feature) end class Ext < self - def skip_install?(files) + def requirable_features # install ext only when it's configured - !File.exist?("#{$ext_build_dir}/#{relative_base}/Makefile") + return [] unless File.exist?(makefile_path) + + ruby_features + ext_features + end + + private + + def ruby_features + Dir.glob("**/*.rb", base: "#{makefile_dir}/lib") + end + + def ext_features + features_from_makefile(makefile_path) end - def ruby_libraries - Dir.glob("lib/**/*.rb", base: "#{srcdir}/ext/#{relative_base}") + def makefile_path + if File.exist?("#{makefile_dir}/Makefile") + "#{makefile_dir}/Makefile" + else + # for out-of-place build + "#{$ext_build_dir}/#{relative_base}/Makefile" + end + end + + def makefile_dir + "#{root}/#{relative_base}" + end + + def root + File.expand_path($ext_build_dir, srcdir) end end class Lib < self - def skip_install?(files) - files.empty? + def requirable_features + ruby_features + ext_features end - def ruby_libraries + private + + def ruby_features gemname = File.basename(gemspec, ".gemspec") base = relative_base || gemname # for lib/net/net-smtp.gemspec if m = /.*(?=-(.*)\z)/.match(gemname) base = File.join(base, *m.to_a.select {|n| !base.include?(n)}) end - files = Dir.glob("lib/#{base}{.rb,/**/*.rb}", base: srcdir) + files = Dir.glob("#{base}{.rb,/**/*.rb}", base: root) if !relative_base and files.empty? # no files at the toplevel # pseudo gem like ruby2_keywords - files << "lib/#{gemname}.rb" + files << "#{gemname}.rb" end case gemname when "net-http" - files << "lib/net/https.rb" + files << "net/https.rb" when "optparse" - files << "lib/optionparser.rb" + files << "optionparser.rb" end files end + + def ext_features + loaded_gemspec = load_gemspec("#{root}/#{gemspec}") + extension = loaded_gemspec.extensions.first + return [] unless extension + + extconf = File.expand_path(extension, srcdir) + ext_build_dir = File.dirname(extconf) + makefile_path = "#{ext_build_dir}/Makefile" + return [] unless File.exist?(makefile_path) + + features_from_makefile(makefile_path) + end + + def root + "#{srcdir}/lib" + end + end + + class UnpackedGem < self + def collect + base = @srcdir or return [] + Dir.glob("**/*", File::FNM_DOTMATCH, base: base).select do |n| + next if n == "." + next if File.fnmatch?("*.gemspec", n, File::FNM_DOTMATCH|File::FNM_PATHNAME) + !File.directory?(File.join(base, n)) + end + end end end end @@ -823,10 +708,78 @@ module RbInstall end end - class GemInstaller < Gem::Installer - end + class UnpackedInstaller < Gem::Installer + # This method is mostly copied from old version of Gem::Installer#install + def install_with_default_gem + verify_gem_home + + # The name and require_paths must be verified first, since it could contain + # ruby code that would be eval'ed in #ensure_loadable_spec + verify_spec + + ensure_loadable_spec + + if options[:install_as_default] + Gem.ensure_default_gem_subdirectories gem_home + else + Gem.ensure_gem_subdirectories gem_home + end + + return true if @force + + ensure_dependencies_met unless @ignore_dependencies + + run_pre_install_hooks + + # Set loaded_from to ensure extension_dir is correct + if @options[:install_as_default] + spec.loaded_from = default_spec_file + else + spec.loaded_from = spec_file + end + + # Completely remove any previous gem files + FileUtils.rm_rf gem_dir + FileUtils.rm_rf spec.extension_dir + + dir_mode = options[:dir_mode] + FileUtils.mkdir_p gem_dir, mode: dir_mode && 0o755 + + if @options[:install_as_default] + extract_bin + write_default_spec + else + extract_files + + build_extensions + write_build_info_file + run_post_build_hooks + end + + generate_bin + generate_plugins + + unless @options[:install_as_default] + write_spec + write_cache_file + end + + File.chmod(dir_mode, gem_dir) if dir_mode + + say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? + + Gem::Specification.add_spec(spec) unless @install_dir + + load_plugin + + run_post_install_hooks + + spec + rescue Errno::EACCES => e + # Permission denied - /path/to/foo + raise Gem::FilePermissionError, e.message.split(" - ").last + end - class UnpackedInstaller < GemInstaller def write_cache_file end @@ -850,11 +803,6 @@ module RbInstall super end - def generate_bin_script(filename, bindir) - return if same_bin_script?(filename, bindir) - super - end - def same_bin_script?(filename, bindir) path = File.join(bindir, formatted_program_filename(filename)) begin @@ -873,12 +821,11 @@ module RbInstall super unless $dryrun $installed_list.puts(without_destdir(default_spec_file)) if $installed_list end - end - class GemInstaller def install spec.post_install_message = nil - RbInstall.no_write(options) {super} + dir_creating(without_destdir(gem_dir)) + RbInstall.no_write(options) { install_with_default_gem } end # Now build-ext builds all extensions including bundled gems. @@ -886,10 +833,13 @@ module RbInstall end def generate_bin_script(filename, bindir) + return if same_bin_script?(filename, bindir) name = formatted_program_filename(filename) unless $dryrun super - File.chmod($script_mode, File.join(bindir, name)) + script = File.join(bindir, name) + File.chmod($script_mode, script) + File.unlink("#{script}.lock") rescue nil end $installed_list.puts(File.join(without_destdir(bindir), name)) if $installed_list end @@ -904,38 +854,45 @@ module RbInstall $installed_list.puts(d+"/") if $installed_list end end - end -end -# :startdoc: - -install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do - install_default_gem('lib', srcdir, bindir) -end -install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do - install_default_gem('ext', srcdir, bindir) + def load_plugin + # Suppress warnings for constant re-assignment + verbose, $VERBOSE = $VERBOSE, nil + super + ensure + $VERBOSE = verbose + end + end end -def load_gemspec(file, base = nil) +def load_gemspec(file, base = nil, files: nil) file = File.realpath(file) code = File.read(file, encoding: "utf-8:-") - code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split\([^\)]*\)/m) do - files = [] - if base - Dir.glob("**/*", File::FNM_DOTMATCH, base: base) do |n| - case File.basename(n); when ".", ".."; next; end - next if File.directory?(File.join(base, n)) - files << n.dump - end - end - "[" + files.join(", ") + "]" + + code.gsub!(/^ *#.*/, "") + spec_files = files ? files.map(&:dump).join(", ") : "" + code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split(\([^\)]*\))?/m) do + "[" + spec_files + "]" + end \ + or + code.gsub!(/IO\.popen\(.*git.*?\)/) do + "[" + spec_files + "] || itself" end + spec = eval(code, binding, file) + # for out-of-place build + collected_files = files ? spec.files.concat(files).uniq : spec.files + spec.files = collected_files.map do |f| + if !File.exist?(File.join(base || ".", f)) && f.end_with?(".rb") + "lib/#{f}" + else + f + end + end unless Gem::Specification === spec raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." end spec.loaded_from = base ? File.join(base, File.basename(file)) : file - spec.files.reject! {|n| n.end_with?(".gemspec") or n.start_with?(".git")} spec.date = RUBY_RELEASE_DATE spec @@ -946,6 +903,7 @@ def install_default_gem(dir, srcdir, bindir) install_dir = with_destdir(gem_dir) prepare "default gems from #{dir}", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_default_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -964,14 +922,11 @@ def install_default_gem(dir, srcdir, bindir) base = "#{srcdir}/#{dir}" gems = Dir.glob("**/*.gemspec", base: base).map {|src| - spec = load_gemspec("#{base}/#{src}") - file_collector = RbInstall::Specs::FileCollector.for(srcdir, dir, src) - files = file_collector.collect - if file_collector.skip_install?(files) + files = RbInstall::Specs::FileCollector.for(srcdir, dir, src).collect + if files.empty? next end - spec.files = files - spec + load_gemspec("#{base}/#{src}", files: files) } gems.compact.sort_by(&:name).each do |gemspec| old_gemspecs = Dir[File.join(with_destdir(default_spec_dir), "#{gemspec.name}-*.gemspec")] @@ -990,11 +945,259 @@ def install_default_gem(dir, srcdir, bindir) end end +def mdoc_file?(mdoc) + /^\.Nm / =~ File.read(mdoc, 1024) +end + +# :startdoc: + +install?(:local, :arch, :bin, :'bin-arch') do + prepare "binary commands", (dest = archbindir || bindir) + + def (bins = []).add(name) + push(name) + name + end + + install bins.add(ruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + if rubyw_install_name and !rubyw_install_name.empty? + install bins.add(rubyw_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support + # to load and execute the second .wasm file. Both are required to execute ruby + if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" + install bins.add(ruby_install_name+".wasm"), dest, :mode => $prog_mode, :strip => $strip + end + if File.exist? goruby_install_name+exeext + install bins.add(goruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + if enable_shared and dll != lib + install bins.add(dll), dest, :mode => $prog_mode, :strip => $strip + end + if archbindir + prepare "binary command links", bindir + relpath = Path.relative(archbindir, bindir) + bins.each do |f| + ln_sf(File.join(relpath, f), File.join(bindir, f)) + end + end +end + +install?(:local, :arch, :lib, :'lib-arch') do + prepare "base libraries", libdir + + install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc + install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" + if dll == lib and dll != arc + for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] + ln_sf(dll, File.join(libdir, link)) + end + end + + prepare "arch files", archlibdir + install "rbconfig.rb", archlibdir, :mode => $data_mode + if CONFIG["ARCHFILE"] + for file in CONFIG["ARCHFILE"].split + install file, archlibdir, :mode => $data_mode + end + end +end + +install?(:local, :arch, :data) do + pc = CONFIG["ruby_pc"] + if pc and File.file?(pc) and File.size?(pc) + prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") + install pc, pkgconfigdir, :mode => $data_mode + if (pkgconfig_base = CONFIG["libdir", true]) != libdir + prepare "pkgconfig data link", File.join(pkgconfig_base, "pkgconfig") + ln_sf(File.join("..", Path.relative(pkgconfigdir, pkgconfig_base), pc), + File.join(pkgconfig_base, "pkgconfig", pc)) + end + end +end + +install?(:ext, :arch, :'ext-arch') do + prepare "extension objects", archlibdir + noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split + install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) + prepare "extension objects", sitearchlibdir + prepare "extension objects", vendorarchlibdir + if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and + !(extso = extso.gsub(/\\\n/, '').split).empty? + libpathenv = CONFIG["LIBPATHENV"] + dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] + prepare "external libraries", dest + for file in extso + install file, dest, :mode => $prog_mode + end + end +end + +install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do + prepare "extension headers", archhdrdir + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:ext, :comm, :'ext-comm') do + prepare "extension scripts", rubylibdir + install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) + prepare "extension scripts", sitelibdir + prepare "extension scripts", vendorlibdir +end + +install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do + hdrdir = rubyhdrdir + "/ruby" + prepare "extension headers", hdrdir + install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:doc, :rdoc) do + if $rdocdir + ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") + prepare "rdoc", ridatadir + install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :html) do + if $htmldir + prepare "html-docs", docdir + install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :capi) do + prepare "capi-docs", docdir + install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode +end + +install?(:local, :comm, :bin, :'bin-comm') do + prepare "command scripts", bindir + + install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| + $script_installer.install(src, cmd) + end +end + +install?(:local, :comm, :lib) do + prepare "library scripts", rubylibdir + noinst = %w[*.txt *.rdoc *.gemspec] + install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) +end + +install?(:local, :comm, :hdr, :'comm-hdr') do + prepare "common headers", rubyhdrdir + + noinst = [] + unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ + noinst << "win32.h" + end + noinst = nil if noinst.empty? + install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) +end + +install?(:local, :comm, :man) do + mdocs = Dir["#{srcdir}/man/*.[1-9]"] + prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} + + mantype, suffix, compress = Compressors.for($mantype) + has_goruby = File.exist?(goruby_install_name+exeext) + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ mantype + mdocs.each do |mdoc| + next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' + base = File.basename(mdoc) + if base == "goruby.1" + next unless has_goruby + end + + destdir = File.join(mandir, "man" + (section = mdoc[/\d+$/])) + destname = $script_installer.transform(base.chomp(".#{section}")) + destfile = File.join(destdir, "#{destname}.#{section}") + + if /\Adoc\b/ =~ mantype or !mdoc_file?(mdoc) + if compress + begin + w = IO.popen(compress, "rb", in: mdoc, &:read) + rescue + else + destfile << suffix + end + end + if w + open_for_install(destfile, $data_mode) {w} + else + install mdoc, destfile, :mode => $data_mode + end + else + class << (w = []) + alias print push + end + File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + w = w.join("") + if compress + begin + w = IO.popen(compress, "r+b") do |f| + Thread.start {f.write w; f.close_write} + f.read + end + rescue + else + destfile << suffix + end + end + open_for_install(destfile, $data_mode) {w} + end + end +end + +install?(:dbg, :nodefault) do + prepare "debugger commands", bindir + prepare "debugger scripts", rubylibdir + conf = MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) + Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| + cmd = $script_installer.transform(File.basename(src, ".in")) + open_for_install(File.join(bindir, cmd), $script_mode) { + RbConfig.expand(File.read(src), conf) + } + end + Dir.glob(File.join(srcdir, "misc/lldb_*")) do |src| + if File.directory?(src) + install_recursive src, File.join(rubylibdir, File.basename(src)) + else + install src, rubylibdir + end + end + install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") + if $debug_symbols + { + ruby_install_name => archbindir || bindir, + rubyw_install_name => archbindir || bindir, + goruby_install_name => archbindir || bindir, + dll => libdir, + }.each do |src, dest| + next if src.empty? + src += $debug_symbols + if File.directory?(src) + install_recursive src, File.join(dest, src) + end + end + end +end + +install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do + install_default_gem('lib', srcdir, bindir) +end + +install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do + install_default_gem('ext', srcdir, bindir) +end + install?(:ext, :comm, :gem, :'bundled-gems') do gem_dir = Gem.default_dir install_dir = with_destdir(gem_dir) prepare "bundled gems", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -1023,22 +1226,30 @@ install?(:ext, :comm, :gem, :'bundled-gems') do # the newly installed ruby. ENV.delete('RUBYOPT') + collector = RbInstall::Specs::FileCollector::UnpackedGem File.foreach("#{srcdir}/gems/bundled_gems") do |name| next if /^\s*(?:#|$)/ =~ name next unless /^(\S+)\s+(\S+).*/ =~ name + gem = $1 gem_name = "#$1-#$2" - # Try to find the gemspec file for C ext gems - # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec - # This gemspec keep the original dependencies - path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec" - unless File.exist?(path) - path = "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" - unless File.exist?(path) - skipped[gem_name] = "gemspec not found" - next - end + path = [ + # gemspec that removed duplicated dependencies of bundled gems + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec", + # gemspec for C ext gems, It has the original dependencies + # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec", + # original gemspec generated by rubygems + "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + ].find { |gemspec| File.exist?(gemspec) } + if path.nil? + skipped[gem_name] = "gemspec not found" + next end - spec = load_gemspec(path, "#{srcdir}/.bundle/gems/#{gem_name}") + base = "#{srcdir}/.bundle/gems/#{gem_name}" + files = collector.new(path, base, nil).collect + files.delete("#{gem}.gemspec") + files.delete("#{gem_name}.gemspec") + spec = load_gemspec(path, base, files: files) unless spec.platform == Gem::Platform::RUBY skipped[gem_name] = "not ruby platform (#{spec.platform})" next @@ -1047,7 +1258,13 @@ install?(:ext, :comm, :gem, :'bundled-gems') do skipped[gem_name] = "full name unmatch #{spec.full_name}" next end + # Skip install C ext bundled gem if it is build failed or not found + if !spec.extensions.empty? && !File.exist?("#{build_dir}/#{gem_name}/gem.build_complete") + skipped[gem_name] = "extensions not found or build failed #{spec.full_name}" + next + end spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + package = RbInstall::DirPackage.new spec ins = RbInstall::UnpackedInstaller.new(package, options) puts "#{INDENT}#{spec.name} #{spec.version}" @@ -1067,11 +1284,21 @@ install?(:ext, :comm, :gem, :'bundled-gems') do skipped.default = "not found in bundled_gems" puts "skipped bundled gems:" gems.each do |gem| - printf " %-32s%s\n", File.basename(gem), skipped[gem] + gem = File.basename(gem) + printf " %-31s %s\n", gem, skipped[gem.chomp(".gem")] end end end +install?('modular-gc') do + if modular_gc_dir = CONFIG['modular_gc_dir'] and !modular_gc_dir.empty? + dlext = CONFIG['DLEXT', true] + modular_gc_dir = File.expand_path(modular_gc_dir, CONFIG['prefix']) + prepare "modular GC library", modular_gc_dir + install Dir.glob("gc/*/librubygc.*.#{dlext}"), modular_gc_dir + end +end + parse_args() include FileUtils @@ -1088,7 +1315,6 @@ installs = $install.map do |inst| end installs.flatten! installs -= $exclude.map {|exc| $install_procs[exc]}.flatten -puts "Installing to #$destdir" unless installs.empty? installs.each do |block| dir = Dir.pwd begin @@ -1097,5 +1323,9 @@ installs.each do |block| Dir.chdir(dir) end end +unless installs.empty? or $destdir.empty? + require_relative 'lib/colorize' + puts "Installed under #{Colorize.new.info($destdir)}" +end # vi:set sw=2: diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests index c860ac3b45..4bcb5707a5 100644 --- a/tool/rbs_skip_tests +++ b/tool/rbs_skip_tests @@ -15,44 +15,43 @@ # $(test-class-name) ` ` $(optional comment) # Skipping a test class # -test_replicate(EncodingTest) the method was removed in 3.3 +## Failed tests because of testing environment test_collection_install(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__stdlib_source(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__dependency_no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__rbs_dependency_and__gem_dependency(RBS::CliTest) running tests without Bundler test_collection_install_frozen(RBS::CliTest) running tests without Bundler test_collection_install_gemspec(RBS::CliTest) running tests without Bundler test_collection_update(RBS::CliTest) running tests without Bundler -test_defs(RBS::RbPrototypeTest) Numeric Nodes are added -test_defs_return_type(RBS::RbPrototypeTest) Numeric Nodes are added -test_defs_return_type_with_block(RBS::RbPrototypeTest) Numeric Nodes are added -test_defs_return_type_with_if(RBS::RbPrototypeTest) Numeric Nodes are added -test_endless_method_definition(RBS::RbPrototypeTest) Numeric Nodes are added -test_literal_to_type(RBS::RbPrototypeTest) Numeric Nodes are added -test_literal_types(RBS::RbPrototypeTest) Numeric Nodes are added -test_accessibility(RBS::RbPrototypeTest) Symbol Node is added -test_aliases(RBS::RbPrototypeTest) Symbol Node is added -test_comments(RBS::RbPrototypeTest) Symbol Node is added -test_const(RBS::RbPrototypeTest) Symbol Node is added -test_meta_programming(RBS::RbPrototypeTest) Symbol Node is added -test_module_function(RBS::RbPrototypeTest) Symbol Node is added -test_all(RBS::RbiPrototypeTest) Symbol Node is added -test_block_args(RBS::RbiPrototypeTest) Symbol Node is added -test_implicit_block(RBS::RbiPrototypeTest) Symbol Node is added -test_non_parameter_type_member(RBS::RbiPrototypeTest) Symbol Node is added -test_noreturn(RBS::RbiPrototypeTest) Symbol Node is added -test_optional_block(RBS::RbiPrototypeTest) Symbol Node is added -test_overloading(RBS::RbiPrototypeTest) Symbol Node is added -test_parameter(RBS::RbiPrototypeTest) Symbol Node is added -test_parameter_type_member_variance(RBS::RbiPrototypeTest) Symbol Node is added -test_tuple(RBS::RbiPrototypeTest) Symbol Node is added -test_untyped_block(RBS::RbiPrototypeTest) Symbol Node is added - -test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` - -test_aref(FiberSingletonTest) the method should not accept String keys - NetSingletonTest depending on external resources NetInstanceTest depending on external resources TestHTTPRequest depending on external resources TestSingletonNetHTTPResponse depending on external resources TestInstanceNetHTTPResponse depending on external resources + +test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` + +# Failing because ObjectSpace.count_nodes has been removed +test_count_nodes(ObjectSpaceTest) + +## Unknown failures + +# NoMethodError: undefined method 'inspect' for an instance of RBS::UnitTest::Convertibles::ToInt +test_compile(RegexpSingletonTest) +test_linear_time?(RegexpSingletonTest) +test_new(RegexpSingletonTest) + +## Failed tests caused by unreleased version of Ruby +test_source_location(MethodInstanceTest) +test_source_location(ProcInstanceTest) +test_source_location(UnboundMethodInstanceTest) + +# Errno::ENOENT: No such file or directory - bundle +test_collection_install__pathname_set(RBS::CliTest) +test_collection_install__set_pathname__manifest(RBS::CliTest) diff --git a/tool/rbs_skip_tests_windows b/tool/rbs_skip_tests_windows new file mode 100644 index 0000000000..db12c69419 --- /dev/null +++ b/tool/rbs_skip_tests_windows @@ -0,0 +1,111 @@ +ARGFTest Failing on Windows + +RactorSingletonTest Hangs up on Windows +RactorInstanceTest Hangs up on Windows + +# NotImplementedError: fileno() function is unimplemented on this machine +test_fileno(DirInstanceTest) +test_fchdir(DirSingletonTest) +test_for_fd(DirSingletonTest) + +# ArgumentError: user root doesn't exist +test_home(DirSingletonTest) + +# NameError: uninitialized constant Etc::CS_PATH +test_confstr(EtcSingletonTest) + +# NameError: uninitialized constant Etc::SC_ARG_MAX +test_sysconf(EtcSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-udw6rx/chmod +test_chmod(FileInstanceTest) +test_chmod(FileInstanceTest) +test_truncate(FileInstanceTest) + +# Errno::EISDIR: Is a directory @ rb_sysopen - C:/a/ruby/ruby/src/gems/src/rbs/test/stdlib +test_directory?(FileSingletonTest) + +# NotImplementedError: lutime() function is unimplemented on this machine +test_lutime(FileSingletonTest) + +# NotImplementedError: mkfifo() function is unimplemented on this machine +test_mkfifo(FileSingletonTest) + +# Returns `nil` on Windows +test_getgrgid(EtcSingletonTest) +test_getgrnam(EtcSingletonTest) +test_getpwnam(EtcSingletonTest) +test_getpwuid(EtcSingletonTest) + +# Returns `false` +test_setgid?(FileSingletonTest) +test_setuid?(FileSingletonTest) +test_sticky?(FileSingletonTest) + +test_world_readable?(FileSingletonTest) # Returns `420` +test_world_readable?(FileStatInstanceTest) # Returns `420` +test_world_writable?(FileSingletonTest) # Returns `nil` +test_dev_major(FileStatInstanceTest) # Returns `nil` +test_dev_minor(FileStatInstanceTest) # Returns `nil` +test_rdev_major(FileStatInstanceTest) # Returns `nil` +test_rdev_minor(FileStatInstanceTest) # Returns `nil` + +# ArgumentError: wrong number of arguments (given -403772944, expected 0+) +test_curry(MethodInstanceTest) + +# ArgumentError: no output encoding given +test_tolocale(KconvSingletonTest) + +# Errno::EINVAL: Invalid argument - : +test_system(KernelInstanceTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_load(OpenSSLConfigSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - +test_parse(OpenSSLConfigSingletonTest) +test_parse_config(OpenSSLConfigSingletonTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_each(OpenSSLConfigTest) +test_lookup_and_set(OpenSSLConfigTest) +test_sections(OpenSSLConfigTest) + +# OpenSSL::SSL::SSLError: SSL_connect returned=1 errno=0 peeraddr=185.199.108.153:443 state=error: certificate verify failed (unable to get local issuer certificate) +test_URI_open(OpenURISingletonTest) + +# ArgumentError: both textmode and binmode specified +test_binwrite(PathnameInstanceTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-delete-test20250813-10156-mb3e9i +test_delete(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-binwrite-test20250813-10156-sh8145 +test_open(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_truncate(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_unlink(PathnameInstanceTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /etc/resolv.conf +test_parse_resolv_conf(ResolvDNSConfigSingletonTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvInstanceTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvSingletonTest) + +# ArgumentError: unsupported signal 'SIGUSR2' +test_trap(SignalSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-mgr4tx +test_create(TempfileSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-hp9nzu +test_initialize(TempfileSingletonTest) +test_new(TempfileSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-f8z9pn/test.gz +test_open(ZlibGzipReaderSingletonTest) + +# Errno::EACCES: Permission denied @ rb_file_s_rename +# D:/a/ruby/ruby/src/lib/rubygems/util/atomic_file_writer.rb:42:in 'File.rename' +test_write_binary(GemSingletonTest) diff --git a/tool/rbuninstall.rb b/tool/rbuninstall.rb index f0c286012c..60f5241a4f 100755 --- a/tool/rbuninstall.rb +++ b/tool/rbuninstall.rb @@ -21,15 +21,33 @@ BEGIN { end $dirs = [] $files = [] + COLUMNS = $tty && (ENV["COLUMNS"]&.to_i || begin require 'io/console/size'; rescue; else IO.console_size&.at(1); end)&.then do |n| + n-1 if n > 1 + end + if COLUMNS + $column = 0 + def message(str = nil) + $stdout.print "\b \b" * $column + if str + if str.size > COLUMNS + str = "..." + str[(-COLUMNS+3)..-1] + end + $stdout.print str + end + $stdout.flush + $column = str&.size || 0 + end + else + alias message puts + end } list = ($_.chomp!('/') ? $dirs : $files) list << $_ END { status = true $\ = nil - ors = (!$dryrun and $tty) ? "\e[K\r" : "\n" $files.each do |file| - print "rm #{file}#{ors}" + message "rm #{file}" unless $dryrun file = File.join($destdir, file) if $destdir begin @@ -45,9 +63,10 @@ END { $dirs.each do |dir| unlink[dir] = true end + nonempty = {} while dir = $dirs.pop dir = File.dirname(dir) while File.basename(dir) == '.' - print "rmdir #{dir}#{ors}" + message "rmdir #{dir}" unless $dryrun realdir = $destdir ? File.join($destdir, dir) : dir begin @@ -58,16 +77,23 @@ END { raise unless File.symlink?(realdir) File.unlink(realdir) end - rescue Errno::ENOENT, Errno::ENOTEMPTY + rescue Errno::ENOTEMPTY + nonempty[dir] = true + rescue Errno::ENOENT rescue status = false puts $! else + nonempty.delete(dir) parent = File.dirname(dir) $dirs.push(parent) unless parent == dir or unlink[parent] end end end - print ors.chomp + message + unless nonempty.empty? + puts "Non empty director#{nonempty.size == 1 ? 'y' : 'ies'}:" + nonempty.each_key {|dir| print " #{dir}\n"} + end exit(status) } diff --git a/tool/rdoc-srcdir b/tool/rdoc-srcdir index 10c63caf9e..ecc49b4b2c 100644..100755 --- a/tool/rdoc-srcdir +++ b/tool/rdoc-srcdir @@ -1,5 +1,9 @@ -#!ruby +#!ruby -W0 +%w[tsort rdoc].each do |lib| + path = Dir.glob("#{File.dirname(__dir__)}/.bundle/gems/#{lib}-*").first + $LOAD_PATH.unshift("#{path}/lib") +end require 'rdoc/rdoc' # Make only the output directory relative to the invoked directory. @@ -9,7 +13,13 @@ invoked = Dir.pwd Dir.chdir(File.dirname(__dir__)) options = RDoc::Options.load_options -options.parse ARGV +options.title = options.title.sub(/Ruby \K.*version/) { + File.read("include/ruby/version.h") + .scan(/^ *# *define +RUBY_API_VERSION_(MAJOR|MINOR) +(\d+)/) + .sort # "MAJOR" < "MINOR", fortunately + .to_h.values.join(".") +} +options.parse ARGV + ["#{invoked}/rbconfig.rb"] options.singleton_class.define_method(:finish) do super() diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index 44b44920d4..95a9688cb2 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -10,13 +10,7 @@ require 'optparse' require 'abbrev' require 'pp' require 'shellwords' -begin - require 'readline' -rescue LoadError - module Readline; end -end - -VERSION = '0.0.1' +require 'reline' opts = OptionParser.new target_version = nil @@ -24,10 +18,9 @@ repo_path = nil api_key = nil ssl_verify = true opts.on('-k REDMINE_API_KEY', '--key=REDMINE_API_KEY', 'specify your REDMINE_API_KEY') {|v| api_key = v} -opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 2.1)') {|v| target_version = v} +opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 3.1)') {|v| target_version = v} opts.on('-r RUBY_REPO_PATH', '--repository=RUBY_REPO_PATH', 'specify repository path') {|v| repo_path = v} opts.on('--[no-]ssl-verify', TrueClass, 'use / not use SSL verify') {|v| ssl_verify = v} -opts.version = VERSION opts.parse!(ARGV) http_options = {use_ssl: true} @@ -35,11 +28,11 @@ http_options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify $openuri_options = {} $openuri_options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify -TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (raise 'need to specify TARGET_VERSION') +TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (puts opts.help; raise 'need to specify TARGET_VERSION') RUBY_REPO_PATH = repo_path || ENV['RUBY_REPO_PATH'] BACKPORT_CF_KEY = 'cf_5' STATUS_CLOSE = 5 -REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (raise 'need to specify REDMINE_API_KEY') +REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (puts opts.help; raise 'need to specify REDMINE_API_KEY') REDMINE_BASE = 'https://bugs.ruby-lang.org' @query = { @@ -70,28 +63,28 @@ COLORS = { } class String - def color(fore=nil, back=nil, bold: false, underscore: false) + def color(fore=nil, back=nil, opts={}, bold: false, underscore: false) seq = "" - if bold - seq << "\e[1m" + if bold || opts[:bold] + seq = seq + "\e[1m" end - if underscore - seq << "\e[2m" + if underscore || opts[:underscore] + seq = seq + "\e[2m" end if fore c = COLORS[fore] raise "unknown foreground color #{fore}" unless c - seq << "\e[#{c}m" + seq = seq + "\e[#{c}m" end if back c = COLORS[back] raise "unknown background color #{back}" unless c - seq << "\e[#{c + 10}m" + seq = seq + "\e[#{c + 10}m" end if seq.empty? self else - seq << self << "\e[0m" + seq = seq + self + "\e[0m" end end end @@ -162,84 +155,6 @@ def more(sio) end end -class << Readline - def readline(prompt = '') - console = IO.console - console.binmode - _, lx = console.winsize - if /mswin|mingw/ =~ RUBY_PLATFORM or /^(?:vt\d\d\d|xterm)/i =~ ENV["TERM"] - cls = "\r\e[2K" - else - cls = "\r" << (" " * lx) - end - cls << "\r" << prompt - console.print prompt - console.flush - line = '' - while true - case c = console.getch - when "\r", "\n" - puts - HISTORY << line - return line - when "\C-?", "\b" # DEL/BS - print "\b \b" if line.chop! - when "\C-u" - print cls - line.clear - when "\C-d" - return nil if line.empty? - line << c - when "\C-p" - HISTORY.pos -= 1 - line = HISTORY.current - print cls - print line - when "\C-n" - HISTORY.pos += 1 - line = HISTORY.current - print cls - print line - else - if c >= " " - print c - line << c - end - end - end - end - - HISTORY = [] - def HISTORY.<<(val) - HISTORY.push(val) - @pos = self.size - self - end - def HISTORY.pos - @pos ||= 0 - end - def HISTORY.pos=(val) - @pos = val - if @pos < 0 - @pos = -1 - elsif @pos >= self.size - @pos = self.size - end - end - def HISTORY.current - @pos ||= 0 - if @pos < 0 || @pos >= self.size - '' - else - self[@pos] - end - end -end unless defined?(Readline.readline) - -def find_svn_log(pattern) - `svn log --xml --stop-on-copy --search="#{pattern}" #{RUBY_REPO_PATH}` -end - def find_git_log(pattern) `git #{RUBY_REPO_PATH ? "-C #{RUBY_REPO_PATH.shellescape}" : ""} log --grep="#{pattern}"` end @@ -275,11 +190,13 @@ def backport_command_string next false if c.match(/\A\d{1,6}\z/) # skip SVN revision # check if the Git revision is included in master - has_commit(c, "master") + has_commit(c, "origin/master") + end.sort_by do |changeset| + Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) end @changesets.define_singleton_method(:validated){true} end - " #{merger_path} --ticket=#{@issue} #{@changesets.sort.join(',')}" + "#{merger_path} --ticket=#{@issue} #{@changesets.join(',')}" end def status_char(obj) @@ -294,7 +211,7 @@ end console = IO.console row, = console.winsize @query['limit'] = row - 2 -puts "Backporter #{VERSION}".color(bold: true) + " for #{TARGET_VERSION}" +puts "Redmine Backporter".color(bold: true) + " for Ruby #{TARGET_VERSION}" class CommandSyntaxError < RuntimeError; end commands = { @@ -306,10 +223,11 @@ commands = { @issues = issues = res["issues"] from = res["offset"] + 1 total = res["total_count"] + closed = issues.count { |x, _| x["status"]["name"] == "Closed" } to = from + issues.size - 1 - puts "#{from}-#{to} / #{total}" + puts "#{from}-#{to} / #{total} (closed: #{closed})" issues.each_with_index do |x, i| - id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]]) + id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]], bold: x["status"]["name"] == "Closed") puts "#{'%2d' % i} #{id} #{x["priority"]["name"][0]} #{status_char(x["status"])} #{x["subject"][0,80]}" end }, @@ -376,9 +294,6 @@ eom "rel" => proc{|args| # this feature requires custom redmine which allows add_related_issue API case args - when /\Ar?(\d+)\z/ # SVN - rev = $1 - uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/trunk/revisions/#{rev}/issues.json") when /\A\h{7,40}\z/ # Git rev = args uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/git/revisions/#{rev}/issues.json") @@ -436,33 +351,22 @@ eom next end - if rev - elsif system("svn info #{RUBY_REPO_PATH&.shellescape}", %i(out err) => IO::NULL) # SVN - if (log = find_svn_log("##@issue]")) && (/revision="(?<rev>\d+)/ =~ log) - rev = "r#{rev}" - end - else # Git - if log = find_git_log("##@issue]") - /^commit (?<rev>\h{40})$/ =~ log - end - end - if log && rev - str = log[/merge revision\(s\) ([^:]+)(?=:)/] - if str - str.insert(5, "d") - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev} #{str}." + if rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") + notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." + elsif rev.nil? && (log = find_git_log("##@issue]")) && !(revs = log.scan(/^commit (\h{40})$/).flatten).empty? + commits = revs.map { |rev| "commit:#{rev}" }.join(", ") + if merged_revs = log[/merge revision\(s\) ([^:]+)(?=:)/] + merged_revs.sub!(/\Amerge/, 'merged') + merged_revs.gsub!(/\h{8,40}/, 'commit:\0') + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits} #{merged_revs}." else - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev}." + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits}." end if notes str << "\n" str << notes end notes = str - elsif rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") - # Backport commit's log doesn't have the issue number. - # Instead of that manually it's provided. - notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." else puts "no commit is found whose log include ##@issue" next @@ -571,7 +475,7 @@ list = Abbrev.abbrev(commands.keys) @changesets = nil while true begin - l = Readline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " + l = Reline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " rescue Interrupt break end diff --git a/tool/release.sh b/tool/release.sh index 0988dc2a67..d467d8f24b 100755 --- a/tool/release.sh +++ b/tool/release.sh @@ -1,7 +1,15 @@ #!/bin/bash # Bash version 3.2+ is required for regexp +# Usage: +# tool/release.sh 3.0.0 +# tool/release.sh 3.0.0-rc1 -EXTS='.tar.gz .tar.bz2 .tar.xz .zip' +EXTS='.tar.gz .tar.xz .zip' +if [[ -n $AWS_ACCESS_KEY_ID ]]; then + AWS_CLI_OPTS="" +else + AWS_CLI_OPTS="--profile ruby" +fi ver=$1 if [[ $ver =~ ^([1-9]\.[0-9])\.([0-9]|[1-9][0-9]|0-(preview[1-9]|rc[1-9]))$ ]]; then @@ -15,5 +23,5 @@ short=${BASH_REMATCH[1]} echo $ver echo $short for ext in $EXTS; do - aws --profile ruby s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext + aws $AWS_CLI_OPTS s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext done diff --git a/tool/releng/gen-mail.rb b/tool/releng/gen-mail.rb index 6dc0e4cec1..17fa499d69 100755 --- a/tool/releng/gen-mail.rb +++ b/tool/releng/gen-mail.rb @@ -10,7 +10,7 @@ end # Confirm current directory is www.ruby-lang.org's working directory def confirm_w_r_l_o_wd File.foreach('.git/config') do |line| - return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + return true if line.include?('ruby/www.ruby-lang.org.git') end abort "Run this script in www.ruby-lang.org's working directory" end diff --git a/tool/releng/update-www-meta.rb b/tool/releng/update-www-meta.rb index 8a5651dcd0..0dd5b25631 100755 --- a/tool/releng/update-www-meta.rb +++ b/tool/releng/update-www-meta.rb @@ -1,6 +1,7 @@ #!/usr/bin/env ruby require "open-uri" require "yaml" +require_relative "../ruby-version" class Tarball attr_reader :version, :size, :sha1, :sha256, :sha512 @@ -41,22 +42,7 @@ eom unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" info = YAML.load(URI(uri).read) @@ -74,9 +60,10 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) rubydir = File.expand_path(File.join(__FILE__, '../../../')) puts %`git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` @@ -155,7 +142,7 @@ eom date = Time.now.utc # use utc to use previous day in midnight entry = <<eom - version: #{ver} - tag: v#{ver.tr('-.', '_')} + tag: #{RubyVersion.tag(ver)} date: #{date.strftime("%Y-%m-%d")} post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ stats: diff --git a/tool/rjit/bindgen.rb b/tool/rjit/bindgen.rb deleted file mode 100755 index bf33d92bd2..0000000000 --- a/tool/rjit/bindgen.rb +++ /dev/null @@ -1,663 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -ENV['GEM_HOME'] = File.expand_path('./.bundle', __dir__) -require 'rubygems/source' -require 'bundler/inline' -gemfile(true) do - source 'https://rubygems.org' - gem 'ffi-clang', '0.7.0', require: false -end - -# Help ffi-clang find libclang -# Hint: apt install libclang1 -ENV['LIBCLANG'] ||= Dir.glob("/usr/lib/llvm-*/lib/libclang.so.1").grep_v(/-cpp/).sort.last -require 'ffi/clang' - -require 'etc' -require 'fiddle/import' -require 'set' - -unless build_dir = ARGV.first - abort "Usage: #{$0} BUILD_DIR" -end - -class Node < Struct.new( - :kind, - :spelling, - :type, - :typedef_type, - :bitwidth, - :sizeof_type, - :offsetof, - :enum_value, - :children, - keyword_init: true, -) -end - -# Parse a C header with ffi-clang and return Node objects. -# To ease the maintenance, ffi-clang should be used only inside this class. -class HeaderParser - def initialize(header, cflags:) - @translation_unit = FFI::Clang::Index.new.parse_translation_unit(header, cflags, [], {}) - end - - def parse - parse_children(@translation_unit.cursor) - end - - private - - def parse_children(cursor) - children = [] - cursor.visit_children do |cursor, _parent| - children << parse_cursor(cursor) - next :continue - end - children - end - - def parse_cursor(cursor) - unless cursor.kind.start_with?('cursor_') - raise "unexpected cursor kind: #{cursor.kind}" - end - kind = cursor.kind.to_s.delete_prefix('cursor_').to_sym - children = parse_children(cursor) - - offsetof = {} - if kind == :struct - children.select { |c| c.kind == :field_decl }.each do |child| - offsetof[child.spelling] = cursor.type.offsetof(child.spelling) - end - end - - sizeof_type = nil - if %i[struct union].include?(kind) - sizeof_type = cursor.type.sizeof - end - - enum_value = nil - if kind == :enum_constant_decl - enum_value = cursor.enum_value - end - - Node.new( - kind: kind, - spelling: cursor.spelling, - type: cursor.type.spelling, - typedef_type: cursor.typedef_type.spelling, - bitwidth: cursor.bitwidth, - sizeof_type: sizeof_type, - offsetof: offsetof, - enum_value: enum_value, - children: children, - ) - end -end - -# Convert Node objects to a Ruby binding source. -class BindingGenerator - BINDGEN_BEG = '### RJIT bindgen begin ###' - BINDGEN_END = '### RJIT bindgen end ###' - DEFAULTS = { '_Bool' => 'CType::Bool.new' } - DEFAULTS.default_proc = proc { |_h, k| "CType::Stub.new(:#{k})" } - - attr_reader :src - - # @param src_path [String] - # @param consts [Hash{ Symbol => Array<String> }] - # @param values [Hash{ Symbol => Array<String> }] - # @param funcs [Array<String>] - # @param types [Array<String>] - # @param dynamic_types [Array<String>] #ifdef-dependent immediate types, which need Primitive.cexpr! for type detection - # @param skip_fields [Hash{ Symbol => Array<String> }] Struct fields that are skipped from bindgen - # @param ruby_fields [Hash{ Symbol => Array<String> }] Struct VALUE fields that are considered Ruby objects - def initialize(src_path:, consts:, values:, funcs:, types:, dynamic_types:, skip_fields:, ruby_fields:) - @preamble, @postamble = split_ambles(src_path) - @src = String.new - @consts = consts.transform_values(&:sort) - @values = values.transform_values(&:sort) - @funcs = funcs.sort - @types = types.sort - @dynamic_types = dynamic_types.sort - @skip_fields = skip_fields.transform_keys(&:to_s) - @ruby_fields = ruby_fields.transform_keys(&:to_s) - @references = Set.new - end - - def generate(nodes) - println @preamble - - # Define macros/enums - @consts.each do |type, values| - values.each do |value| - raise "#{value} isn't a valid constant name" unless ('A'..'Z').include?(value[0]) - println " C::#{value} = Primitive.cexpr! %q{ #{type}2NUM(#{value}) }" - end - end - println - - # Define variables - @values.each do |type, values| - values.each do |value| - println " def C.#{value}" - println " Primitive.cexpr! %q{ #{type}2NUM(#{value}) }" - println " end" - println - end - end - - # Define function pointers - @funcs.each do |func| - println " def C.#{func}" - println " Primitive.cexpr! %q{ SIZET2NUM((size_t)#{func}) }" - println " end" - println - end - - # Build a hash table for type lookup by name - nodes_index = flatten_nodes(nodes).group_by(&:spelling).transform_values do |values| - # Try to search a declaration with definitions - node_with_children = values.find { |v| !v.children.empty? } - next node_with_children if node_with_children - - # Otherwise, assume the last one is the main declaration - values.last - end - - # Define types - @types.each do |type| - unless definition = generate_node(nodes_index[type]) - raise "Failed to find or generate type: #{type}" - end - println " def C.#{type}" - println "@#{type} ||= #{definition}".gsub(/^/, " ").chomp - println " end" - println - end - - # Define dynamic types - @dynamic_types.each do |type| - unless generate_node(nodes_index[type])&.start_with?('CType::Immediate') - raise "Non-immediate type is given to dynamic_types: #{type}" - end - println " def C.#{type}" - println " @#{type} ||= CType::Immediate.find(Primitive.cexpr!(\"SIZEOF(#{type})\"), Primitive.cexpr!(\"SIGNED_TYPE_P(#{type})\"))" - println " end" - println - end - - # Leave a stub for types that are referenced but not targeted - (@references - @types - @dynamic_types).each do |type| - println " def C.#{type}" - println " #{DEFAULTS[type]}" - println " end" - println - end - - print @postamble - end - - private - - # Make an array that includes all top-level and nested nodes - def flatten_nodes(nodes) - result = [] - nodes.each do |node| - unless node.children.empty? - result.concat(flatten_nodes(node.children)) - end - end - result.concat(nodes) # prioritize top-level nodes - result - end - - # Return code before BINDGEN_BEG and code after BINDGEN_END - def split_ambles(src_path) - lines = File.read(src_path).lines - - preamble_end = lines.index { |l| l.include?(BINDGEN_BEG) } - raise "`#{BINDGEN_BEG}` was not found in '#{src_path}'" if preamble_end.nil? - - postamble_beg = lines.index { |l| l.include?(BINDGEN_END) } - raise "`#{BINDGEN_END}` was not found in '#{src_path}'" if postamble_beg.nil? - raise "`#{BINDGEN_BEG}` was found after `#{BINDGEN_END}`" if preamble_end >= postamble_beg - - return lines[0..preamble_end].join, lines[postamble_beg..-1].join - end - - # Generate code from a node. Used for constructing a complex nested node. - # @param node [Node] - def generate_node(node, sizeof_type: nil) - case node&.kind - when :struct, :union - # node.spelling is often empty for union, but we'd like to give it a name when it has one. - buf = +"CType::#{node.kind.to_s.sub(/\A[a-z]/, &:upcase)}.new(\n" - buf << " \"#{node.spelling}\", Primitive.cexpr!(\"SIZEOF(#{sizeof_type || node.type})\"),\n" - bit_fields_end = node.children.index { |c| c.bitwidth == -1 } || node.children.size # first non-bit field index - node.children.each_with_index do |child, i| - skip_type = sizeof_type&.gsub(/\(\(struct ([^\)]+) \*\)NULL\)->/, '\1.') || node.spelling - next if @skip_fields.fetch(skip_type, []).include?(child.spelling) - field_builder = proc do |field, type| - if node.kind == :struct - to_ruby = @ruby_fields.fetch(node.spelling, []).include?(field) - if child.bitwidth > 0 - if bit_fields_end <= i # give up offsetof calculation for non-leading bit fields - raise "non-leading bit fields are not supported. consider including '#{field}' in skip_fields." - end - offsetof = node.offsetof.fetch(field) - else - off_type = sizeof_type || "(*((#{node.type} *)NULL))" - offsetof = "Primitive.cexpr!(\"OFFSETOF(#{off_type}, #{field})\")" - end - " #{field}: [#{type}, #{offsetof}#{', true' if to_ruby}],\n" - else - " #{field}: #{type},\n" - end - end - - case child - # BitField is struct-specific. So it must be handled here. - in Node[kind: :field_decl, spelling:, bitwidth:, children: [_grandchild, *]] if bitwidth > 0 - buf << field_builder.call(spelling, "CType::BitField.new(#{bitwidth}, #{node.offsetof.fetch(spelling) % 8})") - # "(unnamed ...)" struct and union are handled here, which are also struct-specific. - in Node[kind: :field_decl, spelling:, type:, children: [grandchild]] if type.match?(/\((unnamed|anonymous) [^)]+\)\z/) - if sizeof_type - child_type = "#{sizeof_type}.#{child.spelling}" - else - child_type = "((#{node.type} *)NULL)->#{child.spelling}" - end - buf << field_builder.call(spelling, generate_node(grandchild, sizeof_type: child_type).gsub(/^/, ' ').sub(/\A +/, '')) - # In most cases, we'd like to let generate_type handle the type unless it's "(unnamed ...)". - in Node[kind: :field_decl, spelling:, type:] if !type.empty? - buf << field_builder.call(spelling, generate_type(type)) - else # forward declarations are ignored - end - end - buf << ")" - when :typedef_decl - case node.children - in [child] - generate_node(child) - in [child, Node[kind: :integer_literal]] - generate_node(child) - in _ unless node.typedef_type.empty? - generate_type(node.typedef_type) - end - when :enum_decl - generate_type('int') - when :type_ref - generate_type(node.spelling) - end - end - - # Generate code from a type name. Used for resolving the name of a simple leaf node. - # @param type [String] - def generate_type(type) - if type.match?(/\[\d+\]\z/) - return "CType::Array.new { #{generate_type(type.sub!(/\[\d+\]\z/, ''))} }" - end - type = type.delete_suffix('const') - if type.end_with?('*') - if type == 'const void *' - # `CType::Pointer.new { CType::Immediate.parse("void") }` is never useful, - # so specially handle that case here. - return 'CType::Immediate.parse("void *")' - end - return "CType::Pointer.new { #{generate_type(type.delete_suffix('*').rstrip)} }" - end - - type = type.gsub(/((const|volatile) )+/, '').rstrip - if type.start_with?(/(struct|union|enum) /) - target = type.split(' ', 2).last - push_target(target) - "self.#{target}" - else - begin - ctype = Fiddle::Importer.parse_ctype(type) - rescue Fiddle::DLError - push_target(type) - "self.#{type}" - else - # Convert any function pointers to void* to workaround FILE* vs int* - if ctype == Fiddle::TYPE_VOIDP - "CType::Immediate.parse(\"void *\")" - else - "CType::Immediate.parse(#{type.dump})" - end - end - end - end - - def print(str) - @src << str - end - - def println(str = "") - @src << str << "\n" - end - - def chomp - @src.delete_suffix!("\n") - end - - def rstrip! - @src.rstrip! - end - - def push_target(target) - unless target.match?(/\A\w+\z/) - raise "invalid target: #{target}" - end - @references << target - end -end - -src_dir = File.expand_path('../..', __dir__) -src_path = File.join(src_dir, 'rjit_c.rb') -build_dir = File.expand_path(build_dir) -cflags = [ - src_dir, - build_dir, - File.join(src_dir, 'include'), - File.join(build_dir, ".ext/include/#{RUBY_PLATFORM}"), -].map { |dir| "-I#{dir}" } - -# Clear .cache/clangd created by the language server, which could break this bindgen -clangd_cache = File.join(src_dir, '.cache/clangd') -if Dir.exist?(clangd_cache) - system('rm', '-rf', clangd_cache, exception: true) -end - -# Parse rjit_c.h and generate rjit_c.rb -nodes = HeaderParser.new(File.join(src_dir, 'rjit_c.h'), cflags: cflags).parse -generator = BindingGenerator.new( - src_path: src_path, - consts: { - LONG: %w[ - UNLIMITED_ARGUMENTS - VM_ENV_DATA_INDEX_ME_CREF - VM_ENV_DATA_INDEX_SPECVAL - ], - SIZET: %w[ - ARRAY_REDEFINED_OP_FLAG - BOP_AND - BOP_AREF - BOP_EQ - BOP_EQQ - BOP_FREEZE - BOP_GE - BOP_GT - BOP_LE - BOP_LT - BOP_MINUS - BOP_MOD - BOP_OR - BOP_PLUS - BUILTIN_ATTR_LEAF - BUILTIN_ATTR_NO_GC - HASH_REDEFINED_OP_FLAG - INTEGER_REDEFINED_OP_FLAG - INVALID_SHAPE_ID - METHOD_VISI_PRIVATE - METHOD_VISI_PROTECTED - METHOD_VISI_PUBLIC - METHOD_VISI_UNDEF - OBJ_TOO_COMPLEX_SHAPE_ID - OPTIMIZED_METHOD_TYPE_BLOCK_CALL - OPTIMIZED_METHOD_TYPE_CALL - OPTIMIZED_METHOD_TYPE_SEND - OPTIMIZED_METHOD_TYPE_STRUCT_AREF - OPTIMIZED_METHOD_TYPE_STRUCT_ASET - RARRAY_EMBED_FLAG - RARRAY_EMBED_LEN_MASK - RARRAY_EMBED_LEN_SHIFT - RMODULE_IS_REFINEMENT - ROBJECT_EMBED - RSTRUCT_EMBED_LEN_MASK - RUBY_EVENT_CLASS - RUBY_EVENT_C_CALL - RUBY_EVENT_C_RETURN - RUBY_FIXNUM_FLAG - RUBY_FLONUM_FLAG - RUBY_FLONUM_MASK - RUBY_FL_SINGLETON - RUBY_IMMEDIATE_MASK - RUBY_SPECIAL_SHIFT - RUBY_SYMBOL_FLAG - RUBY_T_ARRAY - RUBY_T_CLASS - RUBY_T_ICLASS - RUBY_T_HASH - RUBY_T_MASK - RUBY_T_MODULE - RUBY_T_STRING - RUBY_T_SYMBOL - RUBY_T_OBJECT - SHAPE_FLAG_SHIFT - SHAPE_FROZEN - SHAPE_ID_NUM_BITS - SHAPE_IVAR - SHAPE_MASK - SHAPE_ROOT - STRING_REDEFINED_OP_FLAG - T_OBJECT - VM_BLOCK_HANDLER_NONE - VM_CALL_ARGS_BLOCKARG - VM_CALL_ARGS_SPLAT - VM_CALL_FCALL - VM_CALL_KWARG - VM_CALL_KW_SPLAT - VM_CALL_KW_SPLAT_MUT - VM_CALL_KW_SPLAT_bit - VM_CALL_OPT_SEND - VM_CALL_TAILCALL - VM_CALL_TAILCALL_bit - VM_CALL_ZSUPER - VM_ENV_DATA_INDEX_FLAGS - VM_ENV_DATA_SIZE - VM_ENV_FLAG_LOCAL - VM_ENV_FLAG_WB_REQUIRED - VM_FRAME_FLAG_BMETHOD - VM_FRAME_FLAG_CFRAME - VM_FRAME_FLAG_CFRAME_KW - VM_FRAME_FLAG_LAMBDA - VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM - VM_FRAME_MAGIC_BLOCK - VM_FRAME_MAGIC_CFUNC - VM_FRAME_MAGIC_METHOD - VM_METHOD_TYPE_ALIAS - VM_METHOD_TYPE_ATTRSET - VM_METHOD_TYPE_BMETHOD - VM_METHOD_TYPE_CFUNC - VM_METHOD_TYPE_ISEQ - VM_METHOD_TYPE_IVAR - VM_METHOD_TYPE_MISSING - VM_METHOD_TYPE_NOTIMPLEMENTED - VM_METHOD_TYPE_OPTIMIZED - VM_METHOD_TYPE_REFINED - VM_METHOD_TYPE_UNDEF - VM_METHOD_TYPE_ZSUPER - VM_SPECIAL_OBJECT_VMCORE - RUBY_ENCODING_MASK - RUBY_FL_FREEZE - RHASH_PASS_AS_KEYWORDS - ], - }, - values: { - SIZET: %w[ - block_type_iseq - imemo_iseq - imemo_callinfo - rb_block_param_proxy - rb_cArray - rb_cFalseClass - rb_cFloat - rb_cInteger - rb_cNilClass - rb_cString - rb_cSymbol - rb_cTrueClass - rb_rjit_global_events - rb_mRubyVMFrozenCore - rb_vm_insns_count - idRespond_to_missing - ], - }, - funcs: %w[ - rb_ary_entry_internal - rb_ary_push - rb_ary_resurrect - rb_ary_store - rb_ec_ary_new_from_values - rb_ec_str_resurrect - rb_ensure_iv_list_size - rb_fix_aref - rb_fix_div_fix - rb_fix_mod_fix - rb_fix_mul_fix - rb_gc_writebarrier - rb_get_symbol_id - rb_hash_aref - rb_hash_aset - rb_hash_bulk_insert - rb_hash_new - rb_hash_new_with_size - rb_hash_resurrect - rb_ivar_get - rb_obj_as_string_result - rb_obj_is_kind_of - rb_str_concat_literals - rb_str_eql_internal - rb_str_getbyte - rb_vm_bh_to_procval - rb_vm_concat_array - rb_vm_defined - rb_vm_get_ev_const - rb_vm_getclassvariable - rb_vm_ic_hit_p - rb_vm_opt_newarray_min - rb_vm_opt_newarray_max - rb_vm_opt_newarray_hash - rb_vm_setinstancevariable - rb_vm_splat_array - rjit_full_cfunc_return - rjit_optimized_call - rjit_str_neq_internal - rjit_record_exit_stack - rb_ivar_defined - rb_vm_throw - rb_backref_get - rb_reg_last_match - rb_reg_match_pre - rb_reg_match_post - rb_reg_match_last - rb_reg_nth_match - rb_gvar_get - rb_range_new - rb_ary_tmp_new_from_values - rb_reg_new_ary - rb_ary_clear - rb_str_intern - rb_vm_setclassvariable - rb_str_bytesize - rjit_str_simple_append - rb_str_buf_append - rb_str_dup - rb_vm_yield_with_cfunc - rb_vm_set_ivar_id - rb_ary_dup - rjit_rb_ary_subseq_length - rb_ary_unshift_m - rjit_build_kwhash - rb_rjit_entry_stub_hit - rb_rjit_branch_stub_hit - rb_sym_to_proc - ], - types: %w[ - CALL_DATA - IC - ID - IVC - RArray - RB_BUILTIN - RBasic - RObject - RStruct - RString - attr_index_t - iseq_inline_constant_cache - iseq_inline_constant_cache_entry - iseq_inline_iv_cache_entry - iseq_inline_storage_entry - method_optimized_type - rb_block - rb_block_type - rb_builtin_function - rb_call_data - rb_callable_method_entry_struct - rb_callable_method_entry_t - rb_callcache - rb_callinfo - rb_captured_block - rb_control_frame_t - rb_cref_t - rb_execution_context_struct - rb_execution_context_t - rb_iseq_constant_body - rb_iseq_location_t - rb_iseq_struct - rb_iseq_t - rb_method_attr_t - rb_method_bmethod_t - rb_method_cfunc_t - rb_method_definition_struct - rb_method_entry_t - rb_method_iseq_t - rb_method_optimized_t - rb_method_type_t - rb_proc_t - rb_rjit_runtime_counters - rb_serial_t - rb_shape - rb_shape_t - rb_thread_struct - rb_jit_func_t - rb_iseq_param_keyword - rb_rjit_options - rb_callinfo_kwarg - ], - # #ifdef-dependent immediate types, which need Primitive.cexpr! for type detection - dynamic_types: %w[ - VALUE - shape_id_t - ], - skip_fields: { - 'rb_execution_context_struct.machine': %w[regs], # differs between macOS and Linux - rb_execution_context_struct: %w[method_missing_reason], # non-leading bit fields not supported - rb_iseq_constant_body: %w[jit_exception jit_exception_calls yjit_payload yjit_calls_at_interv], # conditionally defined - rb_thread_struct: %w[status has_dedicated_nt to_kill abort_on_exception report_on_exception pending_interrupt_queue_checked], - :'' => %w[is_from_method is_lambda is_isolated], # rb_proc_t - }, - ruby_fields: { - rb_iseq_constant_body: %w[ - rjit_blocks - ], - rb_iseq_location_struct: %w[ - base_label - label - pathobj - ], - rb_callable_method_entry_t: %w[ - defined_class - ], - rb_callable_method_entry_struct: %w[ - defined_class - ], - }, -) -generator.generate(nodes) - -# Write rjit_c.rb -File.write(src_path, generator.src) diff --git a/tool/ruby-version.rb b/tool/ruby-version.rb new file mode 100755 index 0000000000..3bbec576e1 --- /dev/null +++ b/tool/ruby-version.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby + +module RubyVersion + def self.tag(version) + major_version = Integer(version.split('.', 2)[0]) + if major_version >= 4 + "v#{version}" + else + "v#{version.tr('.-', '_')}" + end + end + + # Return the previous version to be used for release diff links. + # For a ".0" version, it returns the previous ".0" version. + # For a non-".0" version, it returns the previous teeny version. + def self.previous(version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + major = Integer($1) + minor = Integer($2) + teeny = Integer($3) + + if teeny != 0 + "#{major}.#{minor}.#{teeny-1}" + elsif minor != 0 # && teeny == 0 + "#{major}.#{minor-1}.#{teeny}" + else # minor == 0 && teeny == 0 + case major + when 3 + "2.7.0" + when 4 + "3.4.0" + else + raise "it doesn't know what is the previous version of '#{version}'" + end + end + end +end + +if __FILE__ == $0 + case ARGV[0] + when "tag" + print RubyVersion.tag(ARGV[1]) + when "previous" + print RubyVersion.previous(ARGV[1]) + when "previous-tag" + print RubyVersion.tag(RubyVersion.previous(ARGV[1])) + else + "#{$0}: unexpected command #{ARGV[0].inspect}" + end +end diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb index e03e54e397..f6c0e39600 100644 --- a/tool/ruby_vm/controllers/application_controller.rb +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb index 2a99e408da..628cb0428b 100644 --- a/tool/ruby_vm/helpers/c_escape.rb +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb index c083dffa7a..f0758dd44f 100644 --- a/tool/ruby_vm/helpers/dumper.rb +++ b/tool/ruby_vm/helpers/dumper.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -33,11 +32,7 @@ class RubyVM::Dumper rescue Errno::ENOENT raise "don't know how to generate #{path}" else - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%-') - else - erb = ERB.new(src, nil, '%-') - end + erb = ERB.new(src, trim_mode: '%-') erb.filename = path.to_path return erb end diff --git a/tool/ruby_vm/helpers/scanner.rb b/tool/ruby_vm/helpers/scanner.rb index ef6de8120e..8998abb2d3 100644 --- a/tool/ruby_vm/helpers/scanner.rb +++ b/tool/ruby_vm/helpers/scanner.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb index 034905f74e..d45d0ba83c 100644 --- a/tool/ruby_vm/loaders/insns_def.rb +++ b/tool/ruby_vm/loaders/insns_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb index aa6fd79e79..0750f1823a 100644 --- a/tool/ruby_vm/loaders/opt_insn_unif_def.rb +++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb index 29aef8a325..e08509a433 100644 --- a/tool/ruby_vm/loaders/opt_operand_def.rb +++ b/tool/ruby_vm/loaders/opt_operand_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb index 3f05c270ee..d626ea0296 100644 --- a/tool/ruby_vm/loaders/vm_opts_h.rb +++ b/tool/ruby_vm/loaders/vm_opts_h.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index ac4122f3ac..177b701b92 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instruction.rb index a810d89f3c..f87dd74179 100755..100644 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -15,7 +14,7 @@ require_relative 'c_expr' require_relative 'typemap' require_relative 'attribute' -class RubyVM::BareInstructions +class RubyVM::BareInstruction attr_reader :template, :name, :operands, :pops, :rets, :decls, :expr def initialize opts = {} @@ -108,14 +107,6 @@ class RubyVM::BareInstructions /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr end - def always_leaf? - @attrs.fetch('leaf').expr.expr == 'true;' - end - - def leaf_without_check_ints? - @attrs.fetch('leaf').expr.expr == 'leafness_of_check_ints;' - end - def handle_canary stmt # Stack canary is basically a good thing that we want to add, however: # @@ -149,6 +140,10 @@ class RubyVM::BareInstructions @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } end + def zjit_profile? + @attrs.fetch('zjit_profile').expr.expr != 'false;' + end + private def check_attribute_consistency @@ -187,6 +182,7 @@ class RubyVM::BareInstructions generate_attribute 'rb_snum_t', 'sp_inc', rets.size - pops.size generate_attribute 'bool', 'handles_sp', default_definition_of_handles_sp generate_attribute 'bool', 'leaf', default_definition_of_leaf + generate_attribute 'bool', 'zjit_profile', false end def default_definition_of_handles_sp @@ -228,13 +224,13 @@ class RubyVM::BareInstructions new h.merge(:template => h) } - def self.fetch name + def self.find(name) @instances.find do |insn| insn.name == name end or raise IndexError, "instruction not found: #{name}" end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb index 4b5aec58dd..095ff4f1d9 100644 --- a/tool/ruby_vm/models/c_expr.rb +++ b/tool/ruby_vm/models/c_expr.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/instructions.rb b/tool/ruby_vm/models/instructions.rb index 1198c7a4a6..7be7064b14 100644 --- a/tool/ruby_vm/models/instructions.rb +++ b/tool/ruby_vm/models/instructions.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -10,13 +9,15 @@ # conditions mentioned in the file COPYING are met. Consult the file for # details. -require_relative 'bare_instructions' -require_relative 'operands_unifications' -require_relative 'instructions_unifications' +require_relative 'bare_instruction' +require_relative 'operands_unification' +require_relative 'instructions_unification' +require_relative 'trace_instruction' +require_relative 'zjit_instruction' -RubyVM::Instructions = RubyVM::BareInstructions.to_a + \ - RubyVM::OperandsUnifications.to_a + \ - RubyVM::InstructionsUnifications.to_a - -require_relative 'trace_instructions' +RubyVM::Instructions = RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all + + RubyVM::TraceInstruction.all + + RubyVM::ZJITInstruction.all RubyVM::Instructions.freeze diff --git a/tool/ruby_vm/models/instructions_unifications.rb b/tool/ruby_vm/models/instructions_unification.rb index 214ba5fcc2..5c798e6d54 100644 --- a/tool/ruby_vm/models/instructions_unifications.rb +++ b/tool/ruby_vm/models/instructions_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,9 +11,9 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_insn_unif_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::InstructionsUnifications +class RubyVM::InstructionsUnification include RubyVM::CEscape attr_reader :name @@ -23,7 +22,7 @@ class RubyVM::InstructionsUnifications @location = opts[:location] @name = namegen opts[:signature] @series = opts[:signature].map do |i| - RubyVM::BareInstructions.fetch i # Misshit is fatal + RubyVM::BareInstruction.find(i) # Misshit is fatal end end @@ -37,7 +36,7 @@ class RubyVM::InstructionsUnifications new h end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/operands_unifications.rb b/tool/ruby_vm/models/operands_unification.rb index 10e5742897..ce118648ca 100644 --- a/tool/ruby_vm/models/operands_unifications.rb +++ b/tool/ruby_vm/models/operands_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,16 +11,16 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_operand_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::OperandsUnifications < RubyVM::BareInstructions +class RubyVM::OperandsUnification < RubyVM::BareInstruction include RubyVM::CEscape attr_reader :preamble, :original, :spec def initialize opts = {} name = opts[:signature][0] - @original = RubyVM::BareInstructions.fetch name + @original = RubyVM::BareInstruction.find(name) template = @original.template parts = compose opts[:location], opts[:signature], template[:signature] json = template.dup @@ -130,12 +129,12 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions new h end - def self.to_a + def self.all @instances end def self.each_group - to_a.group_by(&:original).each_pair do |k, v| + all.group_by(&:original).each_pair do |k, v| yield k, v end end diff --git a/tool/ruby_vm/models/trace_instructions.rb b/tool/ruby_vm/models/trace_instruction.rb index 4ed4c8cb42..6a3ad53c44 100644 --- a/tool/ruby_vm/models/trace_instructions.rb +++ b/tool/ruby_vm/models/trace_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -11,9 +10,9 @@ # details. require_relative '../helpers/c_escape' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::TraceInstructions +class RubyVM::TraceInstruction include RubyVM::CEscape attr_reader :name @@ -61,11 +60,11 @@ class RubyVM::TraceInstructions private - @instances = RubyVM::Instructions.map {|i| new i } + @instances = (RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all).map {|i| new(i) } - def self.to_a + def self.all @instances end - - RubyVM::Instructions.push(*to_a) end diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb index d762dd3321..68ef5a41a5 100644 --- a/tool/ruby_vm/models/typemap.rb +++ b/tool/ruby_vm/models/typemap.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/zjit_instruction.rb b/tool/ruby_vm/models/zjit_instruction.rb new file mode 100644 index 0000000000..04764e4c61 --- /dev/null +++ b/tool/ruby_vm/models/zjit_instruction.rb @@ -0,0 +1,56 @@ +require_relative '../helpers/c_escape' +require_relative 'bare_instruction' + +# Profile YARV instructions to optimize code generated by ZJIT +class RubyVM::ZJITInstruction + include RubyVM::CEscape + + attr_reader :name + + def initialize(orig) + @orig = orig + @name = as_tr_cpp "zjit @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute?(*) + return false + end + + @instances = RubyVM::BareInstruction.all.filter(&:zjit_profile?).map {|i| new(i) } + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb index 47d8da5513..ad8603b1a8 100644 --- a/tool/ruby_vm/scripts/insns2vm.rb +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/tests/.gitkeep b/tool/ruby_vm/tests/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 --- a/tool/ruby_vm/tests/.gitkeep +++ /dev/null diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb index cb895815ce..8bb28db1c1 100644 --- a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -6,6 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. %# +% +% stack_increase = proc do |i| +% if i.has_attribute?('sp_inc') +% '-127' +% else +% sprintf("%4d", i.rets.size - i.pops.size) +% end +% end +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); @@ -13,15 +23,14 @@ rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) { static const signed char t[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map { |i| - if i.has_attribute?('sp_inc') - '-127' - else - sprintf("%4d", i.rets.size - i.pops.size) - end - }.join(', ') -%>, +% insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, % end +#endif }; signed char c = t[insn]; diff --git a/tool/ruby_vm/views/_insn_leaf_info.erb b/tool/ruby_vm/views/_insn_leaf_info.erb new file mode 100644 index 0000000000..f30366ffda --- /dev/null +++ b/tool/ruby_vm/views/_insn_leaf_info.erb @@ -0,0 +1,18 @@ +MAYBE_UNUSED(static bool insn_leaf(int insn, const VALUE *opes)); +static bool +insn_leaf(int insn, const VALUE *opes) +{ + switch (insn) { +% RubyVM::Instructions.each do |insn| +% next if insn.is_a?(RubyVM::TraceInstruction) || insn.is_a?(RubyVM::ZJITInstruction) + case <%= insn.bin %>: + return attr_leaf_<%= insn.name %>(<%= + insn.operands.map.with_index do |ope, i| + "(#{ope[:type]})opes[#{i}]" + end.join(', ') + %>); +% end + default: + return false; + } +} diff --git a/tool/ruby_vm/views/_insn_len_info.erb b/tool/ruby_vm/views/_insn_len_info.erb index 569dca5845..b29a405918 100644 --- a/tool/ruby_vm/views/_insn_len_info.erb +++ b/tool/ruby_vm/views/_insn_len_info.erb @@ -5,6 +5,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% CONSTFUNC(MAYBE_UNUSED(static int insn_len(VALUE insn))); RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ @@ -13,9 +16,14 @@ RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO const uint8_t rb_vm_insn_len_info[] = { -% RubyVM::Instructions.each_slice 23 do |a| - <%= a.map(&:width).join(', ') -%>, +% insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, % end +#if USE_ZJIT +% zjit_insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_len_info); diff --git a/tool/ruby_vm/views/_insn_name_info.erb b/tool/ruby_vm/views/_insn_name_info.erb index e7ded75e65..2862908631 100644 --- a/tool/ruby_vm/views/_insn_name_info.erb +++ b/tool/ruby_vm/views/_insn_name_info.erb @@ -6,10 +6,14 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.name } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% next_offset = 0 +% name_offset = proc do |i| +% offset = sprintf("%4d", next_offset) +% next_offset += i.name.length + 1 # insn.name + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_name(VALUE insn))); @@ -20,18 +24,29 @@ extern const unsigned short rb_vm_insn_name_offset[VM_INSTRUCTION_SIZE]; RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO -const int rb_vm_max_insn_name_size = <%= a.map(&:size).max %>; +%# "trace_" is longer than "zjit_", so USE_ZJIT doesn't impact the max name size. +const int rb_vm_max_insn_name_size = <%= RubyVM::Instructions.map { |i| i.name.size }.max %>; const char rb_vm_insn_name_base[] = -% a.each do |i| - <%=cstr i%> "\0" +% insns.each do |i| + <%= cstr i.name %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each do |i| + <%= cstr i.name %> "\0" % end +#endif ; const unsigned short rb_vm_insn_name_offset[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%4d", i) }.join(', ') %>, +% insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, % end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_name_offset); diff --git a/tool/ruby_vm/views/_insn_operand_info.erb b/tool/ruby_vm/views/_insn_operand_info.erb index 996c33e960..410869fcd3 100644 --- a/tool/ruby_vm/views/_insn_operand_info.erb +++ b/tool/ruby_vm/views/_insn_operand_info.erb @@ -6,10 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.operands_info } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% operands_info = proc { |i| sprintf("%-6s", cstr(i.operands_info)) } +% +% next_offset = 0 +% op_offset = proc do |i| +% offset = sprintf("%3d", next_offset) +% next_offset += i.operands_info.length + 1 # insn.operands_info + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_op_types(VALUE insn))); CONSTFUNC(MAYBE_UNUSED(static int insn_op_type(VALUE insn, long pos))); @@ -21,15 +27,25 @@ RUBY_SYMBOL_EXPORT_END #ifdef RUBY_VM_INSNS_INFO const char rb_vm_insn_op_base[] = -% a.each_slice 5 do |d| - <%= d.map {|i| sprintf("%-6s", cstr(i)) }.join(' "\0" ') %> "\0" +% insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" % end +#endif ; const unsigned short rb_vm_insn_op_offset[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%3d", i) }.join(', ') %>, +% insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, % end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_op_offset); diff --git a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb b/tool/ruby_vm/views/_insn_sp_pc_dependency.erb deleted file mode 100644 index 95528fbbf4..0000000000 --- a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb +++ /dev/null @@ -1,27 +0,0 @@ -%# -*- C -*- -%# Copyright (c) 2019 Takashi Kokubun. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -%# -PUREFUNC(MAYBE_UNUSED(static bool insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes))); - -static bool -insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes) -{ - switch (insn) { -% RubyVM::Instructions.each do |insn| -% # handles_sp?: If true, it requires to move sp in JIT -% # always_leaf?: If false, it may call an arbitrary method. pc should be moved -% # before the call, and the method may refer to caller's pc (lineno). -% unless !insn.is_a?(RubyVM::TraceInstructions) && !insn.handles_sp? && insn.always_leaf? - case <%= insn.bin %>: -% end -% end - return true; - default: - return false; - } -} diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb index f740107a0a..2756fa2dec 100644 --- a/tool/ruby_vm/views/_leaf_helpers.erb +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -10,10 +10,6 @@ #include "iseq.h" -// This is used to tell RJIT that this insn would be leaf if CHECK_INTS didn't exist. -// It should be used only when RUBY_VM_CHECK_INTS is directly written in insns.def. -static bool leafness_of_check_ints = false; - static bool leafness_of_defined(rb_num_t op_type) { @@ -25,7 +21,7 @@ leafness_of_defined(rb_num_t op_type) case DEFINED_YIELD: case DEFINED_REF: case DEFINED_ZSUPER: - return false; + return true; case DEFINED_CONST: case DEFINED_CONST_FROM: /* has rb_autoload_load(); */ diff --git a/tool/ruby_vm/views/_sp_inc_helpers.erb b/tool/ruby_vm/views/_sp_inc_helpers.erb index d0b0bd79ef..740fe10142 100644 --- a/tool/ruby_vm/views/_sp_inc_helpers.erb +++ b/tool/ruby_vm/views/_sp_inc_helpers.erb @@ -18,7 +18,7 @@ sp_inc_of_sendish(const struct rb_callinfo *ci) * 3. Pop receiver. * 4. Push return value. */ - const int argb = (vm_ci_flag(ci) & VM_CALL_ARGS_BLOCKARG) ? 1 : 0; + const int argb = (vm_ci_flag(ci) & (VM_CALL_ARGS_BLOCKARG | VM_CALL_FORWARDING)) ? 1 : 0; const int argc = vm_ci_argc(ci); const int recv = 1; const int retn = 1; diff --git a/tool/ruby_vm/views/_zjit_helpers.erb b/tool/ruby_vm/views/_zjit_helpers.erb new file mode 100644 index 0000000000..1185dbd9d8 --- /dev/null +++ b/tool/ruby_vm/views/_zjit_helpers.erb @@ -0,0 +1,31 @@ +#if USE_ZJIT + +MAYBE_UNUSED(static int vm_bare_insn_to_zjit_insn(int insn)); +static int +vm_bare_insn_to_zjit_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case BIN(<%= insn.jump_destination %>): + return <%= insn.bin %>; +% end + default: + return insn; + } +} + +MAYBE_UNUSED(static int vm_zjit_insn_to_bare_insn(int insn)); +static int +vm_zjit_insn_to_bare_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case <%= insn.bin %>: + return BIN(<%= insn.jump_destination %>); +% end + default: + return insn; + } +} + +#endif diff --git a/tool/ruby_vm/views/_zjit_instruction.erb b/tool/ruby_vm/views/_zjit_instruction.erb new file mode 100644 index 0000000000..7fd657697c --- /dev/null +++ b/tool/ruby_vm/views/_zjit_instruction.erb @@ -0,0 +1,12 @@ +#if USE_ZJIT + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + rb_zjit_profile_insn(BIN(<%= insn.jump_destination %>), ec); + DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} + +#endif diff --git a/tool/ruby_vm/views/insns.inc.erb b/tool/ruby_vm/views/insns.inc.erb index 29981a8a2d..6521a89b8a 100644 --- a/tool/ruby_vm/views/insns.inc.erb +++ b/tool/ruby_vm/views/insns.inc.erb @@ -6,21 +6,36 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'contains YARV instruction list', edit: __FILE__, } -%> +#ifndef INSNS_INC +#define INSNS_INC 1 + /* BIN : Basic Instruction Name */ #define BIN(n) YARVINSN_##n enum ruby_vminsn_type { -% RubyVM::Instructions.each do |i| +% insns.each do |i| + <%= i.bin %>, +% end +#if USE_ZJIT +% zjit_insns.each do |i| <%= i.bin %>, % end +#endif VM_INSTRUCTION_SIZE }; +#define VM_BARE_INSTRUCTION_SIZE <%= RubyVM::Instructions.count { |i| i.name !~ /\A(trace|zjit)_/ } %> + #define ASSERT_VM_INSTRUCTION_SIZE(array) \ STATIC_ASSERT(numberof_##array, numberof(array) == VM_INSTRUCTION_SIZE) + +#endif diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb index 2ca5aca7cf..48dd0e8832 100644 --- a/tool/ruby_vm/views/insns_info.inc.erb +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -11,12 +11,16 @@ this_file: 'contains instruction information for yarv instruction sequence.', edit: __FILE__, } %> +#ifndef INSNS_INFO_INC +#define INSNS_INFO_INC 1 <%= render 'insn_type_chars' %> <%= render 'insn_name_info' %> <%= render 'insn_len_info' %> <%= render 'insn_operand_info' %> <%= render 'leaf_helpers' %> <%= render 'sp_inc_helpers' %> +<%= render 'zjit_helpers' %> <%= render 'attributes' %> +<%= render 'insn_leaf_info' %> <%= render 'comptime_insn_stack_increase' %> -<%= render 'insn_sp_pc_dependency' %> +#endif diff --git a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb index 4f08524a77..793528af5d 100644 --- a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb +++ b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb @@ -7,7 +7,7 @@ module RubyVM::RJIT # :nodoc: all name: :<%= insn.name %>, bin: <%= i %>, # BIN(<%= insn.name %>) len: <%= insn.width %>, # insn_len - operands: <%= (insn.operands unless insn.name.start_with?('trace_')).inspect %>, + operands: <%= (insn.operands unless insn.name.start_with?(/trace_|zjit_/)).inspect %>, ), % end } diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb index de7bb210ea..9d9cf0a43a 100644 --- a/tool/ruby_vm/views/optinsn.inc.erb +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -23,7 +23,7 @@ insn_operands_unification(INSN *iobj) /* do nothing */; break; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| case <%= orig.bin %>: % unifs.each do |insn| @@ -56,7 +56,7 @@ rb_insn_unified_local_var_level(VALUE insn) switch (insn) { default: return -1; /* do nothing */; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| % unifs.each do|insn| case <%= insn.bin %>: % insn.spec.map{|(var,val)|val}.reject{|i| i == '*' }.each do |val| diff --git a/tool/ruby_vm/views/optunifs.inc.erb b/tool/ruby_vm/views/optunifs.inc.erb index e92a95beff..c096712936 100644 --- a/tool/ruby_vm/views/optunifs.inc.erb +++ b/tool/ruby_vm/views/optunifs.inc.erb @@ -7,7 +7,6 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % raise ':FIXME:TBW' if RubyVM::VmOptsH['INSTRUCTIONS_UNIFICATION'] -% n = RubyVM::Instructions.size <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -16,6 +15,4 @@ /* Let .bss section automatically initialize this variable */ /* cf. Section 6.7.8 of ISO/IEC 9899:1999 */ -static const int *const *const unified_insns_data[<%= n %>]; - -ASSERT_VM_INSTRUCTION_SIZE(unified_insns_data); +static const int *const *const unified_insns_data[VM_INSTRUCTION_SIZE]; diff --git a/tool/ruby_vm/views/vm.inc.erb b/tool/ruby_vm/views/vm.inc.erb index c1a3faf60a..38bf5f05ae 100644 --- a/tool/ruby_vm/views/vm.inc.erb +++ b/tool/ruby_vm/views/vm.inc.erb @@ -13,18 +13,22 @@ } -%> #include "vm_insnhelper.h" -% RubyVM::BareInstructions.to_a.each do |insn| +% RubyVM::BareInstruction.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::OperandsUnifications.to_a.each do |insn| +% RubyVM::OperandsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::InstructionsUnifications.to_a.each do |insn| +% RubyVM::InstructionsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::TraceInstructions.to_a.each do |insn| +% RubyVM::ZJITInstruction.all.each do |insn| +<%= render 'zjit_instruction', locals: { insn: insn } -%> +% end +% +% RubyVM::TraceInstruction.all.each do |insn| <%= render 'trace_instruction', locals: { insn: insn } -%> % end diff --git a/tool/ruby_vm/views/vmtc.inc.erb b/tool/ruby_vm/views/vmtc.inc.erb index 99cbd92614..39dc8bfa6b 100644 --- a/tool/ruby_vm/views/vmtc.inc.erb +++ b/tool/ruby_vm/views/vmtc.inc.erb @@ -6,6 +6,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' -%> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -13,9 +16,14 @@ } -%> static const void *const insns_address_table[] = { -% RubyVM::Instructions.each do |i| +% insns.each do |i| LABEL_PTR(<%= i.name %>), % end +#if USE_ZJIT +% zjit_insns.each do |i| + LABEL_PTR(<%= i.name %>), +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(insns_address_table); diff --git a/tool/rubyspec_temp.rb b/tool/rubyspec_temp.rb deleted file mode 100644 index 339bfce211..0000000000 --- a/tool/rubyspec_temp.rb +++ /dev/null @@ -1,13 +0,0 @@ -require "tmpdir" -require "fileutils" - -if (tmpdir = Dir.mktmpdir("rubyspec_temp.")).size > 80 - # On macOS, the default TMPDIR is very long, inspite of UNIX socket - # path length is limited. - Dir.rmdir(tmpdir) - tmpdir = Dir.mktmpdir("rubyspec_temp.", "/tmp") -end -# warn "tmpdir(#{tmpdir.size}) = #{tmpdir}" -END {FileUtils.rm_rf(tmpdir)} - -ENV["TMPDIR"] = ENV["SPEC_TEMP_DIR"] = tmpdir diff --git a/tool/run-gcov.rb b/tool/run-gcov.rb index 5df7622aa3..46626e4703 100644 --- a/tool/run-gcov.rb +++ b/tool/run-gcov.rb @@ -47,7 +47,8 @@ Pathname.glob("**/*.gcda").sort.each do |gcda| )? Creating\ .*\n \n - )+\z + )+ + (Lines\ executed:.*\n)?\z )x raise "Unexpected gcov output" end diff --git a/tool/run-lcov.rb b/tool/run-lcov.rb index f27578200a..bdccc29a11 100644 --- a/tool/run-lcov.rb +++ b/tool/run-lcov.rb @@ -20,7 +20,7 @@ def backup_gcda_files(gcda_files) end def run_lcov(*args) - system("lcov", "--rc", "lcov_branch_coverage=1", *args) + system("lcov", "--rc", "geninfo_unexecuted_blocks=1", "--rc", "lcov_branch_coverage=1", *args, exception: true) end $info_files = [] @@ -41,11 +41,19 @@ def run_lcov_remove(info_src, info_out) ext/-test-/* ext/nkf/nkf-utf8/nkf.c ).each {|f| dirs << File.join(File.dirname(__dir__), f) } - run_lcov("--remove", info_src, *dirs, "-o", info_out) + run_lcov("--ignore-errors", "unused", "--remove", info_src, *dirs, "-o", info_out) end def run_genhtml(info, out) - system("genhtml", "--branch-coverage", "--ignore-errors", "source", info, "-o", out) + base_dir = File.dirname(File.dirname(__dir__)) + ignore_errors = %w(source unmapped category).reject do |a| + Open3.capture3("genhtml", "--ignore-errors", a)[1].include?("unknown argument for --ignore-errors") + end + system("genhtml", + "--branch-coverage", + "--prefix", base_dir, + *ignore_errors.flat_map {|a| ["--ignore-errors", a] }, + info, "-o", out, exception: true) end def gen_rb_lcov(file) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index d1b761078e..14d7a3893d 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -3,6 +3,9 @@ # See `tool/sync_default_gems.rb --help` for how to use this. require 'fileutils' +require "rbconfig" +require "find" +require "tempfile" module SyncDefaultGems include FileUtils @@ -10,98 +13,337 @@ module SyncDefaultGems module_function - REPOSITORIES = { - "io-console": 'ruby/io-console', - "io-nonblock": 'ruby/io-nonblock', - "io-wait": 'ruby/io-wait', - "net-http": "ruby/net-http", - "net-protocol": "ruby/net-protocol", - "open-uri": "ruby/open-uri", - "resolv-replace": "ruby/resolv-replace", - English: "ruby/English", - abbrev: "ruby/abbrev", - base64: "ruby/base64", - benchmark: "ruby/benchmark", - bigdecimal: "ruby/bigdecimal", - cgi: "ruby/cgi", - csv: 'ruby/csv', - date: 'ruby/date', - delegate: "ruby/delegate", - did_you_mean: "ruby/did_you_mean", - digest: "ruby/digest", - drb: "ruby/drb", - erb: "ruby/erb", - error_highlight: "ruby/error_highlight", - etc: 'ruby/etc', - fcntl: 'ruby/fcntl', - fiddle: 'ruby/fiddle', - fileutils: 'ruby/fileutils', - find: "ruby/find", - forwardable: "ruby/forwardable", - ipaddr: 'ruby/ipaddr', - irb: 'ruby/irb', - json: 'flori/json', - logger: 'ruby/logger', - nkf: "ruby/nkf", - observer: "ruby/observer", - open3: "ruby/open3", - openssl: "ruby/openssl", - optparse: "ruby/optparse", - ostruct: 'ruby/ostruct', - pathname: "ruby/pathname", - pp: "ruby/pp", - prettyprint: "ruby/prettyprint", - prism: ["ruby/prism", "main"], - pstore: "ruby/pstore", - psych: 'ruby/psych', - rdoc: 'ruby/rdoc', - readline: "ruby/readline", - reline: 'ruby/reline', - resolv: "ruby/resolv", - rinda: "ruby/rinda", - rubygems: 'rubygems/rubygems', - securerandom: "ruby/securerandom", - set: "ruby/set", - shellwords: "ruby/shellwords", - singleton: "ruby/singleton", - stringio: 'ruby/stringio', - strscan: 'ruby/strscan', - syntax_suggest: ["ruby/syntax_suggest", "main"], - syslog: "ruby/syslog", - tempfile: "ruby/tempfile", - time: "ruby/time", - timeout: "ruby/timeout", - tmpdir: "ruby/tmpdir", - tsort: "ruby/tsort", - un: "ruby/un", - uri: "ruby/uri", - weakref: "ruby/weakref", - win32ole: "ruby/win32ole", - yaml: "ruby/yaml", - zlib: 'ruby/zlib', - }.transform_keys(&:to_s) + # upstream: "owner/repo" + # branch: "branch_name" + # mappings: [ ["path_in_upstream", "path_in_ruby"], ... ] + # NOTE: path_in_ruby is assumed to be "owned" by this gem, and the contents + # will be removed before sync + # exclude: [ "fnmatch_pattern_after_mapping", ... ] + Repository = Data.define(:upstream, :branch, :mappings, :exclude) do + def excluded?(newpath) + p = newpath + until p == "." + return true if exclude.any? {|pat| File.fnmatch?(pat, p, File::FNM_PATHNAME|File::FNM_EXTGLOB)} + p = File.dirname(p) + end + false + end + + def rewrite_for_ruby(path) + newpath = mappings.find do |src, dst| + if path == src || path.start_with?(src + "/") + break path.sub(src, dst) + end + end + return nil unless newpath + return nil if excluded?(newpath) + newpath + end + end CLASSICAL_DEFAULT_BRANCH = "master" - class << REPOSITORIES - def [](gem) - repo, branch = super(gem) - return repo, branch || CLASSICAL_DEFAULT_BRANCH - end + def repo((upstream, branch), mappings, exclude: []) + branch ||= CLASSICAL_DEFAULT_BRANCH + exclude += ["ext/**/depend"] + Repository.new(upstream:, branch:, mappings:, exclude:) + end + + def lib((upstream, branch), gemspec_in_subdir: false) + _org, name = upstream.split("/") + gemspec_dst = gemspec_in_subdir ? "lib/#{name}/#{name}.gemspec" : "lib/#{name}.gemspec" + repo([upstream, branch], [ + ["lib/#{name}.rb", "lib/#{name}.rb"], + ["lib/#{name}", "lib/#{name}"], + ["test/test_#{name}.rb", "test/test_#{name}.rb"], + ["test/#{name}", "test/#{name}"], + ["#{name}.gemspec", gemspec_dst], + ]) + end - def each_pair - super do |gem, (repo, branch)| - yield gem, [repo, branch || CLASSICAL_DEFAULT_BRANCH] + # Note: tool/auto_review_pr.rb also depends on these constants. + NO_UPSTREAM = [ + "lib/unicode_normalize", # not to match with "lib/un" + ] + REPOSITORIES = { + Onigmo: repo("k-takata/Onigmo", [ + ["regcomp.c", "regcomp.c"], + ["regenc.c", "regenc.c"], + ["regenc.h", "regenc.h"], + ["regerror.c", "regerror.c"], + ["regexec.c", "regexec.c"], + ["regint.h", "regint.h"], + ["regparse.c", "regparse.c"], + ["regparse.h", "regparse.h"], + ["regsyntax.c", "regsyntax.c"], + ["onigmo.h", "include/ruby/onigmo.h"], + ["enc", "enc"], + ]), + "io-console": repo("ruby/io-console", [ + ["ext/io/console", "ext/io/console"], + ["test/io/console", "test/io/console"], + ["lib/io/console", "ext/io/console/lib/console"], + ["io-console.gemspec", "ext/io/console/io-console.gemspec"], + ]), + "io-nonblock": repo("ruby/io-nonblock", [ + ["ext/io/nonblock", "ext/io/nonblock"], + ["test/io/nonblock", "test/io/nonblock"], + ["io-nonblock.gemspec", "ext/io/nonblock/io-nonblock.gemspec"], + ]), + "io-wait": repo("ruby/io-wait", [ + ["ext/io/wait", "ext/io/wait"], + ["test/io/wait", "test/io/wait"], + ["io-wait.gemspec", "ext/io/wait/io-wait.gemspec"], + ]), + "net-http": repo("ruby/net-http", [ + ["lib/net/http.rb", "lib/net/http.rb"], + ["lib/net/http", "lib/net/http"], + ["test/net/http", "test/net/http"], + ["net-http.gemspec", "lib/net/http/net-http.gemspec"], + ]), + "net-protocol": repo("ruby/net-protocol", [ + ["lib/net/protocol.rb", "lib/net/protocol.rb"], + ["test/net/protocol", "test/net/protocol"], + ["net-protocol.gemspec", "lib/net/net-protocol.gemspec"], + ]), + "open-uri": lib("ruby/open-uri"), + "win32-registry": repo("ruby/win32-registry", [ + ["lib/win32/registry.rb", "ext/win32/lib/win32/registry.rb"], + ["test/win32/test_registry.rb", "test/win32/test_registry.rb"], + ["win32-registry.gemspec", "ext/win32/win32-registry.gemspec"], + ]), + English: lib("ruby/English"), + cgi: repo("ruby/cgi", [ + ["ext/cgi", "ext/cgi"], + ["lib/cgi/escape.rb", "lib/cgi/escape.rb"], + ["test/cgi/test_cgi_escape.rb", "test/cgi/test_cgi_escape.rb"], + ["test/cgi/update_env.rb", "test/cgi/update_env.rb"], + ]), + date: repo("ruby/date", [ + ["doc/date", "doc/date"], + ["ext/date", "ext/date"], + ["lib", "ext/date/lib"], + ["test/date", "test/date"], + ["date.gemspec", "ext/date/date.gemspec"], + ], exclude: [ + "ext/date/lib/date_core.bundle", + ]), + delegate: lib("ruby/delegate"), + did_you_mean: repo("ruby/did_you_mean", [ + ["lib/did_you_mean.rb", "lib/did_you_mean.rb"], + ["lib/did_you_mean", "lib/did_you_mean"], + ["test", "test/did_you_mean"], + ["did_you_mean.gemspec", "lib/did_you_mean/did_you_mean.gemspec"], + ], exclude: [ + "test/did_you_mean/lib", + "test/did_you_mean/tree_spell/test_explore.rb", + ]), + digest: repo("ruby/digest", [ + ["ext/digest/lib/digest/sha2", "ext/digest/sha2/lib/sha2"], + ["ext/digest", "ext/digest"], + ["lib/digest.rb", "ext/digest/lib/digest.rb"], + ["lib/digest/version.rb", "ext/digest/lib/digest/version.rb"], + ["lib/digest/sha2.rb", "ext/digest/sha2/lib/sha2.rb"], + ["test/digest", "test/digest"], + ["digest.gemspec", "ext/digest/digest.gemspec"], + ]), + erb: repo("ruby/erb", [ + ["ext/erb", "ext/erb"], + ["lib/erb", "lib/erb"], + ["lib/erb.rb", "lib/erb.rb"], + ["test/erb", "test/erb"], + ["erb.gemspec", "lib/erb/erb.gemspec"], + ["libexec/erb", "libexec/erb"], + ]), + error_highlight: repo("ruby/error_highlight", [ + ["lib/error_highlight.rb", "lib/error_highlight.rb"], + ["lib/error_highlight", "lib/error_highlight"], + ["test", "test/error_highlight"], + ["error_highlight.gemspec", "lib/error_highlight/error_highlight.gemspec"], + ]), + etc: repo("ruby/etc", [ + ["ext/etc", "ext/etc"], + ["test/etc", "test/etc"], + ["etc.gemspec", "ext/etc/etc.gemspec"], + ]), + fcntl: repo("ruby/fcntl", [ + ["ext/fcntl", "ext/fcntl"], + ["fcntl.gemspec", "ext/fcntl/fcntl.gemspec"], + ]), + fileutils: lib("ruby/fileutils"), + find: lib("ruby/find"), + forwardable: lib("ruby/forwardable", gemspec_in_subdir: true), + ipaddr: lib("ruby/ipaddr"), + json: repo("ruby/json", [ + ["ext/json/ext", "ext/json"], + ["test/json", "test/json"], + ["lib", "ext/json/lib"], + ["json.gemspec", "ext/json/json.gemspec"], + ], exclude: [ + "ext/json/lib/json/ext/.keep", + "ext/json/lib/json/pure.rb", + "ext/json/lib/json/pure", + "ext/json/lib/json/truffle_ruby", + "test/json/lib", + "ext/json/extconf.rb", + ]), + mmtk: repo(["ruby/mmtk", "main"], [ + ["gc/mmtk", "gc/mmtk"], + ]), + open3: lib("ruby/open3", gemspec_in_subdir: true).tap { + it.exclude << "lib/open3/jruby_windows.rb" + }, + openssl: repo("ruby/openssl", [ + ["ext/openssl", "ext/openssl"], + ["lib", "ext/openssl/lib"], + ["test/openssl", "test/openssl"], + ["sample", "sample/openssl"], + ["openssl.gemspec", "ext/openssl/openssl.gemspec"], + ["History.md", "ext/openssl/History.md"], + ], exclude: [ + "test/openssl/envutil.rb", + "ext/openssl/depend", + ]), + optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap { + it.mappings << ["doc/optparse", "doc/optparse"] + }, + pathname: repo("ruby/pathname", [ + ["ext/pathname/pathname.c", "pathname.c"], + ["lib/pathname_builtin.rb", "pathname_builtin.rb"], + ["lib/pathname.rb", "lib/pathname.rb"], + ["test/pathname", "test/pathname"], + ]), + pp: lib("ruby/pp"), + prettyprint: lib("ruby/prettyprint"), + prism: repo(["ruby/prism", "main"], [ + ["ext/prism", "prism"], + ["lib/prism.rb", "lib/prism.rb"], + ["lib/prism", "lib/prism"], + ["test/prism", "test/prism"], + ["src", "prism"], + ["prism.gemspec", "lib/prism/prism.gemspec"], + ["include/prism", "prism"], + ["include/prism.h", "prism/prism.h"], + ["config.yml", "prism/config.yml"], + ["templates", "prism/templates"], + ], exclude: [ + "prism/templates/{javascript,java,rbi,sig}", + "test/prism/snapshots_test.rb", + "test/prism/snapshots", + "prism/extconf.rb", + "prism/srcs.mk*", + ]), + psych: repo("ruby/psych", [ + ["ext/psych", "ext/psych"], + ["lib", "ext/psych/lib"], + ["test/psych", "test/psych"], + ["psych.gemspec", "ext/psych/psych.gemspec"], + ], exclude: [ + "ext/psych/lib/org", + "ext/psych/lib/psych.jar", + "ext/psych/lib/psych_jars.rb", + "ext/psych/lib/psych.{bundle,so}", + "ext/psych/lib/2.*", + "ext/psych/yaml/LICENSE", + "ext/psych/.gitignore", + ]), + resolv: repo("ruby/resolv", [ + ["lib/resolv.rb", "lib/resolv.rb"], + ["test/resolv", "test/resolv"], + ["resolv.gemspec", "lib/resolv.gemspec"], + ["ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32/resolv.rb"], + ["ext/win32/resolv", "ext/win32/resolv"], + ]), + rubygems: repo("ruby/rubygems", [ + ["lib/rubygems.rb", "lib/rubygems.rb"], + ["lib/rubygems", "lib/rubygems"], + ["test/rubygems", "test/rubygems"], + ["bundler/lib/bundler.rb", "lib/bundler.rb"], + ["bundler/lib/bundler", "lib/bundler"], + ["bundler/exe/bundle", "libexec/bundle"], + ["bundler/exe/bundler", "libexec/bundler"], + ["bundler/bundler.gemspec", "lib/bundler/bundler.gemspec"], + ["bundler/spec", "spec/bundler"], + *["bundle", "parallel_rspec", "rspec"].map {|binstub| + ["bundler/bin/#{binstub}", "spec/bin/#{binstub}"] + }, + *%w[dev_gems test_gems rubocop_gems standard_gems].flat_map {|gemfile| + ["rb.lock", "rb"].map do |ext| + ["tool/bundler/#{gemfile}.#{ext}", "tool/bundler/#{gemfile}.#{ext}"] + end + }, + ], exclude: [ + "spec/bundler/bin", + "spec/bundler/support/artifice/vcr_cassettes", + "spec/bundler/support/artifice/used_cassettes.txt", + "lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}", + ]), + securerandom: lib("ruby/securerandom"), + shellwords: lib("ruby/shellwords"), + singleton: lib("ruby/singleton"), + stringio: repo("ruby/stringio", [ + ["ext/stringio", "ext/stringio"], + ["test/stringio", "test/stringio"], + ["stringio.gemspec", "ext/stringio/stringio.gemspec"], + ["doc/stringio", "doc/stringio"], + ], exclude: [ + "ext/stringio/README.md", + ]), + strscan: repo("ruby/strscan", [ + ["ext/strscan", "ext/strscan"], + ["lib", "ext/strscan/lib"], + ["test/strscan", "test/strscan"], + ["strscan.gemspec", "ext/strscan/strscan.gemspec"], + ["doc/strscan", "doc/strscan"], + ], exclude: [ + "ext/strscan/regenc.h", + "ext/strscan/regint.h", + ]), + syntax_suggest: lib(["ruby/syntax_suggest", "main"], gemspec_in_subdir: true), + tempfile: lib("ruby/tempfile"), + time: lib("ruby/time"), + timeout: lib("ruby/timeout"), + tmpdir: lib("ruby/tmpdir"), + un: lib("ruby/un"), + uri: lib("ruby/uri", gemspec_in_subdir: true), + weakref: lib("ruby/weakref"), + yaml: lib("ruby/yaml", gemspec_in_subdir: true), + zlib: repo("ruby/zlib", [ + ["ext/zlib", "ext/zlib"], + ["test/zlib", "test/zlib"], + ["zlib.gemspec", "ext/zlib/zlib.gemspec"], + ]), + }.transform_keys(&:to_s) + + class << Repository + def find_upstream(file) + return if NO_UPSTREAM.any? {|dst| file.start_with?(dst) } + REPOSITORIES.find do |repo_name, repository| + if repository.mappings.any? {|_src, dst| file.start_with?(dst) } + break repo_name + end end end + + def group(files) + files.group_by {|file| find_upstream(file)} + end end + # Allow synchronizing commits up to this FETCH_DEPTH. We've historically merged PRs + # with about 250 commits to ruby/ruby, so we use this depth for ruby/ruby in general. + FETCH_DEPTH = 500 + def pipe_readlines(args, rs: "\0", chomp: true) IO.popen(args) do |f| f.readlines(rs, chomp: chomp) end end + def porcelain_status(*pattern) + pipe_readlines(%W"git status --porcelain --no-renames -z --" + pattern) + end + def replace_rdoc_ref(file) src = File.binread(file) changed = false @@ -120,7 +362,7 @@ module SyncDefaultGems end def replace_rdoc_ref_all - result = pipe_readlines(%W"git status --porcelain -z -- *.c *.rb *.rdoc") + result = porcelain_status("*.c", "*.rb", "*.rdoc") result.map! {|line| line[/\A.M (.*)/, 1]} result.compact! return if result.empty? @@ -128,328 +370,111 @@ module SyncDefaultGems result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} end + def replace_rdoc_ref_all_full + Dir.glob("**/*.{c,rb,rdoc}").inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def rubygems_do_fixup + gemspec_content = File.readlines("lib/bundler/bundler.gemspec").map do |line| + next if line =~ /LICENSE\.md/ + + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") + end.compact.join + File.write("lib/bundler/bundler.gemspec", gemspec_content) + + ["bundle", "parallel_rspec", "rspec"].each do |binstub| + path = "spec/bin/#{binstub}" + next unless File.exist?(path) + content = File.read(path).gsub("../spec", "../bundler") + File.write(path, content) + chmod("+x", path) + end + end + # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. def sync_default_gems(gem) - repo, = REPOSITORIES[gem] - puts "Sync #{repo}" - - upstream = File.join("..", "..", repo) - - case gem - when "rubygems" - rm_rf(%w[lib/rubygems lib/rubygems.rb test/rubygems]) - cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") - cp_r("#{upstream}/test/rubygems", "test") - rm_rf(%w[lib/bundler lib/bundler.rb libexec/bundler libexec/bundle spec/bundler tool/bundler/*]) - cp_r(Dir.glob("#{upstream}/bundler/lib/bundler*"), "lib") - cp_r(Dir.glob("#{upstream}/bundler/exe/bundle*"), "libexec") - - gemspec_content = File.readlines("#{upstream}/bundler/bundler.gemspec").map do |line| - next if line =~ /LICENSE\.md/ - - line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") - end.compact.join - File.write("lib/bundler/bundler.gemspec", gemspec_content) - - cp_r("#{upstream}/bundler/spec", "spec/bundler") - %w[dev_gems test_gems rubocop_gems standard_gems].each do |gemfile| - cp_r("#{upstream}/tool/bundler/#{gemfile}.rb", "tool/bundler") - end - rm_rf Dir.glob("spec/bundler/support/artifice/{vcr_cassettes,used_cassettes.txt}") - rm_rf Dir.glob("lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}") - when "rdoc" - rm_rf(%w[lib/rdoc lib/rdoc.rb test/rdoc libexec/rdoc libexec/ri]) - cp_r(Dir.glob("#{upstream}/lib/rdoc*"), "lib") - cp_r("#{upstream}/doc/rdoc", "doc") - cp_r("#{upstream}/test/rdoc", "test") - cp_r("#{upstream}/rdoc.gemspec", "lib/rdoc") - cp_r("#{upstream}/Gemfile", "lib/rdoc") - cp_r("#{upstream}/Rakefile", "lib/rdoc") - cp_r("#{upstream}/exe/rdoc", "libexec") - cp_r("#{upstream}/exe/ri", "libexec") - parser_files = { - 'lib/rdoc/markdown.kpeg' => 'lib/rdoc/markdown.rb', - 'lib/rdoc/markdown/literals.kpeg' => 'lib/rdoc/markdown/literals.rb', - 'lib/rdoc/rd/block_parser.ry' => 'lib/rdoc/rd/block_parser.rb', - 'lib/rdoc/rd/inline_parser.ry' => 'lib/rdoc/rd/inline_parser.rb' - } - Dir.chdir(upstream) do - `bundle install` - parser_files.each_value do |dst| - `bundle exec rake #{dst}` + config = REPOSITORIES[gem] + puts "Sync #{config.upstream}" + + upstream = File.join("..", "..", config.upstream) + + config.mappings.each do |src, dst| + rm_rf(dst) + end + + copied = Set.new + config.mappings.each do |src, dst| + prefix = File.join(upstream, src) + # Maybe mapping needs to be updated? + next unless File.exist?(prefix) + Find.find(prefix) do |path| + next if File.directory?(path) + if copied.add?(path) + newpath = config.rewrite_for_ruby(path.sub(%r{\A#{Regexp.escape(upstream)}/}, "")) + next unless newpath + mkdir_p(File.dirname(newpath)) + cp(path, newpath) end end - parser_files.each_pair do |src, dst| - rm_rf(src) - cp_r("#{upstream}/#{dst}", dst) + end + + porcelain_status().each do |line| + /\A(?:.)(?:.) (?<path>.*)\z/ =~ line or raise + if config.excluded?(path) + puts "Restoring excluded file: #{path}" + IO.popen(%W"git checkout --" + [path], "rb", &:read) end - `git checkout lib/rdoc/.document` - rm_rf(%w[lib/rdoc/Gemfile lib/rdoc/Rakefile]) - when "reline" - rm_rf(%w[lib/reline lib/reline.rb test/reline]) - cp_r(Dir.glob("#{upstream}/lib/reline*"), "lib") - cp_r("#{upstream}/test/reline", "test") - cp_r("#{upstream}/reline.gemspec", "lib/reline") - when "irb" - rm_rf(%w[lib/irb lib/irb.rb test/irb]) - cp_r(Dir.glob("#{upstream}/lib/irb*"), "lib") - cp_r("#{upstream}/test/irb", "test") - cp_r("#{upstream}/irb.gemspec", "lib/irb") - cp_r("#{upstream}/man/irb.1", "man/irb.1") - cp_r("#{upstream}/doc/irb", "doc") - when "json" - rm_rf(%w[ext/json test/json]) - cp_r("#{upstream}/ext/json/ext", "ext/json") - cp_r("#{upstream}/tests", "test/json") - rm_rf("test/json/lib") - cp_r("#{upstream}/lib", "ext/json") - cp_r("#{upstream}/json.gemspec", "ext/json") - rm_rf(%w[ext/json/lib/json/ext ext/json/lib/json/pure.rb ext/json/lib/json/pure]) - `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend ext/json/depend` - when "psych" - rm_rf(%w[ext/psych test/psych]) - cp_r("#{upstream}/ext/psych", "ext") - cp_r("#{upstream}/lib", "ext/psych") - cp_r("#{upstream}/test/psych", "test") - rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) - rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) - rm_rf(["ext/psych/yaml/LICENSE"]) - cp_r("#{upstream}/psych.gemspec", "ext/psych") - `git checkout ext/psych/depend ext/psych/.gitignore` - when "fiddle" - rm_rf(%w[ext/fiddle test/fiddle]) - cp_r("#{upstream}/ext/fiddle", "ext") - cp_r("#{upstream}/lib", "ext/fiddle") - cp_r("#{upstream}/test/fiddle", "test") - cp_r("#{upstream}/fiddle.gemspec", "ext/fiddle") - `git checkout ext/fiddle/depend` - rm_rf(%w[ext/fiddle/lib/fiddle.{bundle,so}]) - when "stringio" - rm_rf(%w[ext/stringio test/stringio]) - cp_r("#{upstream}/ext/stringio", "ext") - cp_r("#{upstream}/test/stringio", "test") - cp_r("#{upstream}/stringio.gemspec", "ext/stringio") - `git checkout ext/stringio/depend ext/stringio/README.md` - when "io-console" - rm_rf(%w[ext/io/console test/io/console]) - cp_r("#{upstream}/ext/io/console", "ext/io") - cp_r("#{upstream}/test/io/console", "test/io") - mkdir_p("ext/io/console/lib") - cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") - rm_rf("ext/io/console/lib/console/ffi") - cp_r("#{upstream}/io-console.gemspec", "ext/io/console") - `git checkout ext/io/console/depend` - when "io-nonblock" - rm_rf(%w[ext/io/nonblock test/io/nonblock]) - cp_r("#{upstream}/ext/io/nonblock", "ext/io") - cp_r("#{upstream}/test/io/nonblock", "test/io") - cp_r("#{upstream}/io-nonblock.gemspec", "ext/io/nonblock") - `git checkout ext/io/nonblock/depend` - when "io-wait" - rm_rf(%w[ext/io/wait test/io/wait]) - cp_r("#{upstream}/ext/io/wait", "ext/io") - cp_r("#{upstream}/test/io/wait", "test/io") - cp_r("#{upstream}/io-wait.gemspec", "ext/io/wait") - `git checkout ext/io/wait/depend` - when "etc" - rm_rf(%w[ext/etc test/etc]) - cp_r("#{upstream}/ext/etc", "ext") - cp_r("#{upstream}/test/etc", "test") - cp_r("#{upstream}/etc.gemspec", "ext/etc") - `git checkout ext/etc/depend` - when "date" - rm_rf(%w[ext/date test/date]) - cp_r("#{upstream}/doc/date", "doc") - cp_r("#{upstream}/ext/date", "ext") - cp_r("#{upstream}/lib", "ext/date") - cp_r("#{upstream}/test/date", "test") - cp_r("#{upstream}/date.gemspec", "ext/date") - `git checkout ext/date/depend` - rm_rf(["ext/date/lib/date_core.bundle"]) - when "zlib" - rm_rf(%w[ext/zlib test/zlib]) - cp_r("#{upstream}/ext/zlib", "ext") - cp_r("#{upstream}/test/zlib", "test") - cp_r("#{upstream}/zlib.gemspec", "ext/zlib") - `git checkout ext/zlib/depend` - when "fcntl" - rm_rf(%w[ext/fcntl]) - cp_r("#{upstream}/ext/fcntl", "ext") - cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") - `git checkout ext/fcntl/depend` - when "strscan" - rm_rf(%w[ext/strscan test/strscan]) - cp_r("#{upstream}/ext/strscan", "ext") - cp_r("#{upstream}/test/strscan", "test") - cp_r("#{upstream}/strscan.gemspec", "ext/strscan") - rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) - `git checkout ext/strscan/depend` - when "cgi" - rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) - cp_r("#{upstream}/ext/cgi", "ext") - cp_r("#{upstream}/lib", ".") - rm_rf("lib/cgi/escape.jar") - cp_r("#{upstream}/test/cgi", "test") - cp_r("#{upstream}/cgi.gemspec", "lib/cgi") - `git checkout ext/cgi/escape/depend` - when "openssl" - rm_rf(%w[ext/openssl test/openssl]) - cp_r("#{upstream}/ext/openssl", "ext") - cp_r("#{upstream}/lib", "ext/openssl") - cp_r("#{upstream}/test/openssl", "test") - rm_rf("test/openssl/envutil.rb") - cp_r("#{upstream}/openssl.gemspec", "ext/openssl") - cp_r("#{upstream}/History.md", "ext/openssl") - `git checkout ext/openssl/depend` - when "net-protocol" - rm_rf(%w[lib/net/protocol.rb lib/net/net-protocol.gemspec test/net/protocol]) - cp_r("#{upstream}/lib/net/protocol.rb", "lib/net") - cp_r("#{upstream}/test/net/protocol", "test/net") - cp_r("#{upstream}/net-protocol.gemspec", "lib/net") - when "net-http" - rm_rf(%w[lib/net/http.rb lib/net/http test/net/http]) - cp_r("#{upstream}/lib/net/http.rb", "lib/net") - cp_r("#{upstream}/lib/net/http", "lib/net") - cp_r("#{upstream}/test/net/http", "test/net") - cp_r("#{upstream}/net-http.gemspec", "lib/net/http") - when "did_you_mean" - rm_rf(%w[lib/did_you_mean lib/did_you_mean.rb test/did_you_mean]) - cp_r(Dir.glob("#{upstream}/lib/did_you_mean*"), "lib") - cp_r("#{upstream}/did_you_mean.gemspec", "lib/did_you_mean") - cp_r("#{upstream}/test", "test/did_you_mean") - rm_rf("test/did_you_mean/lib") - rm_rf(%w[test/did_you_mean/tree_spell/test_explore.rb]) - when "erb" - rm_rf(%w[lib/erb* test/erb libexec/erb]) - cp_r("#{upstream}/lib/erb.rb", "lib") - cp_r("#{upstream}/test/erb", "test") - cp_r("#{upstream}/erb.gemspec", "lib") - cp_r("#{upstream}/libexec/erb", "libexec") - when "nkf" - rm_rf(%w[ext/nkf test/nkf]) - cp_r("#{upstream}/ext/nkf", "ext") - cp_r("#{upstream}/lib", "ext/nkf") - cp_r("#{upstream}/test/nkf", "test") - cp_r("#{upstream}/nkf.gemspec", "ext/nkf") - `git checkout ext/nkf/depend` - when "syslog" - rm_rf(%w[ext/syslog test/syslog test/test_syslog.rb]) - cp_r("#{upstream}/ext/syslog", "ext") - cp_r("#{upstream}/lib", "ext/syslog") - cp_r("#{upstream}/test/syslog", "test") - cp_r("#{upstream}/test/test_syslog.rb", "test") - cp_r("#{upstream}/syslog.gemspec", "ext/syslog") - `git checkout ext/syslog/depend` - when "bigdecimal" - rm_rf(%w[ext/bigdecimal test/bigdecimal]) - cp_r("#{upstream}/ext/bigdecimal", "ext") - cp_r("#{upstream}/sample", "ext/bigdecimal") - cp_r("#{upstream}/lib", "ext/bigdecimal") - cp_r("#{upstream}/test/bigdecimal", "test") - cp_r("#{upstream}/bigdecimal.gemspec", "ext/bigdecimal") - `git checkout ext/bigdecimal/depend` - when "pathname" - rm_rf(%w[ext/pathname test/pathname]) - cp_r("#{upstream}/ext/pathname", "ext") - cp_r("#{upstream}/test/pathname", "test") - cp_r("#{upstream}/lib", "ext/pathname") - cp_r("#{upstream}/pathname.gemspec", "ext/pathname") - `git checkout ext/pathname/depend` - when "digest" - rm_rf(%w[ext/digest test/digest]) - cp_r("#{upstream}/ext/digest", "ext") - mkdir_p("ext/digest/lib/digest") - cp_r("#{upstream}/lib/digest.rb", "ext/digest/lib/") - cp_r("#{upstream}/lib/digest/version.rb", "ext/digest/lib/digest/") - mkdir_p("ext/digest/sha2/lib") - cp_r("#{upstream}/lib/digest/sha2.rb", "ext/digest/sha2/lib") - move("ext/digest/lib/digest/sha2", "ext/digest/sha2/lib") - cp_r("#{upstream}/test/digest", "test") - cp_r("#{upstream}/digest.gemspec", "ext/digest") - `git checkout ext/digest/depend ext/digest/*/depend` - when "set" - sync_lib gem, upstream - cp_r(Dir.glob("#{upstream}/test/*"), "test/set") - when "optparse" - sync_lib gem, upstream - rm_rf(%w[doc/optparse]) - mkdir_p("doc/optparse") - cp_r("#{upstream}/doc/optparse", "doc") - when "error_highlight" - rm_rf(%w[lib/error_highlight lib/error_highlight.rb test/error_highlight]) - cp_r(Dir.glob("#{upstream}/lib/error_highlight*"), "lib") - cp_r("#{upstream}/error_highlight.gemspec", "lib/error_highlight") - cp_r("#{upstream}/test", "test/error_highlight") - when "win32ole" - sync_lib gem, upstream - rm_rf(%w[ext/win32ole/lib]) - Dir.mkdir(*%w[ext/win32ole/lib]) - move("lib/win32ole/win32ole.gemspec", "ext/win32ole") - move(Dir.glob("lib/win32ole*"), "ext/win32ole/lib") - when "open3" - sync_lib gem, upstream - rm_rf("lib/open3/jruby_windows.rb") - when "syntax_suggest" - sync_lib gem, upstream - rm_rf(%w[spec/syntax_suggest libexec/syntax_suggest]) - cp_r("#{upstream}/spec", "spec/syntax_suggest") - cp_r("#{upstream}/exe/syntax_suggest", "libexec/syntax_suggest") - when "prism" - rm_rf(%w[test/prism prism]) - - cp_r("#{upstream}/ext/prism", "prism") - cp_r("#{upstream}/lib/.", "lib") - cp_r("#{upstream}/test/prism", "test") - cp_r("#{upstream}/src/.", "prism") - - cp_r("#{upstream}/prism.gemspec", "lib/prism") - cp_r("#{upstream}/include/prism/.", "prism") - cp_r("#{upstream}/include/prism.h", "prism") - - cp_r("#{upstream}/config.yml", "prism/") - cp_r("#{upstream}/templates", "prism/") - rm_rf("prism/templates/javascript") - rm_rf("prism/templates/java") - rm_rf("prism/templates/rbi") - rm_rf("prism/templates/sig") - - rm("prism/extconf.rb") - else - sync_lib gem, upstream end - replace_rdoc_ref_all - end - def ignore_file_pattern_for(gem) - patterns = [] + # RubyGems/Bundler needs special care + if gem == "rubygems" + rubygems_do_fixup + end - # Common patterns - patterns << %r[\A(?: - [^/]+ # top-level entries - |\.git.* - |bin/.* - |ext/.*\.java - |rakelib/.* - |test/(?:lib|fixtures)/.* - |tool/(?!bundler/).* - )\z]mx + check_prerelease_version(gem) - # Gem-specific patterns - case gem - when nil - end&.tap do |pattern| - patterns << pattern - end + # Architecture-dependent files must not pollute libdir. + rm_rf(Dir["lib/**/*.#{RbConfig::CONFIG['DLEXT']}"]) + replace_rdoc_ref_all + end - Regexp.union(*patterns) + def check_prerelease_version(gem) + return if ["rubygems", "mmtk", "cgi", "pathname", "Onigmo"].include?(gem) + + require "net/https" + require "json" + require "uri" + + uri = URI("https://rubygems.org/api/v1/versions/#{gem.downcase}/latest.json") + response = Net::HTTP.get(uri) + latest_version = JSON.parse(response)["version"] + + gemspec = [ + "lib/#{gem}/#{gem}.gemspec", + "lib/#{gem}.gemspec", + "ext/#{gem}/#{gem}.gemspec", + "ext/#{gem.split("-").join("/")}/#{gem}.gemspec", + "lib/#{gem.split("-").first}/#{gem}.gemspec", + "ext/#{gem.split("-").first}/#{gem}.gemspec", + "lib/#{gem.split("-").join("/")}/#{gem}.gemspec", + ].find{|gemspec| File.exist?(gemspec)} + spec = Gem::Specification.load(gemspec) + puts "#{gem}-#{spec.version} is not latest version of rubygems.org" if spec.version.to_s != latest_version end - def message_filter(repo, sha, input: ARGF) - log = input.read - log.delete!("\r") - log << "\n" if !log.end_with?("\n") + def message_filter(repo, sha, log, context: nil) + unless repo.count("/") == 1 and /\A\S+\z/ =~ repo + raise ArgumentError, "invalid repository: #{repo}" + end + unless /\A\h{10,40}\z/ =~ sha + raise ArgumentError, "invalid commit-hash: #{sha}" + end repo_url = "https://github.com/#{repo}" + # Log messages generated by GitHub web UI have inconsistent line endings + log = log.delete("\r") + log << "\n" if !log.end_with?("\n") + # Split the subject from the log message according to git conventions. # SPECIAL TREAT: when the first line ends with a dot `.` (which is not # obeying the conventions too), takes only that line. @@ -471,41 +496,69 @@ module SyncDefaultGems end end commit_url = "#{repo_url}/commit/#{sha[0,10]}\n" + sync_note = context ? "#{commit_url}\n#{context}" : commit_url if log and !log.empty? log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last conv[log] log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) { - ($~.begin(1) ? "" : "\n\n") + commit_url + ($~.begin(2) ? "\n" : "") + ($~.begin(1) ? "" : "\n\n") + sync_note + ($~.begin(2) ? "\n" : "") } else - log = commit_url + log = sync_note end - puts subject, "\n", log + "#{subject}\n\n#{log}" end - # Returns commit list as array of [commit_hash, subject]. - def commits_in_ranges(gem, repo, default_branch, ranges) - # If -a is given, discover all commits since the last picked commit - if ranges == true - # \r? needed in the regex in case the commit has windows-style line endings (because e.g. we're running - # tests on Windows) - pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)\r?$" - log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", &:read) - ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] - end + def log_format(format, args, &block) + IO.popen(%W[git -c core.autocrlf=false -c core.eol=lf + log --no-show-signature --format=#{format}] + args, "rb", &block) + end - # Parse a given range with git log - ranges.flat_map do |range| - unless range.include?("..") - range = "#{range}~1..#{range}" - end + def commits_in_range(upto, exclude, toplevel:) + args = [upto, *exclude.map {|s|"^#{s}"}] + log_format('%H,%P,%s', %W"--first-parent" + args) do |f| + f.read.split("\n").reverse.flat_map {|commit| + hash, parents, subject = commit.split(',', 3) + parents = parents.split + + # Non-merge commit + if parents.size <= 1 + puts "#{hash} #{subject}" + next [[hash, subject]] + end - IO.popen(%W"git log --format=%H,%s #{range} --") do |f| - f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} - end + # Clean 2-parent merge commit: follow the other parent as long as it + # contains no potentially-non-clean merges + if parents.size == 2 && + IO.popen(%W"git diff-tree --remerge-diff #{hash}", "rb", &:read).empty? + puts "\e[2mChecking the other parent of #{hash} #{subject}\e[0m" + ret = catch(:quit) { + commits_in_range(parents[1], exclude + [parents[0]], toplevel: false) + } + next ret if ret + end + + unless toplevel + puts "\e[1mMerge commit with possible conflict resolution #{hash} #{subject}\e[0m" + throw :quit + end + + puts "#{hash} #{subject} " \ + "\e[1m[merge commit with possible conflicts, will do a squash merge]\e[0m" + [[hash, subject]] + } end end + # Returns commit list as array of [commit_hash, subject, sync_note]. + def commits_in_ranges(ranges) + ranges.flat_map do |range| + exclude, upto = range.include?("..") ? range.split("..", 2) : ["#{range}~1", range] + puts "Looking for commits in range #{exclude}..#{upto}" + commits_in_range(upto, exclude.empty? ? [] : [exclude], toplevel: true) + end.uniq + end + #-- # Following methods used by sync_default_gems_with_commits return # true: success @@ -514,27 +567,9 @@ module SyncDefaultGems #++ def resolve_conflicts(gem, sha, edit) - # Skip this commit if everything has been removed as `ignored_paths`. - changes = pipe_readlines(%W"git status --porcelain -z") - if changes.empty? - puts "Skip empty commit #{sha}" - return false - end - - # We want to skip DD: deleted by both. - deleted = changes.grep(/^DD /) {$'} - system(*%W"git rm -f --", *deleted) unless deleted.empty? - - # Import UA: added by them - added = changes.grep(/^UA /) {$'} - system(*%W"git add --", *added) unless added.empty? - - # Discover unmerged files - # AU: unmerged, added by us - # DU: unmerged, deleted by us - # UU: unmerged, both modified - # AA: unmerged, both added - conflict = changes.grep(/\A(?:.U|AA) /) {$'} + # Discover unmerged files: any unstaged changes + changes = porcelain_status() + conflict = changes.grep(/\A(?:.[^ ?]) /) {$'} # If -e option is given, open each conflicted file with an editor unless conflict.empty? if edit @@ -555,123 +590,170 @@ module SyncDefaultGems return true end - def preexisting?(base, file) - system(*%w"git cat-file -e", "#{base}:#{file}", err: File::NULL) + def collect_cacheinfo(tree) + pipe_readlines(%W"git ls-tree -r -t -z #{tree}").filter_map do |line| + fields, path = line.split("\t", 2) + mode, type, object = fields.split(" ", 3) + next unless type == "blob" + [mode, type, object, path] + end end - def filter_pickup_files(changed, ignore_file_pattern, base) - toplevels = {} - remove = [] - ignore = [] - changed = changed.reject do |f| - case - when toplevels.fetch(top = f[%r[\A[^/]+(?=/|\z)]m]) { - remove << top if toplevels[top] = !preexisting?(base, top) - } - # Remove any new top-level directories. - true - when ignore_file_pattern.match?(f) - # Forcibly reset any changes matching ignore_file_pattern. - (preexisting?(base, f) ? ignore : remove) << f - end + def rewrite_cacheinfo(gem, blobs) + config = REPOSITORIES[gem] + rewritten = [] + ignored = blobs.dup + ignored.delete_if do |mode, type, object, path| + newpath = config.rewrite_for_ruby(path) + next unless newpath + rewritten << [mode, type, object, newpath] end - return changed, remove, ignore + [rewritten, ignored] end - def pickup_files(gem, changed, picked) - # Forcibly remove any files that we don't want to copy to this - # repository. - - ignore_file_pattern = ignore_file_pattern_for(gem) + def make_commit_info(gem, sha) + config = REPOSITORIES[gem] + headers, orig = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + /^author (?<author_name>.+?) <(?<author_email>.*?)> (?<author_date>.+?)$/ =~ headers or + raise "unable to parse author info for commit #{sha}" + author = { + "GIT_AUTHOR_NAME" => author_name, + "GIT_AUTHOR_EMAIL" => author_email, + "GIT_AUTHOR_DATE" => author_date, + } + context = nil + if /^parent (?<first_parent>.{40})\nparent .{40}$/ =~ headers + # Squashing a merge commit: keep authorship information + context = IO.popen(%W"git shortlog #{first_parent}..#{sha} --", "rb", &:read) + end + message = message_filter(config.upstream, sha, orig, context: context) + [author, message] + end - base = picked ? "HEAD~" : "HEAD" - changed, remove, ignore = filter_pickup_files(changed, ignore_file_pattern, base) + def fixup_commit(gem, commit) + wt = File.join("tmp", "sync_default_gems-fixup-worktree") + if File.directory?(wt) + IO.popen(%W"git -C #{wt} clean -xdf", "rb", &:read) + IO.popen(%W"git -C #{wt} reset --hard #{commit}", "rb", &:read) + else + IO.popen(%W"git worktree remove --force #{wt}", "rb", err: File::NULL, &:read) + IO.popen(%W"git worktree add --detach #{wt} #{commit}", "rb", &:read) + end + raise "git worktree prepare failed for commit #{commit}" unless $?.success? - unless remove.empty? - puts "Remove added files: #{remove.join(', ')}" - system(*%w"git rm -fr --", *remove) - if picked - system(*%w"git commit --amend --no-edit --", *remove, %i[out err] => File::NULL) + Dir.chdir(wt) do + if gem == "rubygems" + rubygems_do_fixup end + replace_rdoc_ref_all_full end - unless ignore.empty? - puts "Reset ignored files: #{ignore.join(', ')}" - system(*%W"git rm -r --", *ignore) - ignore.each {|f| system(*%W"git checkout -f", base, "--", f)} - end + IO.popen(%W"git -C #{wt} add -u", "rb", &:read) + IO.popen(%W"git -C #{wt} commit --amend --no-edit", "rb", &:read) + IO.popen(%W"git -C #{wt} rev-parse HEAD", "rb", &:read).chomp + end - if changed.empty? - return nil + def make_and_fixup_commit(gem, original_commit, cacheinfo, parent: nil, message: nil, author: nil) + tree = Tempfile.create("sync_default_gems-#{gem}-index") do |f| + File.unlink(f.path) + IO.popen({"GIT_INDEX_FILE" => f.path}, + %W"git update-index --index-info", "wb", out: IO::NULL) do |io| + cacheinfo.each do |mode, type, object, path| + io.puts("#{mode} #{type} #{object}\t#{path}") + end + end + raise "git update-index failed" unless $?.success? + + IO.popen({"GIT_INDEX_FILE" => f.path}, %W"git write-tree --missing-ok", "rb", &:read).chomp end - return changed + args = ["-m", message || "Rewriten commit for #{original_commit}"] + args += ["-p", parent] if parent + commit = IO.popen({**author}, %W"git commit-tree #{tree}" + args, "rb", &:read).chomp + + # Apply changes that require a working tree + commit = fixup_commit(gem, commit) + + commit end - def pickup_commit(gem, sha, edit) - # Attempt to cherry-pick a commit - result = IO.popen(%W"git cherry-pick #{sha}", &:read) - picked = $?.success? - if result =~ /nothing\ to\ commit/ - `git reset` - puts "Skip empty commit #{sha}" + def rewrite_commit(gem, sha) + author, message = make_commit_info(gem, sha) + new_blobs = collect_cacheinfo("#{sha}") + new_rewritten, new_ignored = rewrite_cacheinfo(gem, new_blobs) + + headers, _ = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + first_parent = headers[/^parent (.{40})$/, 1] + unless first_parent + # Root commit, first time to sync this repo + return make_and_fixup_commit(gem, sha, new_rewritten, message: message, author: author) + end + + old_blobs = collect_cacheinfo(first_parent) + old_rewritten, old_ignored = rewrite_cacheinfo(gem, old_blobs) + if old_ignored != new_ignored + paths = (old_ignored + new_ignored - (old_ignored & new_ignored)) + .map {|*_, path| path}.uniq + puts "\e\[1mIgnoring file changes not in mappings: #{paths.join(" ")}\e\[0m" + end + changed_paths = (old_rewritten + new_rewritten - (old_rewritten & new_rewritten)) + .map {|*_, path| path}.uniq + if changed_paths.empty? + puts "Skip commit only for tools or toplevel" return false end - # Skip empty commits - if result.empty? - return false - end + # Build commit objects from "cacheinfo" + new_parent = make_and_fixup_commit(gem, first_parent, old_rewritten) + new_commit = make_and_fixup_commit(gem, sha, new_rewritten, parent: new_parent, message: message, author: author) + puts "Created a temporary commit for cherry-pick: #{new_commit}" + new_commit + end - if picked - changed = pipe_readlines(%w"git diff-tree --name-only -r -z HEAD~..HEAD --") - else - changed = pipe_readlines(%w"git diff --name-only -r -z HEAD --") - end + def pickup_commit(gem, sha, edit) + rewritten = rewrite_commit(gem, sha) - # Pick up files to merge. - unless changed = pickup_files(gem, changed, picked) - puts "Skip commit #{sha} only for tools or toplevel" - if picked - `git reset --hard HEAD~` - else - `git cherry-pick --abort` - end - return false - end + # No changes remaining after rewriting + return false unless rewritten - # If the cherry-pick attempt failed, try to resolve conflicts. - # Skip the commit, if it contains unresolved conflicts or no files to pick up. - unless picked or resolve_conflicts(gem, sha, edit) - `git reset` && `git checkout .` && `git clean -fd` - return picked || nil # Fail unless cherry-picked - end + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{rewritten}", "rb", err: [:child, :out], &:read) + unless $?.success? + if result =~ /The previous cherry-pick is now empty/ + system(*%w"git cherry-pick --skip") + puts "Skip empty commit #{sha}" + return false + end - # Commit cherry-picked commit - if picked - system(*%w"git commit --amend --no-edit") - else - system(*%w"git cherry-pick --continue --no-edit") - end or return nil + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless resolve_conflicts(gem, sha, edit) + system(*%w"git --no-pager diff") if !edit # If failed, show `git diff` unless editing + `git reset` && `git checkout .` && `git clean -fd` # Clean up un-committed diffs + return nil # Fail unless cherry-picked + end - # Amend the commit if RDoc references need to be replaced - head = `git log --format=%H -1 HEAD`.chomp - system(*%w"git reset --quiet HEAD~ --") - amend = replace_rdoc_ref_all - system(*%W"git reset --quiet #{head} --") - if amend - `git commit --amend --no-edit --all` + # Commit cherry-picked commit + if porcelain_status().empty? + system(*%w"git cherry-pick --skip") + return false + else + system(*%w"git cherry-pick --continue --no-edit") + return nil unless $?.success? + end end + new_head = IO.popen(%W"git rev-parse HEAD", "rb", &:read).chomp + puts "Committed cherry-pick as #{new_head}" return true end - # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. - # @param ranges [Array<String>] "before..after". Note that it will NOT sync "before" (but commits after that). + # @param ranges [Array<String>, true] "commit", "before..after", or true. Note that it will NOT sync "before" (but commits after that). # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo, default_branch = REPOSITORIES[gem] + config = REPOSITORIES[gem] + repo, default_branch = config.upstream, config.branch puts "Sync #{repo} with commit history." # Fetch the repository to be synchronized @@ -680,57 +762,37 @@ module SyncDefaultGems `git remote add #{gem} https://github.com/#{repo}.git` end end - system(*%W"git fetch --no-tags #{gem}") - - commits = commits_in_ranges(gem, repo, default_branch, ranges) + system(*%W"git fetch --no-tags --depth=#{FETCH_DEPTH} #{gem} #{default_branch}") - # Ignore Merge commits and already-merged commits. - commits.delete_if do |sha, subject| - subject.start_with?("Merge", "Auto Merge") + # If -a is given, discover all commits since the last picked commit + if ranges == true + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" + log = log_format('%B', %W"-E --grep=#{pattern} -n1 --", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] end - + commits = commits_in_ranges(ranges) if commits.empty? puts "No commits to pick" return true end - puts "Try to pick these commits:" - puts commits.map{|commit| commit.join(": ")} - puts "----" - failed_commits = [] - - require 'shellwords' - filter = [ - ENV.fetch('RUBY', 'ruby').shellescape, - File.realpath(__FILE__).shellescape, - "--message-filter", - ] commits.each do |sha, subject| - puts "Pick #{sha} from #{repo}." + puts "----" + puts "Pick #{sha} #{subject}" case pickup_commit(gem, sha, edit) when false - next + # skipped when nil - failed_commits << sha - next - end - - puts "Update commit message: #{sha}" - - # Run this script itself (tool/sync_default_gems.rb --message-filter) as a message filter - IO.popen({"FILTER_BRANCH_SQUELCH_WARNING" => "1"}, - %W[git filter-branch -f --msg-filter #{[filter, repo, sha].join(' ')} -- HEAD~1..HEAD], - &:read) - unless $?.success? - puts "Failed to modify commit message of #{sha}" - break + failed_commits << [sha, subject] end end unless failed_commits.empty? puts "---- failed commits ----" - puts failed_commits + failed_commits.each do |sha, subject| + puts "#{sha} #{subject}" + end return false end return true @@ -757,9 +819,9 @@ module SyncDefaultGems end def update_default_gems(gem, release: false) - - repository, default_branch = REPOSITORIES[gem] - author, repository = repository.split('/') + config = REPOSITORIES[gem] + author, repository = config.upstream.split('/') + default_branch = config.branch puts "Update #{author}/#{repository}" @@ -799,28 +861,18 @@ module SyncDefaultGems REPOSITORIES.each_key {|gem| update_default_gems(gem)} end when "all" - if ARGV[1] == "release" - REPOSITORIES.each_key do |gem| - update_default_gems(gem, release: true) - sync_default_gems(gem) - end - else - REPOSITORIES.each_key {|gem| sync_default_gems(gem)} + REPOSITORIES.each_key do |gem| + next if ["Onigmo"].include?(gem) + update_default_gems(gem, release: true) if ARGV[1] == "release" + sync_default_gems(gem) end when "list" ARGV.shift pattern = Regexp.new(ARGV.join('|')) - REPOSITORIES.each_pair do |name, (gem)| - next unless pattern =~ name or pattern =~ gem - printf "%-15s https://github.com/%s\n", name, gem - end - when "--message-filter" - ARGV.shift - if ARGV.size < 2 - abort "usage: #{$0} --message-filter repository commit-hash [input...]" + REPOSITORIES.each do |gem, config| + next unless pattern =~ gem or pattern =~ config.upstream + printf "%-15s https://github.com/%s\n", gem, config.upstream end - message_filter(*ARGV.shift(2)) - exit when "rdoc-ref" ARGV.shift pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV @@ -836,7 +888,13 @@ module SyncDefaultGems puts <<-HELP \e[1mSync with upstream code of default libraries\e[0m -\e[1mImport a default library through `git clone` and `cp -rf` (git commits are lost)\e[0m +\e[1mImport all default gems through `git clone` and `cp -rf` (git commits are lost)\e[0m + ruby #$0 all + +\e[1mImport all released version of default gems\e[0m + ruby #$0 all release + +\e[1mImport a default gem with specific gem same as all command\e[0m ruby #$0 rubygems \e[1mPick a single commit from the upstream repository\e[0m @@ -848,6 +906,9 @@ module SyncDefaultGems \e[1mPick all commits since the last picked commit\e[0m ruby #$0 -a rubygems +\e[1mUpdate repositories of default gems\e[0m + ruby #$0 up + \e[1mList known libraries\e[0m ruby #$0 list diff --git a/tool/test-annocheck.sh b/tool/test-annocheck.sh index 0224152d00..6896869e07 100755 --- a/tool/test-annocheck.sh +++ b/tool/test-annocheck.sh @@ -14,7 +14,8 @@ set -x DOCKER="$(command -v docker || command -v podman)" TAG=ruby-fedora-annocheck -TOOL_DIR=$(dirname "${0}") +TOOL_DIR="$(dirname "${0}")" +TMP_DIR="tmp/annocheck" DOCKER_RUN_VOLUME_OPTS= if [ -z "${CI-}" ]; then @@ -27,7 +28,13 @@ else # volume in container in container on GitHub Actions # <.github/workflows/compilers.yml>. TAG="${TAG}-copy" - "${DOCKER}" build --rm -t "${TAG}" --build-arg=FILES="${*}" -f ${TOOL_DIR}/annocheck/Dockerfile-copy . + rm -rf "${TMP_DIR}" + mkdir -p "${TMP_DIR}" + for file in "${@}"; do + cp -p "${file}" "${TMP_DIR}" + done + "${DOCKER}" build --rm -t "${TAG}" --build-arg=IN_DIR="${TMP_DIR}" -f ${TOOL_DIR}/annocheck/Dockerfile-copy . + rm -rf "${TMP_DIR}" fi "${DOCKER}" run --rm -t ${DOCKER_RUN_VOLUME_OPTS} "${TAG}" annocheck --verbose ${TEST_ANNOCHECK_OPTS-} "${@}" diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb index 32ac56d759..778fe3311a 100644 --- a/tool/test-bundled-gems.rb +++ b/tool/test-bundled-gems.rb @@ -1,38 +1,48 @@ require 'rbconfig' require 'timeout' require 'fileutils' +require 'shellwords' require_relative 'lib/colorize' +require_relative 'lib/gem_env' ENV.delete("GNUMAKEFLAGS") github_actions = ENV["GITHUB_ACTIONS"] == "true" +DEFAULT_ALLOWED_FAILURES = RUBY_PLATFORM =~ /mswin|mingw/ ? [ + 'debug', + 'irb', + 'csv', +] : [] allowed_failures = ENV['TEST_BUNDLED_GEMS_ALLOW_FAILURES'] || '' -allowed_failures = allowed_failures.split(',').reject(&:empty?) +allowed_failures = allowed_failures.split(',').concat(DEFAULT_ALLOWED_FAILURES).uniq.reject(&:empty?) -ENV["GEM_PATH"] = [File.realpath('.bundle'), File.realpath('../.bundle', __dir__)].join(File::PATH_SEPARATOR) +# make test-bundled-gems BUNDLED_GEMS=gem1,gem2,gem3 +bundled_gems = nil if (bundled_gems = ARGV.first&.split(","))&.empty? colorize = Colorize.new rake = File.realpath("../../.bundle/bin/rake", __FILE__) gem_dir = File.realpath('../../gems', __FILE__) rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +run_opts = ENV["RUN_OPTS"]&.shellsplit exit_code = 0 ruby = ENV['RUBY'] || RbConfig.ruby failed = [] File.foreach("#{gem_dir}/bundled_gems") do |line| - next if /^\s*(?:#|$)/ =~ line - gem = line.split.first - next if ARGV.any? {|pat| !File.fnmatch?(pat, gem)} - # 93(bright yellow) is copied from .github/workflows/mingw.yml - puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" + next unless gem = line[/^[^\s\#]+/] + next if bundled_gems&.none? {|pat| File.fnmatch?(pat, gem)} + next unless File.directory?("#{gem_dir}/src/#{gem}/test") - test_command = "#{ruby} -C #{gem_dir}/src/#{gem} #{rake} test" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", rake, "test"] first_timeout = 600 # 10min toplib = gem - case gem - when "typeprof" + unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + toplib = gem.tr("-", "/") + next unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + end + case gem when "rbs" # TODO: We should skip test file instead of test class/methods skip_test_files = %w[ @@ -43,7 +53,15 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| File.unlink(path) if File.exist?(path) end - test_command << " stdlib_test validate RBS_SKIP_TESTS=#{__dir__}/rbs_skip_tests SKIP_RBS_VALIDATION=true" + rbs_skip_tests = [ + File.join(__dir__, "/rbs_skip_tests") + ] + + if /mswin|mingw/ =~ RUBY_PLATFORM + rbs_skip_tests << File.join(__dir__, "/rbs_skip_tests_windows") + end + + test_command.concat %W[stdlib_test validate RBS_SKIP_TESTS=#{rbs_skip_tests.join(File::PATH_SEPARATOR)} SKIP_RBS_VALIDATION=true] first_timeout *= 3 when "debug" @@ -53,29 +71,42 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| load_path = true when "test-unit" - test_command = "#{ruby} -C #{gem_dir}/src/#{gem} test/run-test.rb" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", "test/run.rb"] - when /\Anet-/ - toplib = gem.tr("-", "/") + when "csv" + first_timeout = 30 + + when "win32ole" + next unless /mswin|mingw/ =~ RUBY_PLATFORM end if load_path libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) next unless $?.success? - puts libs ENV["RUBYLIB"] = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) else ENV["RUBYLIB"] = rubylib end + # 93(bright yellow) is copied from .github/workflows/mingw.yml + puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" print "[command]" if github_actions - puts test_command - pid = Process.spawn(test_command, "#{/mingw|mswin/ =~ RUBY_PLATFORM ? 'new_' : ''}pgroup": true) - {nil => first_timeout, INT: 30, TERM: 10, KILL: nil}.each do |sig, sec| + p test_command + timeouts = {nil => first_timeout, INT: 30, TERM: 10, KILL: nil} + if /mingw|mswin/ =~ RUBY_PLATFORM + timeouts.delete(:TERM) # Inner process signal on Windows + group = :new_pgroup + pg = "" + else + group = :pgroup + pg = "-" + end + pid = Process.spawn(*test_command, group => true) + timeouts.each do |sig, sec| if sig puts "Sending #{sig} signal" - Process.kill("-#{sig}", pid) + Process.kill("#{pg}#{sig}", pid) end begin break Timeout.timeout(sec) {Process.wait(pid)} @@ -83,7 +114,7 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| end rescue Interrupt exit_code = Signal.list["INT"] - Process.kill("-KILL", pid) + Process.kill("#{pg}KILL", pid) Process.wait(pid) break end @@ -96,11 +127,11 @@ File.foreach("#{gem_dir}/bundled_gems") do |line| "with exit code #{$?.exitstatus}") puts colorize.decorate(mesg, "fail") if allowed_failures.include?(gem) - mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES" + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" puts colorize.decorate(mesg, "skip") else failed << gem - exit_code = $?.exitstatus if $?.exitstatus + exit_code = 1 end end end diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb index 055577feea..28ef0bf7f8 100644 --- a/tool/test-coverage.rb +++ b/tool/test-coverage.rb @@ -114,6 +114,10 @@ pid = $$ pwd = Dir.pwd at_exit do + # Some tests leave GC.stress enabled, causing slow coverage processing. + # Reset it here to avoid performance issues. + GC.stress = false + exit_exc = $! Dir.chdir(pwd) do diff --git a/tool/test/init.rb b/tool/test/init.rb index 3a1143d01d..3fd1419a9c 100644 --- a/tool/test/init.rb +++ b/tool/test/init.rb @@ -1,7 +1,15 @@ -# This file includes the settings for "make test-all". +# This file includes the settings for "make test-all" and "make test-tool". # Note that this file is loaded not only by test/runner.rb but also by tool/lib/test/unit/parallel.rb. -ENV["GEM_SKIP"] = ENV["GEM_HOME"] = ENV["GEM_PATH"] = "".freeze +# Prevent test-all from using bundled gems +["GEM_HOME", "GEM_PATH"].each do |gem_env| + # Preserve the gem environment prepared by tool/runruby.rb for test-tool, which uses bundled gems. + ENV["BUNDLED_#{gem_env}"] = ENV[gem_env] + + ENV[gem_env] = "".freeze +end +ENV["GEM_SKIP"] = "".freeze + ENV.delete("RUBY_CODESIGN") Warning[:experimental] = false diff --git a/tool/test/test_commit_email.rb b/tool/test/test_commit_email.rb new file mode 100644 index 0000000000..db441584fd --- /dev/null +++ b/tool/test/test_commit_email.rb @@ -0,0 +1,102 @@ +require 'test/unit' +require 'shellwords' +require 'tmpdir' +require 'fileutils' +require 'open3' + +class TestCommitEmail < Test::Unit::TestCase + STDIN_DELIMITER = "---\n" + + def setup + omit 'git command is not available' unless system('git', '--version', out: File::NULL, err: File::NULL) + + @ruby = Dir.mktmpdir + Dir.chdir(@ruby) do + git('init', '--initial-branch=master') + git('config', 'user.name', 'Jóhän Grübél') + git('config', 'user.email', 'johan@example.com') + env = { + 'GIT_AUTHOR_DATE' => '2025-10-08T12:00:00Z', + 'GIT_CONFIG_GLOBAL' => @ruby + "/gitconfig", + 'TZ' => 'UTC', + } + git('commit', '--allow-empty', '-m', 'New repository initialized by cvs2svn.', env:) + git('commit', '--allow-empty', '-m', 'Initial revision', env:) + git('commit', '--allow-empty', '-m', 'version 1.0.0', env:) + end + + @sendmail = File.join(Dir.mktmpdir, 'sendmail') + File.write(@sendmail, <<~SENDMAIL, mode: "wx", perm: 0755) + #!/bin/sh + echo #{STDIN_DELIMITER.chomp.dump} + exec cat + SENDMAIL + + @commit_email = File.expand_path('../../tool/commit-email.rb', __dir__) + end + + def teardown + # Clean up temporary files if #setup was not omitted + if @sendmail + File.unlink(@sendmail) + Dir.rmdir(File.dirname(@sendmail)) + end + if @ruby + FileUtils.rm_rf(@ruby) + end + end + + def test_sendmail_encoding + omit 'the sendmail script does not work on windows' if windows? + + Dir.chdir(@ruby) do + before_rev = git('rev-parse', 'HEAD^').chomp + after_rev = git('rev-parse', 'HEAD').chomp + short_rev = after_rev[0...10] + + out, _, status = EnvUtil.invoke_ruby([ + { 'SENDMAIL' => @sendmail, 'TZ' => 'UTC' }.merge!(gem_env), + @commit_email, './', 'cvs-admin@ruby-lang.org', + before_rev, after_rev, 'refs/heads/master', + '--viewer-uri', 'https://github.com/ruby/ruby/commit/', + '--error-to', 'cvs-admin@ruby-lang.org', + ], '', true) + stdin = out.b.split(STDIN_DELIMITER.b, 2).last.force_encoding('UTF-8') + + assert_true(status.success?) + assert_equal(stdin, <<~EOS) + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: =?UTF-8?B?SsOzaMOkbiBHcsO8YsOpbA==?= <noreply@ruby-lang.org> + To: cvs-admin@ruby-lang.org + Subject: #{short_rev} (master): =?UTF-8?B?dmVyc2lvbuOAgDEuMC4w?= + J=C3=B3h=C3=A4n Gr=C3=BCb=C3=A9l\t2025-10-08 12:00:00 +0000 (Wed, 08 Oct 2= + 025) + + New Revision: #{short_rev} + + https://github.com/ruby/ruby/commit/#{short_rev} + + Log: + version=E3=80=801.0.0= + EOS + end + end + + private + + # Resurrect the gem environment preserved by tool/test/init.rb. + # This should work as long as you have run `make up` or `make install`. + def gem_env + { 'GEM_PATH' => ENV['BUNDLED_GEM_PATH'], 'GEM_HOME' => ENV['BUNDLED_GEM_HOME'] } + end + + def git(*cmd, env: {}) + out, status = Open3.capture2(env, 'git', *cmd) + unless status.success? + raise "git #{cmd.shelljoin}\n#{out}" + end + out + end +end diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb index e64c6c6fda..252687f3f3 100755 --- a/tool/test/test_sync_default_gems.rb +++ b/tool/test/test_sync_default_gems.rb @@ -2,6 +2,7 @@ require 'test/unit' require 'stringio' require 'tmpdir' +require 'rubygems/version' require_relative '../sync_default_gems' module Test_SyncDefaultGems @@ -19,14 +20,8 @@ module Test_SyncDefaultGems expected.concat(trailers.map {_1+"\n"}) end - out, err = capture_output do - SyncDefaultGems.message_filter(repo, sha, input: StringIO.new(input, "r")) - end - - all_assertions do |a| - a.for("error") {assert_empty err} - a.for("result") {assert_pattern_list(expected, out)} - end + out = SyncDefaultGems.message_filter(repo, sha, input) + assert_pattern_list(expected, out) end def test_subject_only @@ -90,14 +85,41 @@ module Test_SyncDefaultGems @target = nil pend "No git" unless system("git --version", out: IO::NULL) @testdir = Dir.mktmpdir("sync") - @git_config = %W"HOME GIT_CONFIG_GLOBAL".each_with_object({}) {|k, c| c[k] = ENV[k]} + user, email = "Ruby", "test@ruby-lang.org" + @git_config = %W"HOME USER GIT_CONFIG_GLOBAL GNUPGHOME".each_with_object({}) {|k, c| c[k] = ENV[k]} ENV["HOME"] = @testdir + ENV["USER"] = user + ENV["GNUPGHOME"] = @testdir + '/.gnupg' + expire = EnvUtil.apply_timeout_scale(30).to_i + # Generate a new unprotected key with default parameters that + # expires after 30 seconds. + if @gpgsign = system(*%w"gpg --quiet --batch --passphrase", "", + "--quick-generate-key", email, *%W"default default seconds=#{expire}", + err: IO::NULL) + # Fetch the generated public key. + signingkey = IO.popen(%W"gpg --quiet --list-public-key #{email}", &:read)[/^pub .*\n +\K\h+/] + end ENV["GIT_CONFIG_GLOBAL"] = @testdir + "/gitconfig" - git(*%W"config --global user.email test@ruby-lang.org") - git(*%W"config --global user.name", "Ruby") + git(*%W"config --global user.email", email) + git(*%W"config --global user.name", user) git(*%W"config --global init.defaultBranch default") + if signingkey + git(*%W"config --global user.signingkey", signingkey) + git(*%W"config --global commit.gpgsign true") + git(*%W"config --global gpg.program gpg") + git(*%W"config --global log.showSignature true") + end @target = "sync-test" - SyncDefaultGems::REPOSITORIES[@target] = ["ruby/#{@target}", "default"] + SyncDefaultGems::REPOSITORIES[@target] = SyncDefaultGems.repo( + ["ruby/#{@target}", "default"], + [ + ["lib", "lib"], + ["test", "test"], + ], + exclude: [ + "test/fixtures/*", + ], + ) @sha = {} @origdir = Dir.pwd Dir.chdir(@testdir) @@ -129,6 +151,9 @@ module Test_SyncDefaultGems def teardown if @target + if @gpgsign + system(*%W"gpgconf --kill all") + end Dir.chdir(@origdir) SyncDefaultGems::REPOSITORIES.delete(@target) ENV.update(@git_config) @@ -168,7 +193,7 @@ module Test_SyncDefaultGems end def top_commit(dir, format: "%H") - IO.popen(%W[git log --format=#{format} -1], chdir: dir, &:read)&.chomp + IO.popen(%W[git log --no-show-signature --format=#{format} -1], chdir: dir, &:read)&.chomp end def assert_sync(commits = true, success: true, editor: nil) @@ -293,5 +318,56 @@ module Test_SyncDefaultGems assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) assert_not_operator(File, :exist?, "src/lib/bad.rb", out) end - end + + def test_squash_merge + # This test is known to fail with git 2.43.0, which is used by Ubuntu 24.04. + # We don't know which exact version fixed it, but we know git 2.52.0 works. + stdout, status = Open3.capture2('git', '--version', err: File::NULL) + omit 'git version check failed' unless status.success? + git_version = stdout[/\Agit version \K\S+/] + omit "git #{git_version} is too old" if Gem::Version.new(git_version) < Gem::Version.new('2.44.0') + + # 2---. <- branch + # / \ + # 1---3---3'<- merge commit with conflict resolution + File.write("#@target/lib/conflict.rb", "# 1\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict.rb", chdir: @target) + + git(*%W"checkout -q -b branch", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 2\n") + File.write("#@target/lib/new.rb", "# new\n") + git(*%W"add lib/conflict.rb lib/new.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in branch", chdir: @target) + + git(*%W"checkout -q default", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 3\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in default", chdir: @target) + + # How can I suppress "Auto-merging ..." message from git merge? + git(*%W"merge -X ours -m", "Merge commit", "branch", chdir: @target, out: IO::NULL) + + out = assert_sync() + assert_equal("# 3\n", File.read("src/lib/conflict.rb"), out) + subject, body = top_commit("src", format: "%B").split("\n\n", 2) + assert_equal("[ruby/#@target] Merge commit", subject, out) + assert_includes(body, "Commit in branch", out) + end + + def test_no_upstream_file + group = SyncDefaultGems::Repository.group(%w[ + lib/un.rb + lib/unicode_normalize/normalize.rb + lib/unicode_normalize/tables.rb + lib/net/https.rb + ]) + expected = { + "un" => %w[lib/un.rb], + "net-http" => %w[lib/net/https.rb], + nil => %w[lib/unicode_normalize/normalize.rb lib/unicode_normalize/tables.rb], + } + assert_equal(expected, group) + end + end if /darwin|linux/ =~ RUBY_PLATFORM end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index 709b495572..d9bdc8f3c5 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -8,6 +8,8 @@ class TestAssertion < Test::Unit::TestCase end def test_timeout_separately + pend "hang-up" if /mswin|mingw/ =~ RUBY_PLATFORM + assert_raise(Timeout::Error) do assert_separately([], <<~"end;", timeout: 0.1) sleep @@ -15,6 +17,29 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assertion_count_separately + beginning = self._assertions + + assert_separately([], "") + assertions_at_nothing = self._assertions - beginning + + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "assert true") + assert_equal(1, self._assertions - prev_assertions) + + omit unless Process.respond_to?(:fork) + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "Process.fork {assert true}; assert true") + assert_equal(2, self._assertions - prev_assertions) + + prev_assertions = self._assertions + assertions_at_nothing + # TODO: assertions before `fork` are counted twice; it is possible + # to reset `_assertions` at `Process._fork`, but the hook can + # interfere in other tests. + assert_separately([], "assert true; Process.fork {assert true}") + assert_equal(3, self._assertions - prev_assertions) + end + def return_in_assert_raise assert_raise(RuntimeError) do return @@ -27,6 +52,38 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assert_raise_with_message + my_error = Class.new(StandardError) + + assert_raise_with_message(my_error, "with message") do + raise my_error, "with message" + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(RuntimeError, "with message") do + raise my_error, "with message" + end + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(my_error, "without message") do + raise my_error, "with message" + end + end + end + + def test_assert_raise_kind_of + my_error = Class.new(StandardError) + + assert_raise_kind_of(my_error) do + raise my_error + end + + assert_raise_kind_of(StandardError) do + raise my_error + end + end + def test_assert_pattern_list assert_pattern_list([/foo?/], "foo") assert_not_pattern_list([/foo?/], "afoo") @@ -50,4 +107,122 @@ class TestAssertion < Test::Unit::TestCase assert_pattern_list(pattern_list, actual, message) end end + + def test_caller_bactrace_location + begin + line = __LINE__; assert_fail_for_backtrace_location + rescue Test::Unit::AssertionFailedError => e + end + location = Test::Unit::Runner.new.location(e) + assert_equal "#{__FILE__}:#{line}", location + end + + def assert_fail_for_backtrace_location + assert false + end + + VersionClass = Struct.new(:version) do + def version?(*ver) + Test::Unit::CoreAssertions.version_match?(ver, self.version) + end + end + + V14_6_0 = VersionClass.new([14, 6, 0]) + V15_0_0 = VersionClass.new([15, 0, 0]) + + def test_version_match_integer + assert_not_operator(V14_6_0, :version?, 13) + assert_operator(V14_6_0, :version?, 14) + assert_not_operator(V14_6_0, :version?, 15) + assert_not_operator(V15_0_0, :version?, 14) + assert_operator(V15_0_0, :version?, 15) + end + + def test_version_match_integer_range + assert_operator(V14_6_0, :version?, 13..14) + assert_not_operator(V15_0_0, :version?, 13..14) + assert_not_operator(V14_6_0, :version?, 13...14) + assert_not_operator(V15_0_0, :version?, 13...14) + end + + def test_version_match_array_range + assert_operator(V14_6_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V14_6_0, :version?, [14, 0]...[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[14, 6]) + assert_operator(V14_6_0, :version?, [14, 0]..[15]) + assert_operator(V15_0_0, :version?, [14, 0]..[15]) + assert_operator(V14_6_0, :version?, [14, 0]...[15]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[15]) + end + + def test_version_match_integer_endless_range + assert_operator(V14_6_0, :version?, 14..) + assert_operator(V15_0_0, :version?, 14..) + assert_not_operator(V14_6_0, :version?, 15..) + assert_operator(V15_0_0, :version?, 15..) + end + + def test_version_match_integer_endless_range_exclusive + assert_operator(V14_6_0, :version?, 14...) + assert_operator(V15_0_0, :version?, 14...) + assert_not_operator(V14_6_0, :version?, 15...) + assert_operator(V15_0_0, :version?, 15...) + end + + def test_version_match_array_endless_range + assert_operator(V14_6_0, :version?, [14, 5]..) + assert_operator(V15_0_0, :version?, [14, 5]..) + assert_not_operator(V14_6_0, :version?, [14, 7]..) + assert_operator(V15_0_0, :version?, [14, 7]..) + assert_not_operator(V14_6_0, :version?, [15]..) + assert_operator(V15_0_0, :version?, [15]..) + assert_not_operator(V14_6_0, :version?, [15, 0]..) + assert_operator(V15_0_0, :version?, [15, 0]..) + end + + def test_version_match_array_endless_range_exclude_end + assert_operator(V14_6_0, :version?, [14, 5]...) + assert_operator(V15_0_0, :version?, [14, 5]...) + assert_not_operator(V14_6_0, :version?, [14, 7]...) + assert_operator(V15_0_0, :version?, [14, 7]...) + assert_not_operator(V14_6_0, :version?, [15]...) + assert_operator(V15_0_0, :version?, [15]...) + assert_not_operator(V14_6_0, :version?, [15, 0]...) + assert_operator(V15_0_0, :version?, [15, 0]...) + end + + def test_version_match_integer_beginless_range + assert_operator(V14_6_0, :version?, ..14) + assert_not_operator(V15_0_0, :version?, ..14) + assert_operator(V14_6_0, :version?, ..15) + assert_operator(V15_0_0, :version?, ..15) + + assert_not_operator(V14_6_0, :version?, ...14) + assert_not_operator(V15_0_0, :version?, ...14) + assert_operator(V14_6_0, :version?, ...15) + assert_not_operator(V15_0_0, :version?, ...15) + end + + def test_version_match_array_beginless_range + assert_not_operator(V14_6_0, :version?, ..[14, 5]) + assert_not_operator(V15_0_0, :version?, ..[14, 5]) + assert_operator(V14_6_0, :version?, ..[14, 6]) + assert_not_operator(V15_0_0, :version?, ..[14, 6]) + assert_operator(V14_6_0, :version?, ..[15]) + assert_operator(V15_0_0, :version?, ..[15]) + assert_operator(V14_6_0, :version?, ..[15, 0]) + assert_operator(V15_0_0, :version?, ..[15, 0]) + end + + def test_version_match_array_beginless_range_exclude_end + assert_not_operator(V14_6_0, :version?, ...[14, 5]) + assert_not_operator(V15_0_0, :version?, ...[14, 5]) + assert_not_operator(V14_6_0, :version?, ...[14, 6]) + assert_not_operator(V15_0_0, :version?, ...[14, 6]) + assert_operator(V14_6_0, :version?, ...[15]) + assert_not_operator(V15_0_0, :version?, ...[15]) + assert_operator(V14_6_0, :version?, ...[15, 0]) + assert_not_operator(V15_0_0, :version?, ...[15, 0]) + end end diff --git a/tool/test/testunit/test_hideskip.rb b/tool/test/testunit/test_hideskip.rb index 0c4c9b40f2..a470368bca 100644 --- a/tool/test/testunit/test_hideskip.rb +++ b/tool/test/testunit/test_hideskip.rb @@ -6,7 +6,6 @@ class TestHideSkip < Test::Unit::TestCase assert_not_match(/^ *1\) Skipped/, hideskip) assert_match(/^ *1\) Skipped.*^ *2\) Skipped/m, hideskip("--show-skip")) output = hideskip("--hide-skip") - output.gsub!(/Successful RJIT finish\n/, '') if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? assert_match(/assertions\/s.\n+2 tests, 0 assertions, 0 failures, 0 errors, 2 skips/, output) end diff --git a/tool/test/testunit/test_launchable.rb b/tool/test/testunit/test_launchable.rb new file mode 100644 index 0000000000..76be876456 --- /dev/null +++ b/tool/test/testunit/test_launchable.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: false +require 'test/unit' +require 'tempfile' +require 'json' +require_relative '../../lib/launchable' + +class TestLaunchable < Test::Unit::TestCase + def test_json_stream_writer + Tempfile.create(['launchable-test-', '.json']) do |f| + json_stream_writer = Launchable::JsonStreamWriter.new(f.path) + json_stream_writer.write_array('testCases') + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 42, + status: "TEST_FAILED", + stdout: nil, + stderr: nil, + createdAt: "2021-10-05T12:34:00", + data: { + lineNumber: 1 + } + } + ) + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 45, + status: "TEST_PASSED", + stdout: "This is stdout", + stderr: "This is stderr", + createdAt: "2021-10-05T12:36:00", + data: { + lineNumber: 10 + } + } + ) + json_stream_writer.close() + expected = <<JSON +{ + "testCases": [ + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 42, + "status": "TEST_FAILED", + "stdout": null, + "stderr": null, + "createdAt": "2021-10-05T12:34:00", + "data": { + "lineNumber": 1 + } + }, + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 45, + "status": "TEST_PASSED", + "stdout": "This is stdout", + "stderr": "This is stderr", + "createdAt": "2021-10-05T12:36:00", + "data": { + "lineNumber": 10 + } + } + ] +} +JSON + assert_equal(expected, f.read) + end + end +end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb index f79c3a1d80..adf7d62ecd 100644 --- a/tool/test/testunit/test_parallel.rb +++ b/tool/test/testunit/test_parallel.rb @@ -6,7 +6,15 @@ module TestParallel PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" TESTS = "#{__dir__}/tests_for_parallel" # use large timeout for --jit-wait - TIMEOUT = EnvUtil.apply_timeout_scale(defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? ? 100 : 30) + TIMEOUT = EnvUtil.apply_timeout_scale(100) + + def self.timeout(n, &blk) + start_time = Time.now + Timeout.timeout(n, &blk) + rescue Timeout::Error + end_time = Time.now + raise Timeout::Error, "execution expired (start: #{ start_time }, end: #{ end_time })" + end class TestParallelWorker < Test::Unit::TestCase def setup @@ -22,10 +30,10 @@ module TestParallel if @worker_pid && @worker_in begin begin - @worker_in.puts "quit" + @worker_in.puts "quit normal" rescue IOError, Errno::EPIPE end - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@worker_pid) end rescue Timeout::Error @@ -45,7 +53,7 @@ module TestParallel end def test_run - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) @@ -58,7 +66,7 @@ module TestParallel end def test_run_multiple_testcase_in_one_file - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_second.rb test" assert_match(/^okay/,@worker_out.gets) @@ -75,7 +83,7 @@ module TestParallel end def test_accept_run_command_multiple_times - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) @@ -99,7 +107,7 @@ module TestParallel end def test_p - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_first.rb test" while buf = @worker_out.gets break if /^p (.+?)$/ =~ buf @@ -110,7 +118,7 @@ module TestParallel end def test_done - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_forth.rb test" while buf = @worker_out.gets break if /^done (.+?)$/ =~ buf @@ -118,34 +126,34 @@ module TestParallel assert_not_nil($1, "'done' was not found") result = Marshal.load($1.chomp.unpack1("m")) - assert_equal(5, result[0]) - pend "TODO: result[1] returns 17. We should investigate it" do - assert_equal(12, result[1]) - end - assert_kind_of(Array,result[2]) - assert_kind_of(Array,result[3]) - assert_kind_of(Array,result[4]) - assert_kind_of(Array,result[2][1]) - assert_kind_of(Test::Unit::AssertionFailedError,result[2][0][2]) - assert_kind_of(Test::Unit::PendedError,result[2][1][2]) - assert_kind_of(Test::Unit::PendedError,result[2][2][2]) - assert_kind_of(Exception, result[2][3][2]) - assert_equal(result[5], "TestE") + tests, asserts, reports, failures, loadpaths, suite = result + assert_equal(5, tests) + assert_equal(12, asserts) + assert_kind_of(Array, reports) + assert_kind_of(Array, failures) + assert_kind_of(Array, loadpaths) + reports.sort_by! {|_, t| t} + assert_kind_of(Array, reports[1]) + assert_kind_of(Test::Unit::AssertionFailedError, reports[0][2]) + assert_kind_of(Test::Unit::PendedError, reports[1][2]) + assert_kind_of(Test::Unit::PendedError, reports[2][2]) + assert_kind_of(Exception, reports[3][2]) + assert_equal("TestE", suite) end end def test_quit - Timeout.timeout(TIMEOUT) do - @worker_in.puts "quit" + ::TestParallel.timeout(TIMEOUT) do + @worker_in.puts "quit normal" assert_match(/^bye$/m,@worker_out.read) end end end class TestParallel < Test::Unit::TestCase - def spawn_runner(*opt_args, jobs: "t1") + def spawn_runner(*opt_args, jobs: "t1", env: {}) @test_out, o = IO.pipe - @test_pid = spawn(*@__runner_options__[:ruby], TESTS+"/runner.rb", + @test_pid = spawn(env, *@__runner_options__[:ruby], TESTS+"/runner.rb", "--ruby", @__runner_options__[:ruby].join(" "), "-j", jobs, *opt_args, out: o, err: o) o.close @@ -154,7 +162,7 @@ module TestParallel def teardown begin if @test_pid - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@test_pid) end end @@ -167,54 +175,47 @@ module TestParallel def test_ignore_jzero spawn_runner(jobs: "0") - Timeout.timeout(TIMEOUT) { + ::TestParallel.timeout(TIMEOUT) { assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) } end def test_should_run_all_without_any_leaks spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^9 tests/,buf) end def test_should_retry_failed_on_workers - spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner "--retry" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying\.+$/,buf) end def test_no_retry_option spawn_runner "--no-retry" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} refute_match(/^Retrying\.+$/,buf) assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf) end def test_jobs_status spawn_runner "--jobs-status" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/\d+=ptest_(first|second|third|forth) */,buf) end def test_separate # this test depends to --jobs-status spawn_runner "--jobs-status", "--separate" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1, message("retried tests should run in different processes") {buf}) end def test_hungup - spawn_runner "--worker-timeout=1", "test4test_hungup.rb" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} - assert_match(/^Retrying hung up testcases\.+$/, buf) - assert_match(/^2 tests,.* 0 failures,/, buf) - end - - def test_retry_workers - spawn_runner "--worker-timeout=1", "test4test_slow_0.rb", "test4test_slow_1.rb", jobs: "2" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner("--worker-timeout=1", "--retry", "test4test_hungup.rb", env: {"RUBY_CRASH_REPORT"=>nil}) + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying hung up testcases\.+$/, buf) assert_match(/^2 tests,.* 0 failures,/, buf) end diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb index 8831676e19..54474c828d 100644 --- a/tool/test/testunit/tests_for_parallel/ptest_forth.rb +++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb @@ -8,19 +8,19 @@ class TestE < Test::Unit::TestCase assert_equal(1,1) end - def test_always_skip - skip "always" + def test_always_omit + omit "always" end def test_always_fail assert_equal(0,1) end - def test_skip_after_unknown_error + def test_pend_after_unknown_error begin raise UnknownError, "unknown error" rescue - skip "after raise" + pend "after raise" end end diff --git a/tool/test/testunit/tests_for_parallel/slow_helper.rb b/tool/test/testunit/tests_for_parallel/slow_helper.rb index d8372730a8..38067c1f47 100644 --- a/tool/test/testunit/tests_for_parallel/slow_helper.rb +++ b/tool/test/testunit/tests_for_parallel/slow_helper.rb @@ -2,6 +2,7 @@ require 'test/unit' module TestSlowTimeout def test_slow - sleep (ENV['sec'] || 3).to_i if on_parallel_worker? + sleep_for = EnvUtil.apply_timeout_scale((ENV['sec'] || 3).to_i) + sleep sleep_for if on_parallel_worker? end end diff --git a/tool/test/testunit/tests_for_parallel/test4test_hungup.rb b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb index 65a75f7c4d..49f503ba9e 100644 --- a/tool/test/testunit/tests_for_parallel/test4test_hungup.rb +++ b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb @@ -8,7 +8,7 @@ class TestHung < Test::Unit::TestCase def test_hungup_at_worker if on_parallel_worker? - sleep 10 + sleep EnvUtil.apply_timeout_scale(10) end assert true end diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb deleted file mode 100644 index a749b0e1d3..0000000000 --- a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative 'slow_helper' - -class TestSlowV0 < Test::Unit::TestCase - include TestSlowTimeout -end diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb deleted file mode 100644 index 924a3b11fa..0000000000 --- a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative 'slow_helper' - -class TestSlowV1 < Test::Unit::TestCase - include TestSlowTimeout -end diff --git a/tool/test/webrick/.htaccess b/tool/test/webrick/.htaccess deleted file mode 100644 index 69d4659b9f..0000000000 --- a/tool/test/webrick/.htaccess +++ /dev/null @@ -1 +0,0 @@ -this file should not be published. diff --git a/tool/test/webrick/test_cgi.rb b/tool/test/webrick/test_cgi.rb deleted file mode 100644 index a9be8f353d..0000000000 --- a/tool/test/webrick/test_cgi.rb +++ /dev/null @@ -1,148 +0,0 @@ -# coding: US-ASCII -# frozen_string_literal: false -require_relative "utils" -require "webrick" -require "test/unit" - -class TestWEBrickCGI < Test::Unit::TestCase - CRLF = "\r\n" - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_cgi - TestWEBrick.start_cgi_server{|server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.cgi") - http.request(req){|res| assert_equal("/webrick.cgi", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi/path/info") - http.request(req){|res| assert_equal("/path/info", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi/%3F%3F%3F?foo=bar") - http.request(req){|res| assert_equal("/???", res.body, log.call)} - unless RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32|java/ - # Path info of res.body is passed via ENV. - # ENV[] returns different value on Windows depending on locale. - req = Net::HTTP::Get.new("/webrick.cgi/%A4%DB%A4%B2/%A4%DB%A4%B2") - http.request(req){|res| - assert_equal("/\xA4\xDB\xA4\xB2/\xA4\xDB\xA4\xB2", res.body, log.call)} - end - req = Net::HTTP::Get.new("/webrick.cgi?a=1;a=2;b=x") - http.request(req){|res| assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi?a=1&a=2&b=x") - http.request(req){|res| assert_equal("a=1, a=2, b=x", res.body, log.call)} - - req = Net::HTTP::Post.new("/webrick.cgi?a=x;a=y;b=1") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req, "a=1;a=2;b=x"){|res| - assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Post.new("/webrick.cgi?a=x&a=y&b=1") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req, "a=1&a=2&b=x"){|res| - assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Get.new("/") - http.request(req){|res| - ary = res.body.lines.to_a - assert_match(%r{/$}, ary[0], log.call) - assert_match(%r{/webrick.cgi$}, ary[1], log.call) - } - - req = Net::HTTP::Get.new("/webrick.cgi") - req["Cookie"] = "CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001" - http.request(req){|res| - assert_equal( - "CUSTOMER=WILE_E_COYOTE\nPART_NUMBER=ROCKET_LAUNCHER_0001\n", - res.body, log.call) - } - - req = Net::HTTP::Get.new("/webrick.cgi") - cookie = %{$Version="1"; } - cookie << %{Customer="WILE_E_COYOTE"; $Path="/acme"; } - cookie << %{Part_Number="Rocket_Launcher_0001"; $Path="/acme"; } - cookie << %{Shipping="FedEx"; $Path="/acme"} - req["Cookie"] = cookie - http.request(req){|res| - assert_equal("Customer=WILE_E_COYOTE, Shipping=FedEx", - res["Set-Cookie"], log.call) - assert_equal("Customer=WILE_E_COYOTE\n" + - "Part_Number=Rocket_Launcher_0001\n" + - "Shipping=FedEx\n", res.body, log.call) - } - } - end - - def test_bad_request - log_tester = lambda {|log, access_log| - assert_match(/BadRequest/, log.join) - } - TestWEBrick.start_cgi_server({}, log_tester) {|server, addr, port, log| - sock = TCPSocket.new(addr, port) - begin - sock << "POST /webrick.cgi HTTP/1.0" << CRLF - sock << "Content-Type: application/x-www-form-urlencoded" << CRLF - sock << "Content-Length: 1024" << CRLF - sock << CRLF - sock << "a=1&a=2&b=x" - sock.close_write - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, sock.read, log.call) - ensure - sock.close - end - } - end - - def test_cgi_env - TestWEBrick.start_cgi_server do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.cgi/dumpenv") - req['proxy'] = 'http://example.com/' - req['hello'] = 'world' - http.request(req) do |res| - env = Marshal.load(res.body) - assert_equal 'world', env['HTTP_HELLO'] - assert_not_operator env, :include?, 'HTTP_PROXY' - end - end - end - - CtrlSeq = [0x7f, *(1..31)].pack("C*").gsub(/\s+/, '') - CtrlPat = /#{Regexp.quote(CtrlSeq)}/o - DumpPat = /#{Regexp.quote(CtrlSeq.dump[1...-1])}/o - - def test_bad_uri - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR bad URI/, log[0]) - } - TestWEBrick.start_cgi_server({}, log_tester) {|server, addr, port, log| - res = TCPSocket.open(addr, port) {|sock| - sock << "GET /#{CtrlSeq}#{CRLF}#{CRLF}" - sock.close_write - sock.read - } - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, res) - s = log.call.each_line.grep(/ERROR bad URI/)[0] - assert_match(DumpPat, s) - assert_not_match(CtrlPat, s) - } - end - - def test_bad_header - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR bad header/, log[0]) - } - TestWEBrick.start_cgi_server({}, log_tester) {|server, addr, port, log| - res = TCPSocket.open(addr, port) {|sock| - sock << "GET / HTTP/1.0#{CRLF}#{CtrlSeq}#{CRLF}#{CRLF}" - sock.close_write - sock.read - } - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, res) - s = log.call.each_line.grep(/ERROR bad header/)[0] - assert_match(DumpPat, s) - assert_not_match(CtrlPat, s) - } - end -end diff --git a/tool/test/webrick/test_config.rb b/tool/test/webrick/test_config.rb deleted file mode 100644 index a54a667452..0000000000 --- a/tool/test/webrick/test_config.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/config" - -class TestWEBrickConfig < Test::Unit::TestCase - def test_server_name_default - config = WEBrick::Config::General.dup - assert_equal(false, config.key?(:ServerName)) - assert_equal(WEBrick::Utils.getservername, config[:ServerName]) - assert_equal(true, config.key?(:ServerName)) - end - - def test_server_name_set_nil - config = WEBrick::Config::General.dup.update(ServerName: nil) - assert_equal(nil, config[:ServerName]) - end -end diff --git a/tool/test/webrick/test_cookie.rb b/tool/test/webrick/test_cookie.rb deleted file mode 100644 index e46185f127..0000000000 --- a/tool/test/webrick/test_cookie.rb +++ /dev/null @@ -1,141 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/cookie" - -class TestWEBrickCookie < Test::Unit::TestCase - def test_new - cookie = WEBrick::Cookie.new("foo","bar") - assert_equal("foo", cookie.name) - assert_equal("bar", cookie.value) - assert_equal("foo=bar", cookie.to_s) - end - - def test_time - cookie = WEBrick::Cookie.new("foo","bar") - t = 1000000000 - cookie.max_age = t - assert_match(t.to_s, cookie.to_s) - - cookie = WEBrick::Cookie.new("foo","bar") - t = Time.at(1000000000) - cookie.expires = t - assert_equal(Time, cookie.expires.class) - assert_equal(t, cookie.expires) - ts = t.httpdate - cookie.expires = ts - assert_equal(Time, cookie.expires.class) - assert_equal(t, cookie.expires) - assert_match(ts, cookie.to_s) - end - - def test_parse - data = "" - data << '$Version="1"; ' - data << 'Customer="WILE_E_COYOTE"; $Path="/acme"; ' - data << 'Part_Number="Rocket_Launcher_0001"; $Path="/acme"; ' - data << 'Shipping="FedEx"; $Path="/acme"' - cookies = WEBrick::Cookie.parse(data) - assert_equal(3, cookies.size) - assert_equal(1, cookies[0].version) - assert_equal("Customer", cookies[0].name) - assert_equal("WILE_E_COYOTE", cookies[0].value) - assert_equal("/acme", cookies[0].path) - assert_equal(1, cookies[1].version) - assert_equal("Part_Number", cookies[1].name) - assert_equal("Rocket_Launcher_0001", cookies[1].value) - assert_equal(1, cookies[2].version) - assert_equal("Shipping", cookies[2].name) - assert_equal("FedEx", cookies[2].value) - - data = "hoge=moge; __div__session=9865ecfd514be7f7" - cookies = WEBrick::Cookie.parse(data) - assert_equal(2, cookies.size) - assert_equal(0, cookies[0].version) - assert_equal("hoge", cookies[0].name) - assert_equal("moge", cookies[0].value) - assert_equal("__div__session", cookies[1].name) - assert_equal("9865ecfd514be7f7", cookies[1].value) - - # don't allow ,-separator - data = "hoge=moge, __div__session=9865ecfd514be7f7" - cookies = WEBrick::Cookie.parse(data) - assert_equal(1, cookies.size) - assert_equal(0, cookies[0].version) - assert_equal("hoge", cookies[0].name) - assert_equal("moge, __div__session=9865ecfd514be7f7", cookies[0].value) - end - - def test_parse_no_whitespace - data = [ - '$Version="1"; ', - 'Customer="WILE_E_COYOTE";$Path="/acme";', # no SP between cookie-string - 'Part_Number="Rocket_Launcher_0001";$Path="/acme";', # no SP between cookie-string - 'Shipping="FedEx";$Path="/acme"' - ].join - cookies = WEBrick::Cookie.parse(data) - assert_equal(1, cookies.size) - end - - def test_parse_too_much_whitespaces - # According to RFC6265, - # cookie-string = cookie-pair *( ";" SP cookie-pair ) - # So single 0x20 is needed after ';'. We allow multiple spaces here for - # compatibility with older WEBrick versions. - data = [ - '$Version="1"; ', - 'Customer="WILE_E_COYOTE";$Path="/acme"; ', # no SP between cookie-string - 'Part_Number="Rocket_Launcher_0001";$Path="/acme"; ', # no SP between cookie-string - 'Shipping="FedEx";$Path="/acme"' - ].join - cookies = WEBrick::Cookie.parse(data) - assert_equal(3, cookies.size) - end - - def test_parse_set_cookie - data = %(Customer="WILE_E_COYOTE"; Version="1"; Path="/acme") - cookie = WEBrick::Cookie.parse_set_cookie(data) - assert_equal("Customer", cookie.name) - assert_equal("WILE_E_COYOTE", cookie.value) - assert_equal(1, cookie.version) - assert_equal("/acme", cookie.path) - - data = %(Shipping="FedEx"; Version="1"; Path="/acme"; Secure) - cookie = WEBrick::Cookie.parse_set_cookie(data) - assert_equal("Shipping", cookie.name) - assert_equal("FedEx", cookie.value) - assert_equal(1, cookie.version) - assert_equal("/acme", cookie.path) - assert_equal(true, cookie.secure) - end - - def test_parse_set_cookies - data = %(Shipping="FedEx"; Version="1"; Path="/acme"; Secure) - data << %(, CUSTOMER=WILE_E_COYOTE; path=/; expires=Wednesday, 09-Nov-99 23:12:40 GMT; path=/; Secure) - data << %(, name="Aaron"; Version="1"; path="/acme") - cookies = WEBrick::Cookie.parse_set_cookies(data) - assert_equal(3, cookies.length) - - fed_ex = cookies.find { |c| c.name == 'Shipping' } - assert_not_nil(fed_ex) - assert_equal("Shipping", fed_ex.name) - assert_equal("FedEx", fed_ex.value) - assert_equal(1, fed_ex.version) - assert_equal("/acme", fed_ex.path) - assert_equal(true, fed_ex.secure) - - name = cookies.find { |c| c.name == 'name' } - assert_not_nil(name) - assert_equal("name", name.name) - assert_equal("Aaron", name.value) - assert_equal(1, name.version) - assert_equal("/acme", name.path) - - customer = cookies.find { |c| c.name == 'CUSTOMER' } - assert_not_nil(customer) - assert_equal("CUSTOMER", customer.name) - assert_equal("WILE_E_COYOTE", customer.value) - assert_equal(0, customer.version) - assert_equal("/", customer.path) - assert_equal(Time.utc(1999, 11, 9, 23, 12, 40), customer.expires) - end -end diff --git a/tool/test/webrick/test_do_not_reverse_lookup.rb b/tool/test/webrick/test_do_not_reverse_lookup.rb deleted file mode 100644 index efcb5a9299..0000000000 --- a/tool/test/webrick/test_do_not_reverse_lookup.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick" -require_relative "utils" - -class TestDoNotReverseLookup < Test::Unit::TestCase - class DNRL < WEBrick::GenericServer - def run(sock) - sock << sock.do_not_reverse_lookup.to_s - end - end - - @@original_do_not_reverse_lookup_value = Socket.do_not_reverse_lookup - - def teardown - Socket.do_not_reverse_lookup = @@original_do_not_reverse_lookup_value - end - - def do_not_reverse_lookup?(config) - result = nil - TestWEBrick.start_server(DNRL, config) do |server, addr, port, log| - TCPSocket.open(addr, port) do |sock| - result = {'true' => true, 'false' => false}[sock.gets] - end - end - result - end - - # +--------------------------------------------------------------------------+ - # | Expected interaction between Socket.do_not_reverse_lookup | - # | and WEBrick::Config::General[:DoNotReverseLookup] | - # +----------------------------+---------------------------------------------+ - # | |WEBrick::Config::General[:DoNotReverseLookup]| - # +----------------------------+--------------+---------------+--------------+ - # |Socket.do_not_reverse_lookup| TRUE | FALSE | NIL | - # +----------------------------+--------------+---------------+--------------+ - # | TRUE | true | false | true | - # +----------------------------+--------------+---------------+--------------+ - # | FALSE | true | false | false | - # +----------------------------+--------------+---------------+--------------+ - - def test_socket_dnrl_true_server_dnrl_true - Socket.do_not_reverse_lookup = true - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => true)) - end - - def test_socket_dnrl_true_server_dnrl_false - Socket.do_not_reverse_lookup = true - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => false)) - end - - def test_socket_dnrl_true_server_dnrl_nil - Socket.do_not_reverse_lookup = true - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => nil)) - end - - def test_socket_dnrl_false_server_dnrl_true - Socket.do_not_reverse_lookup = false - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => true)) - end - - def test_socket_dnrl_false_server_dnrl_false - Socket.do_not_reverse_lookup = false - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => false)) - end - - def test_socket_dnrl_false_server_dnrl_nil - Socket.do_not_reverse_lookup = false - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => nil)) - end -end diff --git a/tool/test/webrick/test_filehandler.rb b/tool/test/webrick/test_filehandler.rb deleted file mode 100644 index 452667d4f4..0000000000 --- a/tool/test/webrick/test_filehandler.rb +++ /dev/null @@ -1,397 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require_relative "utils.rb" -require "webrick" -require "stringio" -require "tmpdir" - -class WEBrick::TestFileHandler < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def default_file_handler(filename) - klass = WEBrick::HTTPServlet::DefaultFileHandler - klass.new(WEBrick::Config::HTTP, filename) - end - - def windows? - File.directory?("\\") - end - - def get_res_body(res) - sio = StringIO.new - sio.binmode - res.send_body(sio) - sio.string - end - - def make_range_request(range_spec) - msg = <<-END_OF_REQUEST - GET / HTTP/1.0 - Range: #{range_spec} - - END_OF_REQUEST - return StringIO.new(msg.gsub(/^ {6}/, "")) - end - - def make_range_response(file, range_spec) - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(make_range_request(range_spec)) - res = WEBrick::HTTPResponse.new(WEBrick::Config::HTTP) - size = File.size(file) - handler = default_file_handler(file) - handler.make_partial_content(req, res, file, size) - return res - end - - def test_make_partial_content - filename = __FILE__ - filesize = File.size(filename) - - res = make_range_response(filename, "bytes=#{filesize-100}-") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=-100") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-99") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=100-199") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-0") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(1, get_res_body(res).size) - - res = make_range_response(filename, "bytes=-1") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(1, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-0, -2") - assert_match(%r{^multipart/byteranges}, res["content-type"]) - body = get_res_body(res) - boundary = /; boundary=(.+)/.match(res['content-type'])[1] - off = filesize - 2 - last = filesize - 1 - - exp = "--#{boundary}\r\n" \ - "Content-Type: text/plain\r\n" \ - "Content-Range: bytes 0-0/#{filesize}\r\n" \ - "\r\n" \ - "#{File.read(__FILE__, 1)}\r\n" \ - "--#{boundary}\r\n" \ - "Content-Type: text/plain\r\n" \ - "Content-Range: bytes #{off}-#{last}/#{filesize}\r\n" \ - "\r\n" \ - "#{File.read(__FILE__, 2, off)}\r\n" \ - "--#{boundary}--\r\n" - assert_equal exp, body - end - - def test_filehandler - config = { :DocumentRoot => File.dirname(__FILE__), } - this_file = File.basename(__FILE__) - filesize = File.size(__FILE__) - this_data = File.binread(__FILE__) - range = nil - bug2593 = '[ruby-dev:40030]' - - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - begin - server[:DocumentRootOptions][:NondisclosureName] = [] - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/html", res.content_type, log.call) - assert_match(/HREF="#{this_file}"/, res.body, log.call) - } - req = Net::HTTP::Get.new("/#{this_file}") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_equal(this_data, res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=#{filesize-100}-") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-100)..(filesize-1), range, log.call) - assert_equal(this_data[-100..-1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=-100") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-100)..(filesize-1), range, log.call) - assert_equal(this_data[-100..-1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-99") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(0..99, range, log.call) - assert_equal(this_data[0..99], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=100-199") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(100..199, range, log.call) - assert_equal(this_data[100..199], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-0") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(0..0, range, log.call) - assert_equal(this_data[0..0], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=-1") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-1)..(filesize-1), range, log.call) - assert_equal(this_data[-1, 1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-0, -2") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("multipart/byteranges", res.content_type, log.call) - } - ensure - server[:DocumentRootOptions].delete :NondisclosureName - end - end - end - - def test_non_disclosure_name - config = { :DocumentRoot => File.dirname(__FILE__), } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } - assert_equal([], log) - } - this_file = File.basename(__FILE__) - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - doc_root_opts = server[:DocumentRootOptions] - doc_root_opts[:NondisclosureName] = %w(.ht* *~ test_*) - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/html", res.content_type, log.call) - assert_no_match(/HREF="#{File.basename(__FILE__)}"/, res.body) - } - req = Net::HTTP::Get.new("/#{this_file}") - http.request(req){|res| - assert_equal("404", res.code, log.call) - } - doc_root_opts[:NondisclosureName] = %w(.ht* *~ TEST_*) - http.request(req){|res| - assert_equal("404", res.code, log.call) - } - end - end - - def test_directory_traversal - return if File.executable?(__FILE__) # skip on strange file system - - config = { :DocumentRoot => File.dirname(__FILE__), } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR bad URI/ =~ s } - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/../../") - http.request(req){|res| assert_equal("400", res.code, log.call) } - req = Net::HTTP::Get.new("/..%5c../#{File.basename(__FILE__)}") - http.request(req){|res| assert_equal(windows? ? "200" : "404", res.code, log.call) } - req = Net::HTTP::Get.new("/..%5c..%5cruby.c") - http.request(req){|res| assert_equal("404", res.code, log.call) } - end - end - - def test_unwise_in_path - if windows? - config = { :DocumentRoot => File.dirname(__FILE__), } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/..%5c..") - http.request(req){|res| assert_equal("301", res.code, log.call) } - end - end - end - - def test_short_filename - return if File.executable?(__FILE__) # skip on strange file system - - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } - assert_equal([], log) - } - TestWEBrick.start_cgi_server({}, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - if windows? - root = File.dirname(__FILE__).tr("/", "\\") - fname = IO.popen(%W[dir /x #{root}\\webrick_long_filename.cgi], encoding: "binary", &:read) - fname.sub!(/\A.*$^$.*$^$/m, '') - if fname - fname = fname[/\s(w.+?cgi)\s/i, 1] - fname.downcase! - end - else - fname = "webric~1.cgi" - end - req = Net::HTTP::Get.new("/#{fname}/test") - http.request(req) do |res| - if windows? - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - else - assert_equal("404", res.code, log.call) - end - end - - req = Net::HTTP::Get.new("/.htaccess") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - req = Net::HTTP::Get.new("/htacce~1") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - req = Net::HTTP::Get.new("/HTACCE~1") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - end - end - - def test_multibyte_char_in_path - if Encoding.default_external == Encoding.find('US-ASCII') - reset_encoding = true - verb = $VERBOSE - $VERBOSE = false - Encoding.default_external = Encoding.find('UTF-8') - end - - c = "\u00a7" - begin - c = c.encode('filesystem') - rescue EncodingError - c = c.b - end - Dir.mktmpdir(c) do |dir| - basename = "#{c}.txt" - File.write("#{dir}/#{basename}", "test_multibyte_char_in_path") - Dir.mkdir("#{dir}/#{c}") - File.write("#{dir}/#{c}/#{basename}", "nested") - config = { - :DocumentRoot => dir, - :DirectoryIndex => [basename], - } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - path = "/#{basename}" - req = Net::HTTP::Get.new(WEBrick::HTTPUtils::escape(path)) - http.request(req){|res| assert_equal("200", res.code, log.call + "\nFilesystem encoding is #{Encoding.find('filesystem')}") } - path = "/#{c}/#{basename}" - req = Net::HTTP::Get.new(WEBrick::HTTPUtils::escape(path)) - http.request(req){|res| assert_equal("200", res.code, log.call) } - req = Net::HTTP::Get.new('/') - http.request(req){|res| - assert_equal("test_multibyte_char_in_path", res.body, log.call) - } - end - end - ensure - if reset_encoding - Encoding.default_external = Encoding.find('US-ASCII') - $VERBOSE = verb - end - end - - def test_script_disclosure - return if File.executable?(__FILE__) # skip on strange file system - - config = { - :CGIInterpreter => TestWEBrick::RubyBinArray, - :DocumentRoot => File.dirname(__FILE__), - :CGIPathEnv => ENV['PATH'], - :RequestCallback => Proc.new{|req, res| - def req.meta_vars - meta = super - meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) - meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] - return meta - end - }, - } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - http.read_timeout = EnvUtil.apply_timeout_scale(60) - http.write_timeout = EnvUtil.apply_timeout_scale(60) if http.respond_to?(:write_timeout=) - - req = Net::HTTP::Get.new("/webrick.cgi/test") - http.request(req) do |res| - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - end - - resok = windows? - response_assertion = Proc.new do |res| - if resok - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - else - assert_equal("404", res.code, log.call) - end - end - req = Net::HTTP::Get.new("/webrick.cgi%20/test") - http.request(req, &response_assertion) - req = Net::HTTP::Get.new("/webrick.cgi./test") - http.request(req, &response_assertion) - resok &&= File.exist?(__FILE__+"::$DATA") - req = Net::HTTP::Get.new("/webrick.cgi::$DATA/test") - http.request(req, &response_assertion) - end - end - - def test_erbhandler - config = { :DocumentRoot => File.dirname(__FILE__) } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.rhtml") - http.request(req) do |res| - assert_equal("200", res.code, log.call) - assert_match %r!\Areq to http://[^/]+/webrick\.rhtml {}\n!, res.body - end - end - end -end diff --git a/tool/test/webrick/test_htgroup.rb b/tool/test/webrick/test_htgroup.rb deleted file mode 100644 index 8749711df5..0000000000 --- a/tool/test/webrick/test_htgroup.rb +++ /dev/null @@ -1,19 +0,0 @@ -require "tempfile" -require "test/unit" -require "webrick/httpauth/htgroup" - -class TestHtgroup < Test::Unit::TestCase - def test_htgroup - Tempfile.create('test_htgroup') do |tmpfile| - tmpfile.close - tmp_group = WEBrick::HTTPAuth::Htgroup.new(tmpfile.path) - tmp_group.add 'superheroes', %w[spiderman batman] - tmp_group.add 'supervillains', %w[joker] - tmp_group.flush - - htgroup = WEBrick::HTTPAuth::Htgroup.new(tmpfile.path) - assert_equal(htgroup.members('superheroes'), %w[spiderman batman]) - assert_equal(htgroup.members('supervillains'), %w[joker]) - end - end -end diff --git a/tool/test/webrick/test_htmlutils.rb b/tool/test/webrick/test_htmlutils.rb deleted file mode 100644 index ae1b8efa95..0000000000 --- a/tool/test/webrick/test_htmlutils.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/htmlutils" - -class TestWEBrickHTMLUtils < Test::Unit::TestCase - include WEBrick::HTMLUtils - - def test_escape - assert_equal("foo", escape("foo")) - assert_equal("foo bar", escape("foo bar")) - assert_equal("foo&bar", escape("foo&bar")) - assert_equal("foo"bar", escape("foo\"bar")) - assert_equal("foo>bar", escape("foo>bar")) - assert_equal("foo<bar", escape("foo<bar")) - assert_equal("\u{3053 3093 306B 3061 306F}", escape("\u{3053 3093 306B 3061 306F}")) - bug8425 = '[Bug #8425] [ruby-core:55052]' - assert_nothing_raised(ArgumentError, Encoding::CompatibilityError, bug8425) { - assert_equal("\u{3053 3093 306B}\xff<", escape("\u{3053 3093 306B}\xff<")) - } - end -end diff --git a/tool/test/webrick/test_httpauth.rb b/tool/test/webrick/test_httpauth.rb deleted file mode 100644 index 9fe8af8be2..0000000000 --- a/tool/test/webrick/test_httpauth.rb +++ /dev/null @@ -1,366 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "tempfile" -require "webrick" -require "webrick/httpauth/basicauth" -require "stringio" -require_relative "utils" - -class TestWEBrickHTTPAuth < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_basic_auth - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, log[0]) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth" - - server.mount_proc(path){|req, res| - WEBrick::HTTPAuth.basic_auth(req, res, realm){|user, pass| - user == "webrick" && pass == "supersecretpassword" - } - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("webrick", "supersecretpassword") - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - g.basic_auth("webrick", "not super") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - } - end - - def test_basic_auth_sha - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.puts("webrick:{SHA}GJYFRpBbdchp595jlh3Bhfmgp8k=") - tmpfile.flush - assert_raise(NotImplementedError){ - WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path) - } - } - end - - def test_basic_auth_md5 - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.puts("webrick:$apr1$IOVMD/..$rmnOSPXr0.wwrLPZHBQZy0") - tmpfile.flush - assert_raise(NotImplementedError){ - WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path) - } - } - end - - [nil, :crypt, :bcrypt].each do |hash_algo| - # OpenBSD does not support insecure DES-crypt - next if /openbsd/ =~ RUBY_PLATFORM && hash_algo != :bcrypt - - begin - case hash_algo - when :crypt - # require 'string/crypt' - when :bcrypt - require 'bcrypt' - end - rescue LoadError - next - end - - define_method(:"test_basic_auth_htpasswd_#{hash_algo}") do - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Basic WEBrick's realm: webrick: password unmatch\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth2" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htpasswd = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - users = [] - htpasswd.each{|user, pass| users << user } - assert_equal(2, users.size, log.call) - assert(users.member?("webrick"), log.call) - assert(users.member?("foo"), log.call) - - server.mount_proc(path){|req, res| - auth = WEBrick::HTTPAuth::BasicAuth.new( - :Realm => realm, :UserDB => htpasswd, - :Logger => server.logger - ) - auth.authenticate(req, res) - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("webrick", "supersecretpassword") - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - g.basic_auth("webrick", "not super") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - } - } - end - - define_method(:"test_basic_auth_bad_username_htpasswd_#{hash_algo}") do - log_tester = lambda {|log, access_log| - assert_equal(2, log.length) - assert_match(/ERROR Basic WEBrick's realm: foo\\ebar: the user is not allowed\./, log[0]) - assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, log[1]) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htpasswd = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - users = [] - htpasswd.each{|user, pass| users << user } - server.mount_proc(path){|req, res| - auth = WEBrick::HTTPAuth::BasicAuth.new( - :Realm => realm, :UserDB => htpasswd, - :Logger => server.logger - ) - auth.authenticate(req, res) - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("foo\ebar", "passwd") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call) } - } - } - end - end - - DIGESTRES_ = / - ([a-zA-Z\-]+) - [ \t]*(?:\r\n[ \t]*)* - = - [ \t]*(?:\r\n[ \t]*)* - (?: - "((?:[^"]+|\\[\x00-\x7F])*)" | - ([!\#$%&'*+\-.0-9A-Z^_`a-z|~]+) - )/x - - def test_digest_auth - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Digest WEBrick's realm: no credentials in the request\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/, - /ERROR Digest WEBrick's realm: webrick: digest unmatch\./ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/digest_auth" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htdigest = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - users = [] - htdigest.each{|user, pass| users << user } - assert_equal(2, users.size, log.call) - assert(users.member?("webrick"), log.call) - assert(users.member?("foo"), log.call) - - auth = WEBrick::HTTPAuth::DigestAuth.new( - :Realm => realm, :UserDB => htdigest, - :Algorithm => 'MD5', - :Logger => server.logger - ) - server.mount_proc(path){|req, res| - auth.authenticate(req, res) - res.body = "hoge" - } - - Net::HTTP.start(addr, port) do |http| - g = Net::HTTP::Get.new(path) - params = {} - http.request(g) do |res| - assert_equal('401', res.code, log.call) - res["www-authenticate"].scan(DIGESTRES_) do |key, quoted, token| - params[key.downcase] = token || quoted.delete('\\') - end - params['uri'] = "http://#{addr}:#{port}#{path}" - end - - g['Authorization'] = credentials_for_request('webrick', "supersecretpassword", params) - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - - params['algorithm'].downcase! #4936 - g['Authorization'] = credentials_for_request('webrick', "supersecretpassword", params) - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - - g['Authorization'] = credentials_for_request('webrick', "not super", params) - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - end - } - } - end - - def test_digest_auth_int - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Digest wb auth-int realm: no credentials in the request\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/, - /ERROR Digest wb auth-int realm: foo: digest unmatch\./ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "wb auth-int realm" - path = "/digest_auth_int" - - Tempfile.create("test_webrick_auth_int") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - tmp_pass.set_passwd(realm, "foo", "Hunter2") - tmp_pass.flush - - htdigest = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - users = [] - htdigest.each{|user, pass| users << user } - assert_equal %w(foo), users - - auth = WEBrick::HTTPAuth::DigestAuth.new( - :Realm => realm, :UserDB => htdigest, - :Algorithm => 'MD5', - :Logger => server.logger, - :Qop => %w(auth-int), - ) - server.mount_proc(path){|req, res| - auth.authenticate(req, res) - res.body = "bbb" - } - Net::HTTP.start(addr, port) do |http| - post = Net::HTTP::Post.new(path) - params = {} - data = 'hello=world' - body = StringIO.new(data) - post.content_length = data.bytesize - post['Content-Type'] = 'application/x-www-form-urlencoded' - post.body_stream = body - - http.request(post) do |res| - assert_equal('401', res.code, log.call) - res["www-authenticate"].scan(DIGESTRES_) do |key, quoted, token| - params[key.downcase] = token || quoted.delete('\\') - end - params['uri'] = "http://#{addr}:#{port}#{path}" - end - - body.rewind - cred = credentials_for_request('foo', 'Hunter3', params, body) - post['Authorization'] = cred - post.body_stream = body - http.request(post){|res| - assert_equal('401', res.code, log.call) - assert_not_equal("bbb", res.body, log.call) - } - - body.rewind - cred = credentials_for_request('foo', 'Hunter2', params, body) - post['Authorization'] = cred - post.body_stream = body - http.request(post){|res| assert_equal("bbb", res.body, log.call)} - end - } - } - end - - def test_digest_auth_invalid - digest_auth = WEBrick::HTTPAuth::DigestAuth.new(Realm: 'realm', UserDB: '') - - def digest_auth.error(fmt, *) - end - - def digest_auth.try_bad_request(len) - request = {"Authorization" => %[Digest a="#{'\b'*len}]} - authenticate request, nil - end - - bad_request = WEBrick::HTTPStatus::BadRequest - t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - assert_raise(bad_request) {digest_auth.try_bad_request(10)} - limit = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) - [20, 50, 100, 200].each do |len| - assert_raise(bad_request) do - Timeout.timeout(len*limit) {digest_auth.try_bad_request(len)} - end - end - end - - private - def credentials_for_request(user, password, params, body = nil) - cnonce = "hoge" - nonce_count = 1 - ha1 = "#{user}:#{params['realm']}:#{password}" - if body - dig = Digest::MD5.new - while buf = body.read(16384) - dig.update(buf) - end - body.rewind - ha2 = "POST:#{params['uri']}:#{dig.hexdigest}" - else - ha2 = "GET:#{params['uri']}" - end - - request_digest = - "#{Digest::MD5.hexdigest(ha1)}:" \ - "#{params['nonce']}:#{'%08x' % nonce_count}:#{cnonce}:#{params['qop']}:" \ - "#{Digest::MD5.hexdigest(ha2)}" - "Digest username=\"#{user}\"" \ - ", realm=\"#{params['realm']}\"" \ - ", nonce=\"#{params['nonce']}\"" \ - ", uri=\"#{params['uri']}\"" \ - ", qop=#{params['qop']}" \ - ", nc=#{'%08x' % nonce_count}" \ - ", cnonce=\"#{cnonce}\"" \ - ", response=\"#{Digest::MD5.hexdigest(request_digest)}\"" \ - ", opaque=\"#{params['opaque']}\"" \ - ", algorithm=#{params['algorithm']}" - end -end diff --git a/tool/test/webrick/test_httpproxy.rb b/tool/test/webrick/test_httpproxy.rb deleted file mode 100644 index 66dae6f6f6..0000000000 --- a/tool/test/webrick/test_httpproxy.rb +++ /dev/null @@ -1,467 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require "webrick/httpproxy" -begin - require "webrick/ssl" - require "net/https" -rescue LoadError - # test_connect will be skipped -end -require File.expand_path("utils.rb", File.dirname(__FILE__)) - -class TestWEBrickHTTPProxy < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_fake_proxy - assert_nil(WEBrick::FakeProxyURI.scheme) - assert_nil(WEBrick::FakeProxyURI.host) - assert_nil(WEBrick::FakeProxyURI.port) - assert_nil(WEBrick::FakeProxyURI.path) - assert_nil(WEBrick::FakeProxyURI.userinfo) - assert_raise(NoMethodError){ WEBrick::FakeProxyURI.foo } - end - - def test_proxy - # Testing GET or POST to the proxy server - # Note that the proxy server works as the origin server. - # +------+ - # V | - # client -------> proxy ---+ - # GET / POST GET / POST - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - http = Net::HTTP.new(addr, port, addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_equal("GET / ", res.body, log.call) - } - assert_equal(1, proxy_handler_called, log.call) - assert_equal(2, request_handler_called, log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_nil(res.body, log.call) - } - assert_equal(2, proxy_handler_called, log.call) - assert_equal(4, request_handler_called, log.call) - - req = Net::HTTP::Post.new("/") - req.body = "post-data" - req.content_type = "application/x-www-form-urlencoded" - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_equal("POST / post-data", res.body, log.call) - } - assert_equal(3, proxy_handler_called, log.call) - assert_equal(6, request_handler_called, log.call) - } - end - - def test_no_proxy - # Testing GET or POST to the proxy server without proxy request. - # - # client -------> proxy - # GET / POST - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - http = Net::HTTP.new(addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_equal("GET / ", res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(1, request_handler_called, log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_nil(res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(2, request_handler_called, log.call) - - req = Net::HTTP::Post.new("/") - req.content_type = "application/x-www-form-urlencoded" - req.body = "post-data" - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_equal("POST / post-data", res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(3, request_handler_called, log.call) - } - end - - def test_big_bodies - require 'digest/md5' - rand_str = File.read(__FILE__) - rand_str.freeze - nr = 1024 ** 2 / rand_str.size # bigger works, too - exp = Digest::MD5.new - nr.times { exp.update(rand_str) } - exp = exp.hexdigest - TestWEBrick.start_httpserver do |o_server, o_addr, o_port, o_log| - o_server.mount_proc('/') do |req, res| - case req.request_method - when 'GET' - res['content-type'] = 'application/octet-stream' - if req.path == '/length' - res['content-length'] = (nr * rand_str.size).to_s - else - res.chunked = true - end - res.body = ->(socket) { nr.times { socket.write(rand_str) } } - when 'POST' - dig = Digest::MD5.new - req.body { |buf| dig.update(buf); buf.clear } - res['content-type'] = 'text/plain' - res['content-length'] = '32' - res.body = dig.hexdigest - end - end - - http = Net::HTTP.new(o_addr, o_port) - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Transfer-Encoding' => 'chunked', - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - - TestWEBrick.start_httpproxy do |p_server, p_addr, p_port, p_log| - http = Net::HTTP.new(o_addr, o_port, p_addr, p_port) - http.request_get('/length') do |res| - assert_equal(nr * rand_str.size, res.content_length) - dig = Digest::MD5.new - res.read_body { |buf| dig.update(buf); buf.clear } - assert_equal exp, dig.hexdigest - end - http.request_get('/') do |res| - assert_predicate res, :chunked? - dig = Digest::MD5.new - res.read_body { |buf| dig.update(buf); buf.clear } - assert_equal exp, dig.hexdigest - end - - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Content-Length' => (nr * rand_str.size).to_s, - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Transfer-Encoding' => 'chunked', - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - end - end - end if RUBY_VERSION >= '2.5' - - def test_http10_proxy_chunked - # Testing HTTP/1.0 client request and HTTP/1.1 chunked response - # from origin server. - # +------+ - # V | - # client -------> proxy ---+ - # GET GET - # HTTP/1.0 HTTP/1.1 - # non-chunked chunked - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - log_tester = lambda {|log, access_log| - log.reject! {|str| - %r{WARN chunked is set for an HTTP/1\.0 request\. \(ignored\)} =~ str - } - assert_equal([], log) - } - TestWEBrick.start_httpproxy(config, log_tester){|server, addr, port, log| - body = nil - server.mount_proc("/"){|req, res| - body = "#{req.request_method} #{req.path} #{req.body}" - res.chunked = true - res.body = -> (socket) { body.each_char {|c| socket.write c } } - } - - # Don't use Net::HTTP because it uses HTTP/1.1. - TCPSocket.open(addr, port) {|s| - s.write "GET / HTTP/1.0\r\nHost: localhost.localdomain\r\n\r\n" - response = s.read - assert_equal(body, response[/.*\z/]) - } - } - end - - def make_certificate(key, cn) - subject = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=#{cn}") - exts = [ - ["keyUsage", "keyEncipherment,digitalSignature", true], - ] - cert = OpenSSL::X509::Certificate.new - cert.version = 2 - cert.serial = 1 - cert.subject = subject - cert.issuer = subject - cert.public_key = key - cert.not_before = Time.now - 3600 - cert.not_after = Time.now + 3600 - ef = OpenSSL::X509::ExtensionFactory.new(cert, cert) - exts.each {|args| cert.add_extension(ef.create_extension(*args)) } - cert.sign(key, "sha256") - return cert - end if defined?(OpenSSL::SSL) - - def test_connect - # Testing CONNECT to proxy server - # - # client -----------> proxy -----------> https - # 1. CONNECT establish TCP - # 2. ---- establish SSL session ---> - # 3. ------- GET or POST ----------> - # - key = TEST_KEY_RSA2048 - cert = make_certificate(key, "127.0.0.1") - s_config = { - :SSLEnable =>true, - :ServerName => "localhost", - :SSLCertificate => cert, - :SSLPrivateKey => key, - } - config = { - :ServerName => "localhost.localdomain", - :RequestCallback => Proc.new{|req, res| - assert_equal("CONNECT", req.request_method) - }, - } - TestWEBrick.start_httpserver(s_config){|s_server, s_addr, s_port, s_log| - s_server.mount_proc("/"){|req, res| - res.body = "SSL #{req.request_method} #{req.path} #{req.body}" - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - http = Net::HTTP.new("127.0.0.1", s_port, addr, port) - http.use_ssl = true - http.verify_callback = Proc.new do |preverify_ok, store_ctx| - store_ctx.current_cert.to_der == cert.to_der - end - - req = Net::HTTP::Get.new("/") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req){|res| - assert_equal("SSL GET / ", res.body, s_log.call + log.call) - } - - req = Net::HTTP::Post.new("/") - req["Content-Type"] = "application/x-www-form-urlencoded" - req.body = "post-data" - http.request(req){|res| - assert_equal("SSL POST / post-data", res.body, s_log.call + log.call) - } - } - } - end if defined?(OpenSSL::SSL) - - def test_upstream_proxy - return if /mswin/ =~ RUBY_PLATFORM && ENV.key?('GITHUB_ACTIONS') # not working from the beginning - # Testing GET or POST through the upstream proxy server - # Note that the upstream proxy server works as the origin server. - # +------+ - # V | - # client -------> proxy -------> proxy ---+ - # GET / POST GET / POST GET / POST - # - up_proxy_handler_called = up_request_handler_called = 0 - proxy_handler_called = request_handler_called = 0 - up_config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| up_proxy_handler_called += 1}, - :RequestCallback => Proc.new{|req, res| up_request_handler_called += 1} - } - TestWEBrick.start_httpproxy(up_config){|up_server, up_addr, up_port, up_log| - up_server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - config = { - :ServerName => "localhost.localdomain", - :ProxyURI => URI.parse("http://localhost:#{up_port}"), - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1}, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1}, - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - http = Net::HTTP.new(up_addr, up_port, addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - skip res.message unless res.code == '200' - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_equal("GET / ", res.body) - } - assert_equal(1, up_proxy_handler_called, up_log.call + log.call) - assert_equal(2, up_request_handler_called, up_log.call + log.call) - assert_equal(1, proxy_handler_called, up_log.call + log.call) - assert_equal(1, request_handler_called, up_log.call + log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_nil(res.body, up_log.call + log.call) - } - assert_equal(2, up_proxy_handler_called, up_log.call + log.call) - assert_equal(4, up_request_handler_called, up_log.call + log.call) - assert_equal(2, proxy_handler_called, up_log.call + log.call) - assert_equal(2, request_handler_called, up_log.call + log.call) - - req = Net::HTTP::Post.new("/") - req.body = "post-data" - req.content_type = "application/x-www-form-urlencoded" - http.request(req){|res| - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_equal("POST / post-data", res.body, up_log.call + log.call) - } - assert_equal(3, up_proxy_handler_called, up_log.call + log.call) - assert_equal(6, up_request_handler_called, up_log.call + log.call) - assert_equal(3, proxy_handler_called, up_log.call + log.call) - assert_equal(3, request_handler_called, up_log.call + log.call) - - if defined?(OpenSSL::SSL) - # Testing CONNECT to the upstream proxy server - # - # client -------> proxy -------> proxy -------> https - # 1. CONNECT CONNECT establish TCP - # 2. -------- establish SSL session ------> - # 3. ---------- GET or POST --------------> - # - key = TEST_KEY_RSA2048 - cert = make_certificate(key, "127.0.0.1") - s_config = { - :SSLEnable =>true, - :ServerName => "localhost", - :SSLCertificate => cert, - :SSLPrivateKey => key, - } - TestWEBrick.start_httpserver(s_config){|s_server, s_addr, s_port, s_log| - s_server.mount_proc("/"){|req2, res| - res.body = "SSL #{req2.request_method} #{req2.path} #{req2.body}" - } - http = Net::HTTP.new("127.0.0.1", s_port, addr, port, up_log.call + log.call + s_log.call) - http.use_ssl = true - http.verify_callback = Proc.new do |preverify_ok, store_ctx| - store_ctx.current_cert.to_der == cert.to_der - end - - req2 = Net::HTTP::Get.new("/") - http.request(req2){|res| - assert_equal("SSL GET / ", res.body, up_log.call + log.call + s_log.call) - } - - req2 = Net::HTTP::Post.new("/") - req2.body = "post-data" - req2.content_type = "application/x-www-form-urlencoded" - http.request(req2){|res| - assert_equal("SSL POST / post-data", res.body, up_log.call + log.call + s_log.call) - } - } - end - } - } - end - - if defined?(OpenSSL::SSL) - TEST_KEY_RSA2048 = OpenSSL::PKey.read <<-_end_of_pem_ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAuV9ht9J7k4NBs38jOXvvTKY9gW8nLICSno5EETR1cuF7i4pN -s9I1QJGAFAX0BEO4KbzXmuOvfCpD3CU+Slp1enenfzq/t/e/1IRW0wkJUJUFQign -4CtrkJL+P07yx18UjyPlBXb81ApEmAB5mrJVSrWmqbjs07JbuS4QQGGXLc+Su96D -kYKmSNVjBiLxVVSpyZfAY3hD37d60uG+X8xdW5v68JkRFIhdGlb6JL8fllf/A/bl -NwdJOhVr9mESHhwGjwfSeTDPfd8ZLE027E5lyAVX9KZYcU00mOX+fdxOSnGqS/8J -DRh0EPHDL15RcJjV2J6vZjPb0rOYGDoMcH+94wIDAQABAoIBAAzsamqfYQAqwXTb -I0CJtGg6msUgU7HVkOM+9d3hM2L791oGHV6xBAdpXW2H8LgvZHJ8eOeSghR8+dgq -PIqAffo4x1Oma+FOg3A0fb0evyiACyrOk+EcBdbBeLo/LcvahBtqnDfiUMQTpy6V -seSoFCwuN91TSCeGIsDpRjbG1vxZgtx+uI+oH5+ytqJOmfCksRDCkMglGkzyfcl0 -Xc5CUhIJ0my53xijEUQl19rtWdMnNnnkdbG8PT3LZlOta5Do86BElzUYka0C6dUc -VsBDQ0Nup0P6rEQgy7tephHoRlUGTYamsajGJaAo1F3IQVIrRSuagi7+YpSpCqsW -wORqorkCgYEA7RdX6MDVrbw7LePnhyuaqTiMK+055/R1TqhB1JvvxJ1CXk2rDL6G -0TLHQ7oGofd5LYiemg4ZVtWdJe43BPZlVgT6lvL/iGo8JnrncB9Da6L7nrq/+Rvj -XGjf1qODCK+LmreZWEsaLPURIoR/Ewwxb9J2zd0CaMjeTwafJo1CZvcCgYEAyCgb -aqoWvUecX8VvARfuA593Lsi50t4MEArnOXXcd1RnXoZWhbx5rgO8/ATKfXr0BK/n -h2GF9PfKzHFm/4V6e82OL7gu/kLy2u9bXN74vOvWFL5NOrOKPM7Kg+9I131kNYOw -Ivnr/VtHE5s0dY7JChYWE1F3vArrOw3T00a4CXUCgYEA0SqY+dS2LvIzW4cHCe9k -IQqsT0yYm5TFsUEr4sA3xcPfe4cV8sZb9k/QEGYb1+SWWZ+AHPV3UW5fl8kTbSNb -v4ng8i8rVVQ0ANbJO9e5CUrepein2MPL0AkOATR8M7t7dGGpvYV0cFk8ZrFx0oId -U0PgYDotF/iueBWlbsOM430CgYEAqYI95dFyPI5/AiSkY5queeb8+mQH62sdcCCr -vd/w/CZA/K5sbAo4SoTj8dLk4evU6HtIa0DOP63y071eaxvRpTNqLUOgmLh+D6gS -Cc7TfLuFrD+WDBatBd5jZ+SoHccVrLR/4L8jeodo5FPW05A+9gnKXEXsTxY4LOUC -9bS4e1kCgYAqVXZh63JsMwoaxCYmQ66eJojKa47VNrOeIZDZvd2BPVf30glBOT41 -gBoDG3WMPZoQj9pb7uMcrnvs4APj2FIhMU8U15LcPAj59cD6S6rWnAxO8NFK7HQG -4Jxg3JNNf8ErQoCHb1B3oVdXJkmbJkARoDpBKmTCgKtP8ADYLmVPQw== ------END RSA PRIVATE KEY----- - _end_of_pem_ - end -end diff --git a/tool/test/webrick/test_httprequest.rb b/tool/test/webrick/test_httprequest.rb deleted file mode 100644 index 3c0ea937d9..0000000000 --- a/tool/test/webrick/test_httprequest.rb +++ /dev/null @@ -1,488 +0,0 @@ -# frozen_string_literal: false -require "webrick" -require "stringio" -require "test/unit" - -class TestWEBrickHTTPRequest < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_simple_request - msg = <<-_end_of_message_ -GET / - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert(req.meta_vars) # fails if @header was not initialized and iteration is attempted on the nil reference - end - - def test_parse_09 - msg = <<-_end_of_message_ - GET / - foobar # HTTP/0.9 request don't have header nor entity body. - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/", req.unparsed_uri) - assert_equal(WEBrick::HTTPVersion.new("0.9"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(false, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_parse_10 - msg = <<-_end_of_message_ - GET / HTTP/1.0 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/", req.unparsed_uri) - assert_equal(WEBrick::HTTPVersion.new("1.0"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(false, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_parse_11 - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/path", req.unparsed_uri) - assert_equal("", req.script_name) - assert_equal("/path", req.path_info) - assert_equal(WEBrick::HTTPVersion.new("1.1"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(true, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_request_uri_too_large - msg = <<-_end_of_message_ - GET /#{"a"*2084} HTTP/1.1 - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - assert_raise(WEBrick::HTTPStatus::RequestURITooLarge){ - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - } - end - - def test_parse_headers - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: test.ruby-lang.org:8080 - Connection: close - Accept: text/*;q=0.3, text/html;q=0.7, text/html;level=1, - text/html;level=2;q=0.4, */*;q=0.5 - Accept-Encoding: compress;q=0.5 - Accept-Encoding: gzip;q=1.0, identity; q=0.4, *;q=0 - Accept-Language: en;q=0.5, *; q=0 - Accept-Language: ja - Content-Type: text/plain - Content-Length: 7 - X-Empty-Header: - - foobar - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal( - URI.parse("http://test.ruby-lang.org:8080/path"), req.request_uri) - assert_equal("test.ruby-lang.org", req.host) - assert_equal(8080, req.port) - assert_equal(false, req.keep_alive?) - assert_equal( - %w(text/html;level=1 text/html */* text/html;level=2 text/*), - req.accept) - assert_equal(%w(gzip compress identity *), req.accept_encoding) - assert_equal(%w(ja en *), req.accept_language) - assert_equal(7, req.content_length) - assert_equal("text/plain", req.content_type) - assert_equal("foobar\n", req.body) - assert_equal("", req["x-empty-header"]) - assert_equal(nil, req["x-no-header"]) - assert(req.query.empty?) - end - - def test_parse_header2() - msg = <<-_end_of_message_ - POST /foo/bar/../baz?q=a HTTP/1.0 - Content-Length: 9 - User-Agent: - FOO BAR - BAZ - - hogehoge - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("POST", req.request_method) - assert_equal("/foo/baz", req.path) - assert_equal("", req.script_name) - assert_equal("/foo/baz", req.path_info) - assert_equal("9", req['content-length']) - assert_equal("FOO BAR BAZ", req['user-agent']) - assert_equal("hogehoge\n", req.body) - end - - def test_parse_headers3 - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: test.ruby-lang.org - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://test.ruby-lang.org/path"), req.request_uri) - assert_equal("test.ruby-lang.org", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: 192.168.1.1 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://192.168.1.1/path"), req.request_uri) - assert_equal("192.168.1.1", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: [fe80::208:dff:feef:98c7] - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://[fe80::208:dff:feef:98c7]/path"), - req.request_uri) - assert_equal("[fe80::208:dff:feef:98c7]", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: 192.168.1.1:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://192.168.1.1:8080/path"), req.request_uri) - assert_equal("192.168.1.1", req.host) - assert_equal(8080, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: [fe80::208:dff:feef:98c7]:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://[fe80::208:dff:feef:98c7]:8080/path"), - req.request_uri) - assert_equal("[fe80::208:dff:feef:98c7]", req.host) - assert_equal(8080, req.port) - end - - def test_parse_get_params - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - GET /path?#{param} HTTP/1.1 - Host: test.ruby-lang.org:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - query = req.query - assert_equal("1", query["foo"]) - assert_equal(["1", "2", "3"], query["foo"].to_ary) - assert_equal(["1", "2", "3"], query["foo"].list) - assert_equal("x", query["bar"]) - assert_equal(["x"], query["bar"].list) - end - - def test_parse_post_params - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Length: #{param.size} - Content-Type: application/x-www-form-urlencoded - - #{param} - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - query = req.query - assert_equal("1", query["foo"]) - assert_equal(["1", "2", "3"], query["foo"].to_ary) - assert_equal(["1", "2", "3"], query["foo"].list) - assert_equal("x", query["bar"]) - assert_equal(["x"], query["bar"].list) - end - - def test_chunked - crlf = "\x0d\x0a" - expect = File.binread(__FILE__).freeze - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - Host: test.ruby-lang.org:8080 - Transfer-Encoding: chunked - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - File.open(__FILE__){|io| - while chunk = io.read(100) - msg << chunk.size.to_s(16) << crlf - msg << chunk << crlf - end - } - msg << "0" << crlf - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal(expect, req.body) - - # chunked req.body_reader - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - dst = StringIO.new - IO.copy_stream(req.body_reader, dst) - assert_equal(expect, dst.string) - end - - def test_forwarded - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - User-Agent: w3m/0.5.2 - X-Forwarded-For: 123.123.123.123 - X-Forwarded-Host: forward.example.com - X-Forwarded-Server: server.example.com - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("http://forward.example.com/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(80, req.port) - assert_equal("123.123.123.123", req.remote_ip) - assert(!req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - User-Agent: w3m/0.5.2 - X-Forwarded-For: 192.168.1.10, 172.16.1.1, 123.123.123.123 - X-Forwarded-Host: forward.example.com:8080 - X-Forwarded-Server: server.example.com - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("http://forward.example.com:8080/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(8080, req.port) - assert_equal("123.123.123.123", req.remote_ip) - assert(!req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https, http - X-Forwarded-For: 192.168.1.10, 10.0.0.1, 123.123.123.123 - X-Forwarded-Host: forward.example.com - X-Forwarded-Server: server.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("https://forward.example.com/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(443, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: forward1.example.com:1234, forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://forward1.example.com:1234/foo", req.request_uri.to_s) - assert_equal("forward1.example.com", req.host) - assert_equal(1234, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: [fd20:8b1e:b255:8154:250:56ff:fea8:4d84], forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]/foo", req.request_uri.to_s) - assert_equal("[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]", req.host) - assert_equal(443, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: [fd20:8b1e:b255:8154:250:56ff:fea8:4d84]:1234, forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]:1234/foo", req.request_uri.to_s) - assert_equal("[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]", req.host) - assert_equal(1234, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - end - - def test_continue_sent - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - Expect: 100-continue - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert req['expect'] - l = msg.size - req.continue - assert_not_equal l, msg.size - assert_match(/HTTP\/1.1 100 continue\r\n\r\n\z/, msg) - assert !req['expect'] - end - - def test_continue_not_sent - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert !req['expect'] - l = msg.size - req.continue - assert_equal l, msg.size - end - - def test_empty_post - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Type: application/x-www-form-urlencoded - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - end - - def test_bad_messages - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Type: application/x-www-form-urlencoded - - #{param} - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::LengthRequired){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Length: 100000 - - body is too short. - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::BadRequest){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Transfer-Encoding: foobar - - body is too short. - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::NotImplemented){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - end - - def test_eof_raised_when_line_is_nil - assert_raise(WEBrick::HTTPStatus::EOFError) { - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new("")) - } - end -end diff --git a/tool/test/webrick/test_httpresponse.rb b/tool/test/webrick/test_httpresponse.rb deleted file mode 100644 index 4410f63e89..0000000000 --- a/tool/test/webrick/test_httpresponse.rb +++ /dev/null @@ -1,282 +0,0 @@ -# frozen_string_literal: false -require "webrick" -require "test/unit" -require "stringio" -require "net/http" - -module WEBrick - class TestHTTPResponse < Test::Unit::TestCase - class FakeLogger - attr_reader :messages - - def initialize - @messages = [] - end - - def warn msg - @messages << msg - end - end - - attr_reader :config, :logger, :res - - def setup - super - @logger = FakeLogger.new - @config = Config::HTTP - @config[:Logger] = logger - @res = HTTPResponse.new config - @res.keep_alive = true - end - - def test_prevent_response_splitting_headers_crlf - res['X-header'] = "malicious\r\nCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_crlf - user_input = "malicious\r\nCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_headers_cr - res['X-header'] = "malicious\rCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_cr - user_input = "malicious\rCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_headers_lf - res['X-header'] = "malicious\nCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_lf - user_input = "malicious\nCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_set_redirect_response_splitting - url = "malicious\r\nCookie: cracked_indicator_for_test" - assert_raise(URI::InvalidURIError) do - res.set_redirect(WEBrick::HTTPStatus::MultipleChoices, url) - end - end - - def test_set_redirect_html_injection - url = 'http://example.com////?a</a><head></head><body><img src=1></body>' - assert_raise(WEBrick::HTTPStatus::MultipleChoices) do - res.set_redirect(WEBrick::HTTPStatus::MultipleChoices, url) - end - res.status = 300 - io = StringIO.new - res.send_response(io) - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '300', res.code - refute_match(/<img/, io.string) - end - - def test_304_does_not_log_warning - res.status = 304 - res.setup_header - assert_equal 0, logger.messages.length - end - - def test_204_does_not_log_warning - res.status = 204 - res.setup_header - - assert_equal 0, logger.messages.length - end - - def test_1xx_does_not_log_warnings - res.status = 105 - res.setup_header - - assert_equal 0, logger.messages.length - end - - def test_200_chunked_does_not_set_content_length - res.chunked = false - res["Transfer-Encoding"] = 'chunked' - res.setup_header - assert_nil res.header.fetch('content-length', nil) - end - - def test_send_body_io - IO.pipe {|body_r, body_w| - body_w.write 'hello' - body_w.close - - @res.body = body_r - - IO.pipe {|r, w| - - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string - @res.body = 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_io - @res.body = StringIO.new 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_io_chunked - @res.chunked = true - - IO.pipe {|body_r, body_w| - - body_w.write 'hello' - body_w.close - - @res.body = body_r - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_chunked - @res.chunked = true - - @res.body = 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_io_chunked - @res.chunked = true - - @res.body = StringIO.new 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_proc - @res.body = Proc.new { |out| out.write('hello') } - IO.pipe do |r, w| - @res.send_body(w) - w.close - r.binmode - assert_equal 'hello', r.read - end - assert_equal 0, logger.messages.length - end - - def test_send_body_proc_chunked - @res.body = Proc.new { |out| out.write('hello') } - @res.chunked = true - IO.pipe do |r, w| - @res.send_body(w) - w.close - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - end - assert_equal 0, logger.messages.length - end - - def test_set_error - status = 400 - message = 'missing attribute' - @res.status = status - error = WEBrick::HTTPStatus[status].new(message) - body = @res.set_error(error) - assert_match(/#{@res.reason_phrase}/, body) - assert_match(/#{message}/, body) - end - - def test_no_extraneous_space - [200, 300, 400, 500].each do |status| - @res.status = status - assert_match(/\S\r\n/, @res.status_line) - end - end - end -end diff --git a/tool/test/webrick/test_https.rb b/tool/test/webrick/test_https.rb deleted file mode 100644 index ec0aac354a..0000000000 --- a/tool/test/webrick/test_https.rb +++ /dev/null @@ -1,112 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require "webrick/https" -require "webrick/utils" -require_relative "utils" - -class TestWEBrickHTTPS < Test::Unit::TestCase - empty_log = Object.new - def empty_log.<<(str) - assert_equal('', str) - self - end - NoLog = WEBrick::Log.new(empty_log, WEBrick::BasicLog::WARN) - - class HTTPSNITest < ::Net::HTTP - attr_accessor :sni_hostname - - def ssl_socket_connect(s, timeout) - s.hostname = sni_hostname - super - end - end - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def https_get(addr, port, hostname, path, verifyname = nil) - subject = nil - http = HTTPSNITest.new(addr, port) - http.use_ssl = true - http.verify_mode = OpenSSL::SSL::VERIFY_NONE - http.verify_callback = proc { |x, store| subject = store.chain[0].subject.to_s; x } - http.sni_hostname = hostname - req = Net::HTTP::Get.new(path) - req["Host"] = "#{hostname}:#{port}" - response = http.start { http.request(req).body } - assert_equal("/CN=#{verifyname || hostname}", subject) - response - end - - def test_sni - config = { - :ServerName => "localhost", - :SSLEnable => true, - :SSLCertName => "/CN=localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc("/") {|req, res| res.body = "master" } - - # catch stderr in create_self_signed_cert - stderr_buffer = StringIO.new - old_stderr, $stderr = $stderr, stderr_buffer - - begin - vhost_config1 = { - :ServerName => "vhost1", - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :SSLEnable => true, - :SSLCertName => "/CN=vhost1", - } - vhost1 = WEBrick::HTTPServer.new(vhost_config1) - vhost1.mount_proc("/") {|req, res| res.body = "vhost1" } - server.virtual_host(vhost1) - - vhost_config2 = { - :ServerName => "vhost2", - :ServerAlias => ["vhost2alias"], - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :SSLEnable => true, - :SSLCertName => "/CN=vhost2", - } - vhost2 = WEBrick::HTTPServer.new(vhost_config2) - vhost2.mount_proc("/") {|req, res| res.body = "vhost2" } - server.virtual_host(vhost2) - ensure - # restore stderr - $stderr = old_stderr - end - - assert_match(/\A([.+*]+\n)+\z/, stderr_buffer.string) - assert_equal("master", https_get(addr, port, "localhost", "/localhost")) - assert_equal("master", https_get(addr, port, "unknown", "/unknown", "localhost")) - assert_equal("vhost1", https_get(addr, port, "vhost1", "/vhost1")) - assert_equal("vhost2", https_get(addr, port, "vhost2", "/vhost2")) - assert_equal("vhost2", https_get(addr, port, "vhost2alias", "/vhost2alias", "vhost2")) - } - end - - def test_check_ssl_virtual - config = { - :ServerName => "localhost", - :SSLEnable => true, - :SSLCertName => "/CN=localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - assert_raise ArgumentError do - vhost = WEBrick::HTTPServer.new({:DoNotListen => true, :Logger => NoLog}) - server.virtual_host(vhost) - end - } - end -end diff --git a/tool/test/webrick/test_httpserver.rb b/tool/test/webrick/test_httpserver.rb deleted file mode 100644 index f6b53e142b..0000000000 --- a/tool/test/webrick/test_httpserver.rb +++ /dev/null @@ -1,543 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require_relative "utils" - -class TestWEBrickHTTPServer < Test::Unit::TestCase - empty_log = Object.new - def empty_log.<<(str) - assert_equal('', str) - self - end - NoLog = WEBrick::Log.new(empty_log, WEBrick::BasicLog::WARN) - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_mount - httpd = WEBrick::HTTPServer.new( - :Logger => NoLog, - :DoNotListen=>true - ) - httpd.mount("/", :Root) - httpd.mount("/foo", :Foo) - httpd.mount("/foo/bar", :Bar, :bar1) - httpd.mount("/foo/bar/baz", :Baz, :baz1, :baz2) - - serv, opts, script_name, path_info = httpd.search_servlet("/") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/sub") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/sub", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/sub/") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/sub/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/sub") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("/sub", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/bar") - assert_equal(:Bar, serv) - assert_equal([:bar1], opts) - assert_equal("/foo/bar", script_name) - assert_equal("", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/bar/baz") - assert_equal(:Baz, serv) - assert_equal([:baz1, :baz2], opts) - assert_equal("/foo/bar/baz", script_name) - assert_equal("", path_info) - end - - class Req - attr_reader :port, :host - def initialize(addr, port, host) - @addr, @port, @host = addr, port, host - end - def addr - [0,0,0,@addr] - end - end - - def httpd(addr, port, host, ali) - config ={ - :Logger => NoLog, - :DoNotListen => true, - :BindAddress => addr, - :Port => port, - :ServerName => host, - :ServerAlias => ali, - } - return WEBrick::HTTPServer.new(config) - end - - def assert_eql?(v1, v2) - assert_equal(v1.object_id, v2.object_id) - end - - def test_lookup_server - addr1 = "192.168.100.1" - addr2 = "192.168.100.2" - addrz = "192.168.100.254" - local = "127.0.0.1" - port1 = 80 - port2 = 8080 - port3 = 10080 - portz = 32767 - name1 = "www.example.com" - name2 = "www2.example.com" - name3 = "www3.example.com" - namea = "www.example.co.jp" - nameb = "www.example.jp" - namec = "www2.example.co.jp" - named = "www2.example.jp" - namez = "foobar.example.com" - alias1 = [namea, nameb] - alias2 = [namec, named] - - host1 = httpd(nil, port1, name1, nil) - hosts = [ - host2 = httpd(addr1, port1, name1, nil), - host3 = httpd(addr1, port1, name2, alias1), - host4 = httpd(addr1, port2, name1, nil), - host5 = httpd(addr1, port2, name2, alias1), - httpd(addr1, port2, name3, alias2), - host7 = httpd(addr2, nil, name1, nil), - host8 = httpd(addr2, nil, name2, alias1), - httpd(addr2, nil, name3, alias2), - host10 = httpd(local, nil, nil, nil), - host11 = httpd(nil, port3, nil, nil), - ].sort_by{ rand } - hosts.each{|h| host1.virtual_host(h) } - - # connect to addr1 - assert_eql?(host2, host1.lookup_server(Req.new(addr1, port1, name1))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, name2))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, namea))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, port1, namez))) - assert_eql?(host4, host1.lookup_server(Req.new(addr1, port2, name1))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, name2))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, namea))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, port2, namez))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, name1))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, name2))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, namea))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, namez))) - - # connect to addr2 - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port1, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, port1, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port2, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, port2, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port3, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addr2, port3, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, portz, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, portz, namez))) - - # connect to addrz - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, namez))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, name1))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, name2))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, namea))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, namez))) - - # connect to localhost - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, namez))) - end - - def test_callbacks - accepted = started = stopped = 0 - requested0 = requested1 = 0 - config = { - :ServerName => "localhost", - :AcceptCallback => Proc.new{ accepted += 1 }, - :StartCallback => Proc.new{ started += 1 }, - :StopCallback => Proc.new{ stopped += 1 }, - :RequestCallback => Proc.new{|req, res| requested0 += 1 }, - } - log_tester = lambda {|log, access_log| - assert(log.find {|s| %r{ERROR `/' not found\.} =~ s }) - assert_equal([], log.reject {|s| %r{ERROR `/' not found\.} =~ s }) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - vhost_config = { - :ServerName => "myhostname", - :BindAddress => addr, - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :RequestCallback => Proc.new{|req, res| requested1 += 1 }, - } - server.virtual_host(WEBrick::HTTPServer.new(vhost_config)) - - Thread.pass while server.status != :Running - sleep 1 if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? # server.status behaves unexpectedly with --jit-wait - assert_equal(1, started, log.call) - assert_equal(0, stopped, log.call) - assert_equal(0, accepted, log.call) - - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req["Host"] = "myhostname:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - req["Host"] = "localhost:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - assert_equal(6, accepted, log.call) - assert_equal(3, requested0, log.call) - assert_equal(3, requested1, log.call) - } - assert_equal(started, 1) - assert_equal(stopped, 1) - end - - class CustomRequest < ::WEBrick::HTTPRequest; end - class CustomResponse < ::WEBrick::HTTPResponse; end - class CustomServer < ::WEBrick::HTTPServer - def create_request(config) - CustomRequest.new(config) - end - - def create_response(config) - CustomResponse.new(config) - end - end - - def test_custom_server_request_and_response - config = { :ServerName => "localhost" } - TestWEBrick.start_server(CustomServer, config){|server, addr, port, log| - server.mount_proc("/", lambda {|req, res| - assert_kind_of(CustomRequest, req) - assert_kind_of(CustomResponse, res) - res.body = "via custom response" - }) - Thread.pass while server.status != :Running - - Net::HTTP.start(addr, port) do |http| - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("via custom response", res.body) - } - server.shutdown - end - } - end - - # This class is needed by test_response_io_with_chunked_set method - class EventManagerForChunkedResponseTest - def initialize - @listeners = [] - end - def add_listener( &block ) - @listeners << block - end - def raise_str_event( str ) - @listeners.each{ |e| e.call( :str, str ) } - end - def raise_close_event() - @listeners.each{ |e| e.call( :cls ) } - end - end - def test_response_io_with_chunked_set - evt_man = EventManagerForChunkedResponseTest.new - t = Thread.new do - begin - config = { - :ServerName => "localhost" - } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - body_strs = [ 'aaaaaa', 'bb', 'cccc' ] - server.mount_proc( "/", ->( req, res ){ - # Test for setting chunked... - res.chunked = true - r,w = IO.pipe - evt_man.add_listener do |type,str| - type == :cls ? ( w.close ) : ( w << str ) - end - res.body = r - } ) - Thread.pass while server.status != :Running - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - http.request(req) do |res| - i = 0 - evt_man.raise_str_event( body_strs[i] ) - res.read_body do |s| - assert_equal( body_strs[i], s ) - i += 1 - if i < body_strs.length - evt_man.raise_str_event( body_strs[i] ) - else - evt_man.raise_close_event() - end - end - assert_equal( body_strs.length, i ) - end - end - rescue => err - flunk( 'exception raised in thread: ' + err.to_s ) - end - end - if t.join( 3 ).nil? - evt_man.raise_close_event() - flunk( 'timeout' ) - if t.join( 1 ).nil? - Thread.kill t - end - end - end - - def test_response_io_without_chunked_set - config = { - :ServerName => "localhost" - } - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/WARN Could not determine content-length of response body./, log[0]) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - server.mount_proc("/", lambda { |req, res| - r,w = IO.pipe - # Test for not setting chunked... - # res.chunked = true - res.body = r - w << "foo" - w.close - }) - Thread.pass while server.status != :Running - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req['Connection'] = 'Keep-Alive' - begin - Timeout.timeout(2) do - http.request(req){|res| assert_equal("foo", res.body) } - end - rescue Timeout::Error - flunk('corrupted response') - end - } - end - - def test_request_handler_callback_is_deprecated - requested = 0 - config = { - :ServerName => "localhost", - :RequestHandler => Proc.new{|req, res| requested += 1 }, - } - log_tester = lambda {|log, access_log| - assert_equal(2, log.length) - assert_match(/WARN :RequestHandler is deprecated, please use :RequestCallback/, log[0]) - assert_match(%r{ERROR `/' not found\.}, log[1]) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - Thread.pass while server.status != :Running - - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req["Host"] = "localhost:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - assert_match(%r{:RequestHandler is deprecated, please use :RequestCallback$}, log.call, log.call) - } - assert_equal(1, requested) - end - - def test_shutdown_with_busy_keepalive_connection - requested = 0 - config = { - :ServerName => "localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc("/", lambda {|req, res| res.body = "heffalump" }) - Thread.pass while server.status != :Running - - Net::HTTP.start(addr, port) do |http| - req = Net::HTTP::Get.new("/") - http.request(req){|res| assert_equal('Keep-Alive', res['Connection'], log.call) } - server.shutdown - begin - 10.times {|n| http.request(req); requested += 1 } - rescue - # Errno::ECONNREFUSED or similar - end - end - } - assert_equal(0, requested, "Server responded to #{requested} requests after shutdown") - end - - def test_cntrl_in_path - log_ary = [] - access_log_ary = [] - config = { - :Port => 0, - :BindAddress => '127.0.0.1', - :Logger => WEBrick::Log.new(log_ary, WEBrick::BasicLog::WARN), - :AccessLog => [[access_log_ary, '']], - } - s = WEBrick::HTTPServer.new(config) - s.mount('/foo', WEBrick::HTTPServlet::FileHandler, __FILE__) - th = Thread.new { s.start } - addr = s.listeners[0].addr - - http = Net::HTTP.new(addr[3], addr[1]) - req = Net::HTTP::Get.new('/notexist%0a/foo') - http.request(req) { |res| assert_equal('404', res.code) } - exp = %Q(ERROR `/notexist\\n/foo' not found.\n) - assert_equal 1, log_ary.size - assert_include log_ary[0], exp - ensure - s&.shutdown - th&.join - end - - def test_gigantic_request_header - log_tester = lambda {|log, access_log| - assert_equal 1, log.size - assert_include log[0], 'ERROR headers too large' - } - TestWEBrick.start_httpserver({}, log_tester){|server, addr, port, log| - server.mount('/', WEBrick::HTTPServlet::FileHandler, __FILE__) - TCPSocket.open(addr, port) do |c| - c.write("GET / HTTP/1.0\r\n") - junk = -"X-Junk: #{' ' * 1024}\r\n" - assert_raise(Errno::ECONNRESET, Errno::EPIPE, Errno::EPROTOTYPE) do - loop { c.write(junk) } - end - end - } - end - - def test_eof_in_chunk - log_tester = lambda do |log, access_log| - assert_equal 1, log.size - assert_include log[0], 'ERROR bad chunk data size' - end - TestWEBrick.start_httpserver({}, log_tester){|server, addr, port, log| - server.mount_proc('/', ->(req, res) { res.body = req.body }) - TCPSocket.open(addr, port) do |c| - c.write("POST / HTTP/1.1\r\nHost: example.com\r\n" \ - "Transfer-Encoding: chunked\r\n\r\n5\r\na") - c.shutdown(Socket::SHUT_WR) # trigger EOF in server - res = c.read - assert_match %r{\AHTTP/1\.1 400 }, res - end - } - end - - def test_big_chunks - nr_out = 3 - buf = 'big' # 3 bytes is bigger than 2! - config = { :InputBufferSize => 2 }.freeze - total = 0 - all = '' - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc('/', ->(req, res) { - err = [] - ret = req.body do |chunk| - n = chunk.bytesize - n > config[:InputBufferSize] and err << "#{n} > :InputBufferSize" - total += n - all << chunk - end - ret.nil? or err << 'req.body should return nil' - (buf * nr_out) == all or err << 'input body does not match expected' - res.header['connection'] = 'close' - res.body = err.join("\n") - }) - TCPSocket.open(addr, port) do |c| - c.write("POST / HTTP/1.1\r\nHost: example.com\r\n" \ - "Transfer-Encoding: chunked\r\n\r\n") - chunk = "#{buf.bytesize.to_s(16)}\r\n#{buf}\r\n" - nr_out.times { c.write(chunk) } - c.write("0\r\n\r\n") - head, body = c.read.split("\r\n\r\n") - assert_match %r{\AHTTP/1\.1 200 OK}, head - assert_nil body - end - } - end -end diff --git a/tool/test/webrick/test_httpstatus.rb b/tool/test/webrick/test_httpstatus.rb deleted file mode 100644 index fd0570d5c6..0000000000 --- a/tool/test/webrick/test_httpstatus.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick" - -class TestWEBrickHTTPStatus < Test::Unit::TestCase - def test_info? - assert WEBrick::HTTPStatus.info?(100) - refute WEBrick::HTTPStatus.info?(200) - end - - def test_success? - assert WEBrick::HTTPStatus.success?(200) - refute WEBrick::HTTPStatus.success?(300) - end - - def test_redirect? - assert WEBrick::HTTPStatus.redirect?(300) - refute WEBrick::HTTPStatus.redirect?(400) - end - - def test_error? - assert WEBrick::HTTPStatus.error?(400) - refute WEBrick::HTTPStatus.error?(600) - end - - def test_client_error? - assert WEBrick::HTTPStatus.client_error?(400) - refute WEBrick::HTTPStatus.client_error?(500) - end - - def test_server_error? - assert WEBrick::HTTPStatus.server_error?(500) - refute WEBrick::HTTPStatus.server_error?(600) - end -end diff --git a/tool/test/webrick/test_httputils.rb b/tool/test/webrick/test_httputils.rb deleted file mode 100644 index 00f297bd09..0000000000 --- a/tool/test/webrick/test_httputils.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/httputils" - -class TestWEBrickHTTPUtils < Test::Unit::TestCase - include WEBrick::HTTPUtils - - def test_normilize_path - assert_equal("/foo", normalize_path("/foo")) - assert_equal("/foo/bar/", normalize_path("/foo/bar/")) - - assert_equal("/", normalize_path("/foo/../")) - assert_equal("/", normalize_path("/foo/..")) - assert_equal("/", normalize_path("/foo/bar/../../")) - assert_equal("/", normalize_path("/foo/bar/../..")) - assert_equal("/", normalize_path("/foo/bar/../..")) - assert_equal("/baz", normalize_path("/foo/bar/../../baz")) - assert_equal("/baz", normalize_path("/foo/../bar/../baz")) - assert_equal("/baz/", normalize_path("/foo/../bar/../baz/")) - assert_equal("/...", normalize_path("/bar/../...")) - assert_equal("/.../", normalize_path("/bar/../.../")) - - assert_equal("/foo/", normalize_path("/foo/./")) - assert_equal("/foo/", normalize_path("/foo/.")) - assert_equal("/foo/", normalize_path("/foo/././")) - assert_equal("/foo/", normalize_path("/foo/./.")) - assert_equal("/foo/bar", normalize_path("/foo/./bar")) - assert_equal("/foo/bar/", normalize_path("/foo/./bar/.")) - assert_equal("/foo/bar/", normalize_path("/./././foo/./bar/.")) - - assert_equal("/foo/bar/", normalize_path("//foo///.//bar/.///.//")) - assert_equal("/", normalize_path("//foo///..///bar/.///..//.//")) - - assert_raise(RuntimeError){ normalize_path("foo/bar") } - assert_raise(RuntimeError){ normalize_path("..") } - assert_raise(RuntimeError){ normalize_path("/..") } - assert_raise(RuntimeError){ normalize_path("/./..") } - assert_raise(RuntimeError){ normalize_path("/./../") } - assert_raise(RuntimeError){ normalize_path("/./../..") } - assert_raise(RuntimeError){ normalize_path("/./../../") } - assert_raise(RuntimeError){ normalize_path("/./../") } - assert_raise(RuntimeError){ normalize_path("/../..") } - assert_raise(RuntimeError){ normalize_path("/../../") } - assert_raise(RuntimeError){ normalize_path("/../../..") } - assert_raise(RuntimeError){ normalize_path("/../../../") } - assert_raise(RuntimeError){ normalize_path("/../foo/../") } - assert_raise(RuntimeError){ normalize_path("/../foo/../../") } - assert_raise(RuntimeError){ normalize_path("/foo/bar/../../../../") } - assert_raise(RuntimeError){ normalize_path("/foo/../bar/../../") } - assert_raise(RuntimeError){ normalize_path("/./../bar/") } - assert_raise(RuntimeError){ normalize_path("/./../") } - end - - def test_split_header_value - assert_equal(['foo', 'bar'], split_header_value('foo, bar')) - assert_equal(['"foo"', 'bar'], split_header_value('"foo", bar')) - assert_equal(['foo', '"bar"'], split_header_value('foo, "bar"')) - assert_equal(['*'], split_header_value('*')) - assert_equal(['W/"xyzzy"', 'W/"r2d2xxxx"', 'W/"c3piozzzz"'], - split_header_value('W/"xyzzy", W/"r2d2xxxx", W/"c3piozzzz"')) - end - - def test_escape - assert_equal("/foo/bar", escape("/foo/bar")) - assert_equal("/~foo/bar", escape("/~foo/bar")) - assert_equal("/~foo%20bar", escape("/~foo bar")) - assert_equal("/~foo%20bar", escape("/~foo bar")) - assert_equal("/~foo%09bar", escape("/~foo\tbar")) - assert_equal("/~foo+bar", escape("/~foo+bar")) - bug8425 = '[Bug #8425] [ruby-core:55052]' - assert_nothing_raised(ArgumentError, Encoding::CompatibilityError, bug8425) { - assert_equal("%E3%83%AB%E3%83%93%E3%83%BC%E3%81%95%E3%82%93", escape("\u{30EB 30D3 30FC 3055 3093}")) - } - end - - def test_escape_form - assert_equal("%2Ffoo%2Fbar", escape_form("/foo/bar")) - assert_equal("%2F~foo%2Fbar", escape_form("/~foo/bar")) - assert_equal("%2F~foo+bar", escape_form("/~foo bar")) - assert_equal("%2F~foo+%2B+bar", escape_form("/~foo + bar")) - end - - def test_unescape - assert_equal("/foo/bar", unescape("%2ffoo%2fbar")) - assert_equal("/~foo/bar", unescape("/%7efoo/bar")) - assert_equal("/~foo/bar", unescape("%2f%7efoo%2fbar")) - assert_equal("/~foo+bar", unescape("/%7efoo+bar")) - end - - def test_unescape_form - assert_equal("//foo/bar", unescape_form("/%2Ffoo/bar")) - assert_equal("//foo/bar baz", unescape_form("/%2Ffoo/bar+baz")) - assert_equal("/~foo/bar baz", unescape_form("/%7Efoo/bar+baz")) - end - - def test_escape_path - assert_equal("/foo/bar", escape_path("/foo/bar")) - assert_equal("/foo/bar/", escape_path("/foo/bar/")) - assert_equal("/%25foo/bar/", escape_path("/%foo/bar/")) - end -end diff --git a/tool/test/webrick/test_httpversion.rb b/tool/test/webrick/test_httpversion.rb deleted file mode 100644 index e50ee17971..0000000000 --- a/tool/test/webrick/test_httpversion.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/httpversion" - -class TestWEBrickHTTPVersion < Test::Unit::TestCase - def setup - @v09 = WEBrick::HTTPVersion.new("0.9") - @v10 = WEBrick::HTTPVersion.new("1.0") - @v11 = WEBrick::HTTPVersion.new("1.001") - end - - def test_to_s() - assert_equal("0.9", @v09.to_s) - assert_equal("1.0", @v10.to_s) - assert_equal("1.1", @v11.to_s) - end - - def test_major() - assert_equal(0, @v09.major) - assert_equal(1, @v10.major) - assert_equal(1, @v11.major) - end - - def test_minor() - assert_equal(9, @v09.minor) - assert_equal(0, @v10.minor) - assert_equal(1, @v11.minor) - end - - def test_compar() - assert_equal(0, @v09 <=> "0.9") - assert_equal(0, @v09 <=> "0.09") - - assert_equal(-1, @v09 <=> @v10) - assert_equal(-1, @v09 <=> "1.00") - - assert_equal(1, @v11 <=> @v09) - assert_equal(1, @v11 <=> "1.0") - assert_equal(1, @v11 <=> "0.9") - end -end diff --git a/tool/test/webrick/test_server.rb b/tool/test/webrick/test_server.rb deleted file mode 100644 index 3bd8115c61..0000000000 --- a/tool/test/webrick/test_server.rb +++ /dev/null @@ -1,191 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "tempfile" -require "webrick" -require_relative "utils" - -class TestWEBrickServer < Test::Unit::TestCase - class Echo < WEBrick::GenericServer - def run(sock) - while line = sock.gets - sock << line - end - end - end - - def test_server - TestWEBrick.start_server(Echo){|server, addr, port, log| - TCPSocket.open(addr, port){|sock| - sock.puts("foo"); assert_equal("foo\n", sock.gets, log.call) - sock.puts("bar"); assert_equal("bar\n", sock.gets, log.call) - sock.puts("baz"); assert_equal("baz\n", sock.gets, log.call) - sock.puts("qux"); assert_equal("qux\n", sock.gets, log.call) - } - } - end - - def test_start_exception - stopped = 0 - - log = [] - logger = WEBrick::Log.new(log, WEBrick::BasicLog::WARN) - - assert_raise(SignalException) do - listener = Object.new - def listener.to_io # IO.select invokes #to_io. - raise SignalException, 'SIGTERM' # simulate signal in main thread - end - def listener.shutdown - end - def listener.close - end - - server = WEBrick::HTTPServer.new({ - :BindAddress => "127.0.0.1", :Port => 0, - :StopCallback => Proc.new{ stopped += 1 }, - :Logger => logger, - }) - server.listeners[0].close - server.listeners[0] = listener - - server.start - end - - assert_equal(1, stopped) - assert_equal(1, log.length) - assert_match(/FATAL SignalException: SIGTERM/, log[0]) - end - - def test_callbacks - accepted = started = stopped = 0 - config = { - :AcceptCallback => Proc.new{ accepted += 1 }, - :StartCallback => Proc.new{ started += 1 }, - :StopCallback => Proc.new{ stopped += 1 }, - } - TestWEBrick.start_server(Echo, config){|server, addr, port, log| - true while server.status != :Running - sleep 1 if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? # server.status behaves unexpectedly with --jit-wait - assert_equal(1, started, log.call) - assert_equal(0, stopped, log.call) - assert_equal(0, accepted, log.call) - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - assert_equal(3, accepted, log.call) - } - assert_equal(1, started) - assert_equal(1, stopped) - end - - def test_daemon - begin - r, w = IO.pipe - pid1 = Process.fork{ - r.close - WEBrick::Daemon.start - w.puts(Process.pid) - sleep 10 - } - pid2 = r.gets.to_i - assert(Process.kill(:KILL, pid2)) - assert_not_equal(pid1, pid2) - rescue NotImplementedError - # snip this test - ensure - Process.wait(pid1) if pid1 - r.close - w.close - end - end - - def test_restart_after_shutdown - address = '127.0.0.1' - port = 0 - log = [] - config = { - :BindAddress => address, - :Port => port, - :Logger => WEBrick::Log.new(log, WEBrick::BasicLog::WARN), - } - server = Echo.new(config) - client_proc = lambda {|str| - begin - ret = server.listeners.first.connect_address.connect {|s| - s.write(str) - s.close_write - s.read - } - assert_equal(str, ret) - ensure - server.shutdown - end - } - server_thread = Thread.new { server.start } - client_thread = Thread.new { client_proc.call("a") } - assert_join_threads([client_thread, server_thread]) - server.listen(address, port) - server_thread = Thread.new { server.start } - client_thread = Thread.new { client_proc.call("b") } - assert_join_threads([client_thread, server_thread]) - assert_equal([], log) - end - - def test_restart_after_stop - log = Object.new - class << log - include Test::Unit::Assertions - def <<(msg) - flunk "unexpected log: #{msg.inspect}" - end - end - client_thread = nil - wakeup = -> {client_thread.wakeup} - warn_flunk = WEBrick::Log.new(log, WEBrick::BasicLog::WARN) - server = WEBrick::HTTPServer.new( - :StartCallback => wakeup, - :StopCallback => wakeup, - :BindAddress => '0.0.0.0', - :Port => 0, - :Logger => warn_flunk) - 2.times { - server_thread = Thread.start { - server.start - } - client_thread = Thread.start { - sleep 0.1 until server.status == :Running || !server_thread.status - server.stop - sleep 0.1 until server.status == :Stop || !server_thread.status - } - assert_join_threads([client_thread, server_thread]) - } - end - - def test_port_numbers - config = { - :BindAddress => '0.0.0.0', - :Logger => WEBrick::Log.new([], WEBrick::BasicLog::WARN), - } - - ports = [0, "0"] - - ports.each do |port| - config[:Port]= port - server = WEBrick::GenericServer.new(config) - server_thread = Thread.start { server.start } - client_thread = Thread.start { - sleep 0.1 until server.status == :Running || !server_thread.status - server_port = server.listeners[0].addr[1] - server.stop - assert_equal server.config[:Port], server_port - sleep 0.1 until server.status == :Stop || !server_thread.status - } - assert_join_threads([client_thread, server_thread]) - end - - assert_raise(ArgumentError) do - config[:Port]= "FOO" - WEBrick::GenericServer.new(config) - end - end -end diff --git a/tool/test/webrick/test_ssl_server.rb b/tool/test/webrick/test_ssl_server.rb deleted file mode 100644 index 4e52598bf5..0000000000 --- a/tool/test/webrick/test_ssl_server.rb +++ /dev/null @@ -1,67 +0,0 @@ -require "test/unit" -require "webrick" -require "webrick/ssl" -require_relative "utils" -require 'timeout' - -class TestWEBrickSSLServer < Test::Unit::TestCase - class Echo < WEBrick::GenericServer - def run(sock) - while line = sock.gets - sock << line - end - end - end - - def test_self_signed_cert_server - assert_self_signed_cert( - :SSLEnable => true, - :SSLCertName => [["C", "JP"], ["O", "www.ruby-lang.org"], ["CN", "Ruby"]], - ) - end - - def test_self_signed_cert_server_with_string - assert_self_signed_cert( - :SSLEnable => true, - :SSLCertName => "/C=JP/O=www.ruby-lang.org/CN=Ruby", - ) - end - - def assert_self_signed_cert(config) - TestWEBrick.start_server(Echo, config){|server, addr, port, log| - io = TCPSocket.new(addr, port) - sock = OpenSSL::SSL::SSLSocket.new(io) - sock.connect - sock.puts(server.ssl_context.cert.subject.to_s) - assert_equal("/C=JP/O=www.ruby-lang.org/CN=Ruby\n", sock.gets, log.call) - sock.close - io.close - } - end - - def test_slow_connect - poke = lambda do |io, msg| - begin - sock = OpenSSL::SSL::SSLSocket.new(io) - sock.connect - sock.puts(msg) - assert_equal "#{msg}\n", sock.gets, msg - ensure - sock&.close - io.close - end - end - config = { - :SSLEnable => true, - :SSLCertName => "/C=JP/O=www.ruby-lang.org/CN=Ruby", - } - EnvUtil.timeout(10) do - TestWEBrick.start_server(Echo, config) do |server, addr, port, log| - outer = TCPSocket.new(addr, port) - inner = TCPSocket.new(addr, port) - poke.call(inner, 'fast TLS negotiation') - poke.call(outer, 'slow TLS negotiation') - end - end - end -end diff --git a/tool/test/webrick/test_utils.rb b/tool/test/webrick/test_utils.rb deleted file mode 100644 index c2b7a36e8a..0000000000 --- a/tool/test/webrick/test_utils.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/utils" - -class TestWEBrickUtils < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def assert_expired(m) - Thread.handle_interrupt(Timeout::Error => :never, EX => :never) do - assert_empty(m::TimeoutHandler.instance.instance_variable_get(:@timeout_info)) - end - end - - def assert_not_expired(m) - Thread.handle_interrupt(Timeout::Error => :never, EX => :never) do - assert_not_empty(m::TimeoutHandler.instance.instance_variable_get(:@timeout_info)) - end - end - - EX = Class.new(StandardError) - - def test_no_timeout - m = WEBrick::Utils - assert_equal(:foo, m.timeout(10){ :foo }) - assert_expired(m) - end - - def test_nested_timeout_outer - m = WEBrick::Utils - i = 0 - assert_raise(Timeout::Error){ - m.timeout(1){ - assert_raise(Timeout::Error){ m.timeout(0.1){ i += 1; sleep(1) } } - assert_not_expired(m) - i += 1 - sleep(2) - } - } - assert_equal(2, i) - assert_expired(m) - end - - def test_timeout_default_exception - m = WEBrick::Utils - assert_raise(Timeout::Error){ m.timeout(0.01){ sleep } } - assert_expired(m) - end - - def test_timeout_custom_exception - m = WEBrick::Utils - ex = EX - assert_raise(ex){ m.timeout(0.01, ex){ sleep } } - assert_expired(m) - end - - def test_nested_timeout_inner_custom_exception - m = WEBrick::Utils - ex = EX - i = 0 - assert_raise(ex){ - m.timeout(10){ - m.timeout(0.01, ex){ i += 1; sleep } - } - sleep - } - assert_equal(1, i) - assert_expired(m) - end - - def test_nested_timeout_outer_custom_exception - m = WEBrick::Utils - ex = EX - i = 0 - assert_raise(Timeout::Error){ - m.timeout(0.01){ - m.timeout(1.0, ex){ i += 1; sleep } - } - sleep - } - assert_equal(1, i) - assert_expired(m) - end - - def test_create_listeners - addr = listener_address(0) - port = addr.slice!(1) - assert_kind_of(Integer, port, "dynamically chosen port number") - assert_equal(["AF_INET", "127.0.0.1", "127.0.0.1"], addr) - - assert_equal(["AF_INET", port, "127.0.0.1", "127.0.0.1"], - listener_address(port), - "specific port number") - - assert_equal(["AF_INET", port, "127.0.0.1", "127.0.0.1"], - listener_address(port.to_s), - "specific port number string") - end - - def listener_address(port) - listeners = WEBrick::Utils.create_listeners("127.0.0.1", port) - srv = listeners.first - assert_kind_of TCPServer, srv - srv.addr - ensure - listeners.each(&:close) if listeners - end -end diff --git a/tool/test/webrick/utils.rb b/tool/test/webrick/utils.rb deleted file mode 100644 index c8e84c37f1..0000000000 --- a/tool/test/webrick/utils.rb +++ /dev/null @@ -1,104 +0,0 @@ -# frozen_string_literal: false -require "webrick" -begin - require "webrick/https" -rescue LoadError -end -require "webrick/httpproxy" - -module TestWEBrick - NullWriter = Object.new - def NullWriter.<<(msg) - puts msg if $DEBUG - return self - end - - class WEBrick::HTTPServlet::CGIHandler - remove_const :Ruby - require "envutil" unless defined?(EnvUtil) - Ruby = EnvUtil.rubybin - remove_const :CGIRunner - CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: - remove_const :CGIRunnerArray - CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb"] # :nodoc: - end - - RubyBin = "\"#{EnvUtil.rubybin}\"" - RubyBin << " --disable-gems" - RubyBin << " \"-I#{File.expand_path("../..", File.dirname(__FILE__))}/lib\"" - RubyBin << " \"-I#{File.dirname(EnvUtil.rubybin)}/.ext/common\"" - RubyBin << " \"-I#{File.dirname(EnvUtil.rubybin)}/.ext/#{RUBY_PLATFORM}\"" - - RubyBinArray = [EnvUtil.rubybin] - RubyBinArray << "--disable-gems" - RubyBinArray << "-I" << "#{File.expand_path("../..", File.dirname(__FILE__))}/lib" - RubyBinArray << "-I" << "#{File.dirname(EnvUtil.rubybin)}/.ext/common" - RubyBinArray << "-I" << "#{File.dirname(EnvUtil.rubybin)}/.ext/#{RUBY_PLATFORM}" - - require "test/unit" unless defined?(Test::Unit) - include Test::Unit::Assertions - extend Test::Unit::Assertions - include Test::Unit::CoreAssertions - extend Test::Unit::CoreAssertions - - module_function - - DefaultLogTester = lambda {|log, access_log| assert_equal([], log) } - - def start_server(klass, config={}, log_tester=DefaultLogTester, &block) - log_ary = [] - access_log_ary = [] - log = proc { "webrick log start:\n" + (log_ary+access_log_ary).join.gsub(/^/, " ").chomp + "\nwebrick log end" } - config = ({ - :BindAddress => "127.0.0.1", :Port => 0, - :ServerType => Thread, - :Logger => WEBrick::Log.new(log_ary, WEBrick::BasicLog::WARN), - :AccessLog => [[access_log_ary, ""]] - }.update(config)) - server = capture_output {break klass.new(config)} - server_thread = server.start - server_thread2 = Thread.new { - server_thread.join - if log_tester - log_tester.call(log_ary, access_log_ary) - end - } - addr = server.listeners[0].addr - client_thread = Thread.new { - begin - block.yield([server, addr[3], addr[1], log]) - ensure - server.shutdown - end - } - assert_join_threads([client_thread, server_thread2]) - end - - def start_httpserver(config={}, log_tester=DefaultLogTester, &block) - start_server(WEBrick::HTTPServer, config, log_tester, &block) - end - - def start_httpproxy(config={}, log_tester=DefaultLogTester, &block) - start_server(WEBrick::HTTPProxyServer, config, log_tester, &block) - end - - def start_cgi_server(config={}, log_tester=TestWEBrick::DefaultLogTester, &block) - config = { - :CGIInterpreter => TestWEBrick::RubyBin, - :DocumentRoot => File.dirname(__FILE__), - :DirectoryIndex => ["webrick.cgi"], - :RequestCallback => Proc.new{|req, res| - def req.meta_vars - meta = super - meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) - meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] - return meta - end - }, - }.merge(config) - if RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32/ - config[:CGIPathEnv] = ENV['PATH'] # runtime dll may not be in system dir. - end - start_server(WEBrick::HTTPServer, config, log_tester, &block) - end -end diff --git a/tool/test/webrick/webrick.cgi b/tool/test/webrick/webrick.cgi deleted file mode 100644 index a294fa72f9..0000000000 --- a/tool/test/webrick/webrick.cgi +++ /dev/null @@ -1,38 +0,0 @@ -#!ruby -require "webrick/cgi" - -class TestApp < WEBrick::CGI - def do_GET(req, res) - res["content-type"] = "text/plain" - if req.path_info == "/dumpenv" - res.body = Marshal.dump(ENV.to_hash) - elsif (p = req.path_info) && p.length > 0 - res.body = p - elsif (q = req.query).size > 0 - res.body = q.keys.sort.collect{|key| - q[key].list.sort.collect{|v| - "#{key}=#{v}" - }.join(", ") - }.join(", ") - elsif %r{/$} =~ req.request_uri.to_s - res.body = "" - res.body << req.request_uri.to_s << "\n" - res.body << req.script_name - elsif !req.cookies.empty? - res.body = req.cookies.inject(""){|result, cookie| - result << "%s=%s\n" % [cookie.name, cookie.value] - } - res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") - res.cookies << WEBrick::Cookie.new("Shipping", "FedEx") - else - res.body = req.script_name - end - end - - def do_POST(req, res) - do_GET(req, res) - end -end - -cgi = TestApp.new -cgi.start diff --git a/tool/test/webrick/webrick.rhtml b/tool/test/webrick/webrick.rhtml deleted file mode 100644 index a7bbe43fb5..0000000000 --- a/tool/test/webrick/webrick.rhtml +++ /dev/null @@ -1,4 +0,0 @@ -req to <%= -servlet_request.request_uri -%> <%= -servlet_request.query.inspect %> diff --git a/tool/test/webrick/webrick_long_filename.cgi b/tool/test/webrick/webrick_long_filename.cgi deleted file mode 100644 index 43c1af825c..0000000000 --- a/tool/test/webrick/webrick_long_filename.cgi +++ /dev/null @@ -1,36 +0,0 @@ -#!ruby -require "webrick/cgi" - -class TestApp < WEBrick::CGI - def do_GET(req, res) - res["content-type"] = "text/plain" - if (p = req.path_info) && p.length > 0 - res.body = p - elsif (q = req.query).size > 0 - res.body = q.keys.sort.collect{|key| - q[key].list.sort.collect{|v| - "#{key}=#{v}" - }.join(", ") - }.join(", ") - elsif %r{/$} =~ req.request_uri.to_s - res.body = "" - res.body << req.request_uri.to_s << "\n" - res.body << req.script_name - elsif !req.cookies.empty? - res.body = req.cookies.inject(""){|result, cookie| - result << "%s=%s\n" % [cookie.name, cookie.value] - } - res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") - res.cookies << WEBrick::Cookie.new("Shipping", "FedEx") - else - res.body = req.script_name - end - end - - def do_POST(req, res) - do_GET(req, res) - end -end - -cgi = TestApp.new -cgi.start diff --git a/tool/test_for_warn_bundled_gems/.gitignore b/tool/test_for_warn_bundled_gems/.gitignore deleted file mode 100644 index a9a5aecf42..0000000000 --- a/tool/test_for_warn_bundled_gems/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tmp diff --git a/tool/test_for_warn_bundled_gems/Gemfile b/tool/test_for_warn_bundled_gems/Gemfile deleted file mode 100644 index e69de29bb2..0000000000 --- a/tool/test_for_warn_bundled_gems/Gemfile +++ /dev/null diff --git a/tool/test_for_warn_bundled_gems/Gemfile.lock b/tool/test_for_warn_bundled_gems/Gemfile.lock deleted file mode 100644 index 003cb81444..0000000000 --- a/tool/test_for_warn_bundled_gems/Gemfile.lock +++ /dev/null @@ -1,11 +0,0 @@ -GEM - specs: - -PLATFORMS - arm64-darwin-22 - ruby - -DEPENDENCIES - -BUNDLED WITH - 2.5.0.dev diff --git a/tool/test_for_warn_bundled_gems/README.md b/tool/test_for_warn_bundled_gems/README.md deleted file mode 100644 index dc2d2a6cb9..0000000000 --- a/tool/test_for_warn_bundled_gems/README.md +++ /dev/null @@ -1,3 +0,0 @@ -This directory contains tests for the bundled gems warning under the Bundler. - -see [test.sh](./test.sh) for details. diff --git a/tool/test_for_warn_bundled_gems/test.sh b/tool/test_for_warn_bundled_gems/test.sh deleted file mode 100755 index ce714c7e13..0000000000 --- a/tool/test_for_warn_bundled_gems/test.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -echo "* Show warning require and LoadError" -ruby test_warn_bundled_gems.rb - -echo "* Show warning when bundled gems called as dependency" -ruby test_warn_dependency.rb - -echo "* Show warning sub-feature like bigdecimal/util" -ruby test_warn_sub_feature.rb - -echo "* Show warning dash gem like net/smtp" -ruby test_warn_dash_gem.rb - -echo "* Show warning when bundle exec with ruby and script" -bundle exec ruby test_warn_bundle_exec.rb - -echo "* Show warning when bundle exec with shebang's script" -bundle exec ./test_warn_bundle_exec_shebang.rb - -echo "* Don't show warning bundled gems on Gemfile" -ruby test_no_warn_dependency.rb - -echo "* Don't show warning with bootsnap" -ruby test_no_warn_bootsnap.rb - -echo "* Don't show warning with net/smtp when net-smtp on Gemfile" -ruby test_no_warn_dash_gem.rb - -echo "* Don't show warning bigdecimal/util when bigdecimal on Gemfile" -ruby test_no_warn_sub_feature.rb diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb b/tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb deleted file mode 100644 index eac58de974..0000000000 --- a/tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb +++ /dev/null @@ -1,11 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" - gem "bootsnap", require: false -end - -require 'bootsnap' -Bootsnap.setup(cache_dir: 'tmp/cache') - -require 'csv' diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_dash_gem.rb b/tool/test_for_warn_bundled_gems/test_no_warn_dash_gem.rb deleted file mode 100644 index 72ae23b040..0000000000 --- a/tool/test_for_warn_bundled_gems/test_no_warn_dash_gem.rb +++ /dev/null @@ -1,8 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" - gem "net-smtp" -end - -require "net/smtp" diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_dependency.rb b/tool/test_for_warn_bundled_gems/test_no_warn_dependency.rb deleted file mode 100644 index 94a32a9108..0000000000 --- a/tool/test_for_warn_bundled_gems/test_no_warn_dependency.rb +++ /dev/null @@ -1,10 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" - gem "activesupport", "7.0.7.2" - gem "bigdecimal" - gem "mutex_m" -end - -require "active_support/all" diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_sub_feature.rb b/tool/test_for_warn_bundled_gems/test_no_warn_sub_feature.rb deleted file mode 100644 index 7d62a2f9d0..0000000000 --- a/tool/test_for_warn_bundled_gems/test_no_warn_sub_feature.rb +++ /dev/null @@ -1,8 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" - gem "bigdecimal" -end - -require "bigdecimal/util" diff --git a/tool/test_for_warn_bundled_gems/test_warn_bundle_exec.rb b/tool/test_for_warn_bundled_gems/test_warn_bundle_exec.rb deleted file mode 100644 index 30db47ce61..0000000000 --- a/tool/test_for_warn_bundled_gems/test_warn_bundle_exec.rb +++ /dev/null @@ -1 +0,0 @@ -require "base64" diff --git a/tool/test_for_warn_bundled_gems/test_warn_bundle_exec_shebang.rb b/tool/test_for_warn_bundled_gems/test_warn_bundle_exec_shebang.rb deleted file mode 100755 index 0338928e1e..0000000000 --- a/tool/test_for_warn_bundled_gems/test_warn_bundle_exec_shebang.rb +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env ruby - -require "base64" diff --git a/tool/test_for_warn_bundled_gems/test_warn_bundled_gems.rb b/tool/test_for_warn_bundled_gems/test_warn_bundled_gems.rb deleted file mode 100644 index 13168292e3..0000000000 --- a/tool/test_for_warn_bundled_gems/test_warn_bundled_gems.rb +++ /dev/null @@ -1,8 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" -end - -require "mutex_m" -require "rss" diff --git a/tool/test_for_warn_bundled_gems/test_warn_dash_gem.rb b/tool/test_for_warn_bundled_gems/test_warn_dash_gem.rb deleted file mode 100644 index 04ef2a52c0..0000000000 --- a/tool/test_for_warn_bundled_gems/test_warn_dash_gem.rb +++ /dev/null @@ -1,7 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" -end - -require "net/smtp" diff --git a/tool/test_for_warn_bundled_gems/test_warn_dependency.rb b/tool/test_for_warn_bundled_gems/test_warn_dependency.rb deleted file mode 100644 index 9be3a2f6d9..0000000000 --- a/tool/test_for_warn_bundled_gems/test_warn_dependency.rb +++ /dev/null @@ -1,8 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" - gem "activesupport", "7.0.7.2" -end - -require "active_support/all" diff --git a/tool/test_for_warn_bundled_gems/test_warn_sub_feature.rb b/tool/test_for_warn_bundled_gems/test_warn_sub_feature.rb deleted file mode 100644 index bf7eb3572d..0000000000 --- a/tool/test_for_warn_bundled_gems/test_warn_sub_feature.rb +++ /dev/null @@ -1,7 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" -end - -require "bigdecimal/util" diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb index b19f68bac4..1257a92d38 100644 --- a/tool/transcode-tblgen.rb +++ b/tool/transcode-tblgen.rb @@ -1078,11 +1078,7 @@ if __FILE__ == $0 end libs1 = $".dup - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%') - else - erb = ERB.new(src, nil, '%') - end + erb = ERB.new(src, trim_mode: '%') erb.filename = arg erb_result = erb.result(binding) libs2 = $".dup diff --git a/tool/update-NEWS-gemlist.rb b/tool/update-NEWS-gemlist.rb index 8e4d39046b..0b5503580d 100755 --- a/tool/update-NEWS-gemlist.rb +++ b/tool/update-NEWS-gemlist.rb @@ -6,22 +6,27 @@ prevs = [prev, prev.sub(/\.\d+\z/, '')] update = ->(list, type, desc = "updated") do item = ->(mark = "* ") do - "The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + + "### The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + list.map {|g, v|"#{mark}#{g} #{v}\n"}.join("") + "\n" end - news.sub!(/^(?:\*( +))?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do + news.sub!(/^(?:\*( +)|#+ *)?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do item["#{$1&.<< " "}*#{$2 || ' '}"] end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+( *\* *).*)*)*\n+\K/) do item[$1 || "* "] end end -ARGV.each do |type| - last = JSON.parse(File.read("#{type}_gems.json"))['gems'].filter_map do |g| + +load_gems_json = ->(type) do + JSON.parse(File.read("#{type}_gems.json"))['gems'].filter_map do |g| v = g['versions'].values_at(*prevs).compact.first g = g['gem'] g = 'RubyGems' if g == 'rubygems' [g, v] if v end.to_h +end + +ARGV.each do |type| + last = load_gems_json[type] changed = File.foreach("gems/#{type}_gems").filter_map do |l| next if l.start_with?("#") g, v = l.split(" ", 3) @@ -32,7 +37,13 @@ ARGV.each do |type| update[changed, type] or next if added and !added.empty? if type == 'bundled' - update[added, type, 'promoted from default gems'] or next + default_gems = load_gems_json['default'] + promoted = {} + added.delete_if do |k, v| + default_gems.key?(k) && promoted[k] = v + end + update[added, type, 'added'] + update[promoted, type, 'promoted from default gems'] or next else update[added, type, 'added'] or next end diff --git a/tool/update-NEWS-refs.rb b/tool/update-NEWS-refs.rb index 2b19f0fdaa..f48cac5ee1 100644 --- a/tool/update-NEWS-refs.rb +++ b/tool/update-NEWS-refs.rb @@ -13,8 +13,9 @@ if links.empty? || lines.last != "" raise "NEWS.md must end with a sequence of links" end -labels = links.keys.select {|k| !(k.start_with?("Feature") || k.start_with?("Bug"))} -new_src = lines.join("\n").gsub(/\[?\[((?:Feature|Bug)\s+#(\d+))\]\]?/) do +trackers = ["Feature", "Bug", "Misc"] +labels = links.keys.reject {|k| k.start_with?(*trackers)} +new_src = lines.join("\n").gsub(/\[?\[(#{Regexp.union(trackers)}\s+#(\d+))\]\]?/) do links[$1] ||= "https://bugs.ruby-lang.org/issues/#$2" "[[#$1]]" end.gsub(/\[\[#{Regexp.union(labels)}\]\]?/) do @@ -22,7 +23,7 @@ end.gsub(/\[\[#{Regexp.union(labels)}\]\]?/) do end.chomp + "\n\n" label_width = links.max_by {|k, _| k.size}.first.size + 4 -redmine_links, non_redmine_links = links.partition {|k,| k =~ /\A(Feature|Bug)\s+#\d+\z/ } +redmine_links, non_redmine_links = links.partition {|k,| k =~ /\A#{Regexp.union(trackers)}\s+#\d+\z/ } (redmine_links.sort_by {|k,| k[/\d+/].to_i } + non_redmine_links.reverse).each do |k, v| new_src << "[#{k}]:".ljust(label_width) << v << "\n" diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb index 2842516cac..dec6b49cee 100755 --- a/tool/update-bundled_gems.rb +++ b/tool/update-bundled_gems.rb @@ -1,4 +1,4 @@ -#!ruby -pla +#!ruby -alpF\s+|#.* BEGIN { require 'rubygems' date = nil @@ -9,7 +9,7 @@ output = STDERR if ARGF.file == STDIN END { output.print date.strftime("latest_date=%F") if date } -unless /^[^#]/ !~ (gem = $F[0]) +if gem = $F[0] ver = Gem::Version.new($F[1]) (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| s.platform == "ruby" && s.name == gem @@ -22,7 +22,10 @@ unless /^[^#]/ !~ (gem = $F[0]) else uri = $F[2] end - date = gem.date if !date or gem.date && gem.date > date + if (!date or gem.date && gem.date > date) and gem.date.to_i != 315_619_200 + # DEFAULT_SOURCE_DATE_EPOCH is meaningless + date = gem.date + end if $F[3] if $F[3].include?($F[1]) $F[3][$F[1]] = gem.version.to_s diff --git a/tool/update-deps b/tool/update-deps index 2a07d55e37..2d4a5674be 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -17,6 +17,14 @@ # 3. Use --fix to fix makefiles. # Ex. ./ruby tool/update-deps --fix # +# Usage to create a depend file initially: +# 1. Copy the dependency section from the Makefile generated by extconf.rb. +# Ex. ext/cgi/escape/Makefile +# 2. Add `# AUTOGENERATED DEPENDENCIES START` and `# AUTOGENERATED DEPENDENCIES END` +# sections to top and end of the depend file. +# 3. Run tool/update-deps --fix to fix the depend file. +# 4. Commit the depend file. +# # Other usages: # * Fix makefiles using previously detected dependency problems # Ex. ruby tool/update-deps --actual-fix [file] @@ -88,6 +96,15 @@ result.each {|k,v| # They can be referenced as $(top_srcdir)/filename. # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' FILES_IN_SOURCE_DIRECTORY = %w[ + prism/api_node.c + prism/ast.h + prism/diagnostic.c + prism/diagnostic.h + prism/node.c + prism/prettyprint.c + prism/serialize.c + prism/token_type.c + prism/version.h ] # Files built in the build directory (except extconf.h). @@ -149,14 +166,6 @@ FILES_NEED_VPATH = %w[ enc/trans/single_byte.c enc/trans/utf8_mac.c enc/trans/utf_16_32.c - - prism/api_node.c - prism/ast.h - prism/node.c - prism/prettyprint.c - prism/serialize.c - prism/token_type.c - prism/version.h ] # Multiple files with same filename. @@ -204,7 +213,7 @@ def in_makefile(target, source) when %r{\Acoroutine/} then source2 = "{$(VPATH)}$(COROUTINE_H)" else source2 = "$(top_srcdir)/#{source}" end - ["common.mk", target2, source2] + ["depend", target2, source2] when %r{\Aenc/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source @@ -317,6 +326,8 @@ def read_make_deps(cwd) deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp next if /\.o\z/ !~ target.to_s next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source) + next if /libzjit.o\z/ =~ target.to_s # skip ZJIT Rust object (no corresponding C source) + next if /target\/release\/libruby.o\z/ =~ target.to_s # skip YJIT+ZJIT Rust object (no corresponding C source) next if /\.bundle\// =~ target.to_s next if /\A\./ =~ target.to_s # skip rules such as ".c.o" #p [curdir, target, deps] diff --git a/tool/zjit_bisect.rb b/tool/zjit_bisect.rb new file mode 100755 index 0000000000..997c572f51 --- /dev/null +++ b/tool/zjit_bisect.rb @@ -0,0 +1,158 @@ +#!/usr/bin/env ruby +require 'logger' +require 'optparse' +require 'shellwords' +require 'tempfile' +require 'timeout' + +ARGS = {timeout: 5} +OptionParser.new do |opts| + opts.banner += " <path_to_ruby> -- <options>" + opts.on("--timeout=TIMEOUT_SEC", "Seconds until child process is killed") do |timeout| + ARGS[:timeout] = Integer(timeout) + end + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end +end.parse! + +usage = "Usage: zjit_bisect.rb <path_to_ruby> -- <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +raise(usage) if OPTIONS.empty? +LOGGER = Logger.new($stdout) + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +# Attempt to reduce the `items` argument as much as possible, returning the +# shorter version. `fixed` will always be used as part of the items when +# running `command`. +# `command` should return True if the command succeeded (the failure did not +# reproduce) and False if the command failed (the failure reproduced). +def bisect_impl(command, fixed, items, indent="") + LOGGER.info("#{indent}step fixed[#{fixed.length}] and items[#{items.length}]") + while items.length > 1 + LOGGER.info("#{indent}#{fixed.length + items.length} candidates") + # Return two halves of the given list. For odd-length lists, the second + # half will be larger. + half = items.length / 2 + left = items[0...half] + right = items[half..] + if !command.call(fixed + left) + items = left + next + end + if !command.call(fixed + right) + items = right + next + end + # We need something from both halves to trigger the failure. Try + # holding each half fixed and bisecting the other half to reduce the + # candidates. + new_right = bisect_impl(command, fixed + left, right, indent + "< ") + new_left = bisect_impl(command, fixed + new_right, left, indent + "> ") + return new_left + new_right + end + items +end + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +def run_bisect(command, items) + LOGGER.info("Verifying items") + if command.call(items) + raise StandardError.new("Command succeeded with full items") + end + if !command.call([]) + raise StandardError.new("Command failed with empty items") + end + bisect_impl(command, [], items) +end + +def add_zjit_options cmd + if RUBY == "make" + # Automatically detect that we're running a make command instead of a Ruby + # one. Pass the bisection options via RUN_OPTS/SPECOPTS instead. + zjit_opts = cmd.select { |arg| arg.start_with?("--zjit") } + run_opts_index = cmd.find_index { |arg| arg.start_with?("RUN_OPTS=") } + specopts_index = cmd.find_index { |arg| arg.start_with?("SPECOPTS=") } + if run_opts_index + run_opts = Shellwords.split(cmd[run_opts_index].delete_prefix("RUN_OPTS=")) + run_opts.concat(zjit_opts) + cmd[run_opts_index] = "RUN_OPTS=#{run_opts.shelljoin}" + elsif specopts_index + specopts = Shellwords.split(cmd[specopts_index].delete_prefix("SPECOPTS=")) + specopts.concat(zjit_opts) + cmd[specopts_index] = "SPECOPTS=#{specopts.shelljoin}" + else + raise "Expected RUN_OPTS or SPECOPTS to be present in make command" + end + cmd = cmd - zjit_opts + end + cmd +end + +def run_ruby *cmd + cmd = add_zjit_options(cmd) + pid = Process.spawn(*cmd, { + in: :close, + out: [File::NULL, File::RDWR], + err: [File::NULL, File::RDWR], + }) + begin + status = Timeout.timeout(ARGS[:timeout]) do + Process::Status.wait(pid) + end + rescue Timeout::Error + Process.kill("KILL", pid) + LOGGER.warn("Timed out after #{ARGS[:timeout]} seconds") + status = Process::Status.wait(pid) + end + + status +end + +def run_with_jit_list(ruby, options, jit_list) + # Make a new temporary file containing the JIT list + Tempfile.create("jit_list") do |temp_file| + temp_file.write(jit_list.join("\n")) + temp_file.flush + temp_file.close + # Run the JIT with the temporary file + run_ruby ruby, "--zjit-allowed-iseqs=#{temp_file.path}", *options + end +end + +# Try running with no JIT list to get a stable baseline +unless run_with_jit_list(RUBY, OPTIONS, []).success? + cmd = [RUBY, "--zjit-allowed-iseqs=/dev/null", *OPTIONS].shelljoin + raise "The command failed unexpectedly with an empty JIT list. To reproduce, try running the following: `#{cmd}`" +end +# Collect the JIT list from the failing Ruby process +jit_list = nil +Tempfile.create "jit_list" do |temp_file| + run_ruby RUBY, "--zjit-log-compiled-iseqs=#{temp_file.path}", *OPTIONS + jit_list = File.readlines(temp_file.path).map(&:strip).reject(&:empty?) +end +LOGGER.info("Starting with JIT list of #{jit_list.length} items.") +# Try running without the optimizer +status = run_with_jit_list(RUBY, ["--zjit-disable-hir-opt", *OPTIONS], jit_list) +if status.success? + LOGGER.warn "*** Command suceeded with HIR optimizer disabled. HIR optimizer is probably at fault. ***" +end +# Now narrow it down +command = lambda do |items| + run_with_jit_list(RUBY, OPTIONS, items).success? +end +result = run_bisect(command, jit_list) +File.open("jitlist.txt", "w") do |file| + file.puts(result) +end +puts "Run:" +jitlist_path = File.expand_path("jitlist.txt") +puts add_zjit_options([RUBY, "--zjit-allowed-iseqs=#{jitlist_path}", *OPTIONS]).shelljoin +puts "Reduced JIT list (available in jitlist.txt):" +puts result diff --git a/tool/zjit_iongraph.html b/tool/zjit_iongraph.html new file mode 100644 index 0000000000..993cce9045 --- /dev/null +++ b/tool/zjit_iongraph.html @@ -0,0 +1,551 @@ +<!-- Copyright Mozilla and licensed under Mozilla Public License Version 2.0. + Source can be found at https://github.com/mozilla-spidermonkey/iongraph --> +<!-- Generated by `npm run build-www` on + 39b04fa18f23cbf3fd2ca7339a45341ff3351ba1 in tekknolagi/iongraph --> +<!DOCTYPE html> + +<head> + <title>iongraph</title> + <style>/* iongraph-specific styles inspired by Tachyons */ + +:root { + --ig-size-1: 1rem; + --ig-size-2: 2rem; + --ig-size-3: 4rem; + --ig-size-4: 8rem; + --ig-size-5: 16rem; + + --ig-spacing-1: .25rem; + --ig-spacing-2: .5rem; + --ig-spacing-3: 1rem; + --ig-spacing-4: 2rem; + --ig-spacing-5: 4rem; + --ig-spacing-6: 8rem; + --ig-spacing-7: 16rem; + + --ig-text-color: black; + --ig-text-color-dim: #777; + --ig-background-primary: #ffb54e; + --ig-background-light: white; + --ig-border-color: #0c0c0d; + + --ig-block-header-color: #0c0c0d; + --ig-loop-header-color: #1fa411; + --ig-movable-color: #1048af; + --ig-rob-color: #444; + --ig-in-worklist-color: red; + + --ig-block-selected: #ffc863; + --ig-block-last-selected: #ffb54e; + + --ig-highlight-0: #ffb54e; + --ig-highlight-1: #ffb5c5; + --ig-highlight-2: #a4cbff; + --ig-highlight-3: #8be182; + --ig-highlight-4: #d9a4fd; + + --ig-flash-color: #ffb54e; + + /* + * The heatmap of sample counts will effectively be sampled from a gradient: + * + * |----------|---------------------------------------| + * cold "cool" hot + * + * The "cold" color will simply be transparent. Therefore, the "cool" + * threshold indicates where the instruction will be fully colored and + * noticeable to the user. + */ + --ig-hot-color: #ff849e; + --ig-cool-color: #ffe546; + --ig-cool-threshold: 0.2; +} + +a.ig-link-normal { + color: inherit; + text-decoration: inherit; +} + +.ig-flex { + display: flex; +} + +.ig-flex-column { + flex-direction: column; +} + +.ig-flex-basis-0 { + flex-basis: 0; +} + +.ig-flex-grow-1 { + flex-grow: 1; +} + +.ig-flex-shrink-0 { + flex-shrink: 0; +} + +.ig-flex-shrink-1 { + flex-shrink: 1; +} + +.ig-items-center { + align-items: center; +} + +.ig-relative { + position: relative; +} + +.ig-absolute { + position: absolute; +} + +.ig-absolute-fill { + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.ig-g1 { + gap: var(--ig-spacing-1); +} + +.ig-g2 { + gap: var(--ig-spacing-2); +} + +.ig-g3 { + gap: var(--ig-spacing-3); +} + +.ig-w1 { + width: var(--ig-size-1); +} + +.ig-w2 { + width: var(--ig-size-2); +} + +.ig-w3 { + width: var(--ig-size-3); +} + +.ig-w4 { + width: var(--ig-size-4); +} + +.ig-w5 { + width: var(--ig-size-5); +} + +.ig-w-100 { + width: 100%; +} + +.ig-ba { + border-style: solid; + border-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bt { + border-top-style: solid; + border-top-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-br { + border-right-style: solid; + border-right-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bb { + border-bottom-style: solid; + border-bottom-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bl { + border-left-style: solid; + border-left-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-pa1 { + padding: var(--ig-spacing-1); +} + +.ig-pa2 { + padding: var(--ig-spacing-2); +} + +.ig-pa3 { + padding: var(--ig-spacing-3); +} + +.ig-ph1 { + padding-left: var(--ig-spacing-1); + padding-right: var(--ig-spacing-1); +} + +.ig-ph2 { + padding-left: var(--ig-spacing-2); + padding-right: var(--ig-spacing-2); +} + +.ig-ph3 { + padding-left: var(--ig-spacing-3); + padding-right: var(--ig-spacing-3); +} + +.ig-pv1 { + padding-top: var(--ig-spacing-1); + padding-bottom: var(--ig-spacing-1); +} + +.ig-pv2 { + padding-top: var(--ig-spacing-2); + padding-bottom: var(--ig-spacing-2); +} + +.ig-pv3 { + padding-top: var(--ig-spacing-3); + padding-bottom: var(--ig-spacing-3); +} + +.ig-pt1 { + padding-top: var(--ig-spacing-1); +} + +.ig-pt2 { + padding-top: var(--ig-spacing-2); +} + +.ig-pt3 { + padding-top: var(--ig-spacing-3); +} + +.ig-pr1 { + padding-right: var(--ig-spacing-1); +} + +.ig-pr2 { + padding-right: var(--ig-spacing-2); +} + +.ig-pr3 { + padding-right: var(--ig-spacing-3); +} + +.ig-pb1 { + padding-bottom: var(--ig-spacing-1); +} + +.ig-pb2 { + padding-bottom: var(--ig-spacing-2); +} + +.ig-pb3 { + padding-bottom: var(--ig-spacing-3); +} + +.ig-pl1 { + padding-left: var(--ig-spacing-1); +} + +.ig-pl2 { + padding-left: var(--ig-spacing-2); +} + +.ig-pl3 { + padding-left: var(--ig-spacing-3); +} + +.ig-f1 { + font-size: 3rem; +} + +.ig-f2 { + font-size: 2.25rem; +} + +.ig-f3 { + font-size: 1.5rem; +} + +.ig-f4 { + font-size: 1.25rem; +} + +.ig-f5 { + font-size: 1rem; +} + +.ig-f6 { + font-size: .875rem; +} + +.ig-f7 { + font-size: .75rem; +} + +.ig-text-normal { + color: var(--ig-text-color); +} + +.ig-text-dim { + color: var(--ig-text-color-dim); +} + +.ig-tl { + text-align: left; +} + +.ig-tr { + text-align: right; +} + +.ig-tc { + text-align: center; +} + +.ig-bg-white { + background-color: var(--ig-background-light); +} + +.ig-bg-primary { + background-color: var(--ig-background-primary); +} + +.ig-overflow-hidden { + overflow: hidden; +} + +.ig-overflow-auto { + overflow: auto; +} + +.ig-overflow-x-auto { + overflow-x: auto; +} + +.ig-overflow-y-auto { + overflow-y: auto; +} + +.ig-hide-if-empty:empty { + display: none; +} + +/* Non-utility styles */ + +.ig-graph { + color: var(--ig-text-color); + position: absolute; + left: 0; + top: 0; + isolation: isolate; +} + +.ig-block { + position: absolute; + + .ig-block-header { + font-weight: bold; + text-align: center; + background-color: var(--ig-block-header-color); + color: white; + padding: 0 1em; + border: 1px solid var(--ig-border-color); + border-width: 1px 1px 0; + } + + .ig-instructions { + padding: 0.5em; + border: 1px solid var(--ig-border-color); + border-width: 0 1px 1px; + + table { + border-collapse: collapse; + } + + td, + th { + white-space: nowrap; + padding: 0.1em 0.5em; + } + + th { + font-weight: normal; + } + } + + &.ig-selected { + outline: 4px solid var(--ig-block-selected); + } + + &.ig-last-selected { + outline-color: var(--ig-block-last-selected); + } +} + +.ig-block-att-loopheader { + .ig-block-header { + background-color: var(--ig-loop-header-color); + } +} + +.ig-block-att-splitedge { + .ig-instructions { + border-style: dotted; + border-width: 0 2px 2px; + } +} + +.ig-ins-num { + text-align: right; + cursor: pointer; +} + +.ig-ins-type { + text-align: right; +} + +.ig-ins-samples { + font-size: 0.875em; + text-align: right; + cursor: pointer; +} + +.ig-use { + padding: 0 0.25em; + border-radius: 2px; + cursor: pointer; +} + +.ig-edge-label { + position: absolute; + font-size: 0.8em; + line-height: 1; + bottom: -1em; + padding-left: 4px; +} + +.ig-ins-att-RecoveredOnBailout { + color: var(--ig-rob-color); +} + +.ig-ins-att-Movable { + color: var(--ig-movable-color); +} + +.ig-ins-att-Guard { + text-decoration: underline; +} + +.ig-ins-att-InWorklist { + color: var(--ig-in-worklist-color); +} + +.ig-can-flash { + transition: outline-color 1s ease-out; + outline: 3px solid color-mix(in srgb, var(--ig-flash-color) 0%, transparent); +} + +.ig-flash { + transition: outline-color 0s; + outline-color: var(--ig-flash-color); +} + +.ig-hotness { + --ig-hotness: 0; + + --ig-cold-color: color-mix(in srgb, var(--ig-cool-color) 20%, transparent); + background-color: + /* cool <-> hot */ + color-mix(in oklab, + /* cold <-> cool */ + color-mix(in oklab, + /* dead or cold */ + color-mix(in srgb, transparent, var(--ig-cold-color) clamp(0%, calc(var(--ig-hotness) * 100000000%), 100%)), + var(--ig-cool-color) clamp(0%, calc((var(--ig-hotness) / var(--ig-cool-threshold)) * 100%), 100%)), + var(--ig-hot-color) clamp(0%, calc(((var(--ig-hotness) - var(--ig-cool-threshold)) / (1 - var(--ig-cool-threshold))) * 100%), 100%)); +} + +.ig-highlight { + --ig-highlight-color: transparent; + background-color: var(--ig-highlight-color); +}</style> + <style>* { + box-sizing: border-box; +} + +:root { + font-size: 0.875rem; +} + +body { + margin: 0; + background-color: #e5e8ea; + /* Font, and many other styles, taken from the Firefox Profiler/ */ + font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen-Sans, Ubuntu, Noto Sans, Liberation Sans, Cantarell, Helvetica Neue, sans-serif; +} + +#container { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +.tweaks-panel { + position: absolute; + bottom: 0; + right: 0; + padding: 1rem; + border: 1px solid black; + border-width: 1px 0 0 1px; + background-color: white; +}</style> +</head> + +<body> + <script>"use strict";var iongraph=(()=>{var J=Object.defineProperty;var mt=Object.getOwnPropertyDescriptor;var gt=Object.getOwnPropertyNames;var ft=Object.prototype.hasOwnProperty;var bt=(a,t)=>{for(var e in t)J(a,e,{get:t[e],enumerable:!0})},kt=(a,t,e,s)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of gt(t))!ft.call(a,i)&&i!==e&&J(a,i,{get:()=>t[i],enumerable:!(s=mt(t,i))||s.enumerable});return a};var yt=a=>kt(J({},"__esModule",{value:!0}),a);var Yt={};bt(Yt,{StandaloneUI:()=>et,WebUI:()=>tt});function X(a){a.version===void 0&&(a.version=0);for(let t of a.functions)xt(t,a.version);return a.version=1,a}function xt(a,t){for(let e of a.passes){for(let s of e.mir.blocks)wt(s,t);for(let s of e.lir.blocks)It(s,t)}return a}function wt(a,t){t===0&&(a.ptr=(a.id??a.number)+1,a.id=a.number);for(let e of a.instructions)vt(e,t);return a}function vt(a,t){return t===0&&(a.ptr=a.id),a}function It(a,t){t===0&&(a.ptr=a.id??a.number,a.id=a.number);for(let e of a.instructions)Lt(e,t);return a}function Lt(a,t){return t===0&&(a.ptr=a.id,a.mirPtr=null),a}function q(a,t,e){return Math.max(t,Math.min(e,a))}function R(a,t,e,s){return(a-t)*Math.pow(e,s)+t}function k(a,t,e=!1){if(!a)if(e)console.error(t??"Assertion failed");else throw new Error(t??"Assertion failed")}function L(a,t){return k(a,t),a}function Pt(a){return typeof a=="string"?document.createTextNode(a):a}function Bt(a,t){for(let e of t)e&&a.appendChild(Pt(e))}function x(a,t,e,s){let i=document.createElement(a);if(t&&t.length>0){let c=t.filter(l=>!!l);i.classList.add(...c)}return e?.(i),s&&Bt(i,s),i}var st=Object.prototype.hasOwnProperty;function E(a,t){var e,s;if(a===t)return!0;if(a&&t&&(e=a.constructor)===t.constructor){if(e===Date)return a.getTime()===t.getTime();if(e===RegExp)return a.toString()===t.toString();if(e===Array){if((s=a.length)===t.length)for(;s--&&E(a[s],t[s]););return s===-1}if(!e||typeof a=="object"){s=0;for(e in a)if(st.call(a,e)&&++s&&!st.call(t,e)||!(e in t)||!E(a[e],t[e]))return!1;return Object.keys(t).length===s}}return a!==a&&t!==t}function N(a,t,e={}){let s=t,i=[],c={get(){return s},set(l){s=l;for(let o of i)o(s)},valueOf(){return s},toString(){return String(s)},[Symbol.toPrimitive](l){return l==="string"?String(s):s},onChange(l){i.push(l)},initial:t,name:a,min:e.min??0,max:e.max??100,step:e.step??1};return(e.tweaksObject??K).add(c)}var F=class{constructor(t){this.container=t.container,this.tweaks=[],this.callbacks=[]}add(t){let e=this.tweaks.find(r=>r.name===t.name);if(e)return e;this.tweaks.push(t);let s=document.createElement("div");this.container.appendChild(s),s.style.display="flex",s.style.alignItems="center",s.style.justifyContent="end",s.style.gap="0.5rem";let i=t.name.replace(/[a-zA-Z0-9]/g,"_"),c=document.createElement("label");s.appendChild(c),c.innerText=t.name,c.htmlFor=`tweak-${i}-input`;let l=document.createElement("input");s.appendChild(l),l.type="number",l.value=String(t),l.id=`tweak-${i}-input`,l.style.width="4rem",l.addEventListener("input",()=>{t.set(l.valueAsNumber)});let o=document.createElement("input");s.appendChild(o),o.type="range",o.value=String(t),o.min=String(t.min),o.max=String(t.max),o.step=t.step===0?"any":String(t.step),o.addEventListener("input",()=>{t.set(o.valueAsNumber)});let n=document.createElement("button");return s.appendChild(n),n.innerText="Reset",n.disabled=t.get()===t.initial,n.addEventListener("click",()=>{t.set(t.initial)}),t.onChange(r=>{l.value=String(r),o.value=String(r),n.disabled=t.get()===t.initial;for(let d of this.callbacks)d(t)}),t}onTweak(t){this.callbacks.push(t)}},nt=document.createElement("div");nt.classList.add("tweaks-panel");var K=new F({container:nt});K.onTweak(a=>{window.dispatchEvent(new CustomEvent("tweak",{detail:a}))});window.tweaks=K;var Nt=document.createElement("div"),St=new F({container:Nt}),ot=N("Test Value",3,{tweaksObject:St});ot.set(4);ot=4;var dt=N("Debug?",0,{min:0,max:1}),P=20,C=44,I=16,S=60,T=12,H=36,it=16,D=16,Tt=N("Layout Iterations",2,{min:0,max:6}),rt=N("Nearly Straight Threshold",30,{min:0,max:200}),$t=N("Nearly Straight Iterations",8,{min:0,max:10}),at=N("Stop At Pass",30,{min:0,max:30}),Mt=1.5,Et=.01,Ct=1,Ht=.1,z=40;function _(a){return a.mir.attributes.includes("loopheader")}function Dt(a){return a.loopHeight!==void 0}function $(a){if(k(a),Dt(a))return a;throw new Error(`Block ${a.id} is not a pseudo LoopHeader`)}var j=1,lt=2,U=4,O=0,W=1,w=new Proxy(console,{get(a,t){let e=a[t];return typeof e!="function"?e:+dt?e.bind(a):()=>{}}}),V=class{constructor(t,e,s={}){this.viewport=t;let i=t.getBoundingClientRect();this.viewportSize={x:i.width,y:i.height},this.graphContainer=document.createElement("div"),this.graphContainer.classList.add("ig-graph"),this.graphContainer.style.transformOrigin="top left",this.viewport.appendChild(this.graphContainer),this.sampleCounts=s.sampleCounts,this.maxSampleCounts=[0,0],this.heatmapMode=O;for(let[n,r]of this.sampleCounts?.totalLineHits??[])this.maxSampleCounts[O]=Math.max(this.maxSampleCounts[O],r);for(let[n,r]of this.sampleCounts?.selfLineHits??[])this.maxSampleCounts[W]=Math.max(this.maxSampleCounts[W],r);this.size={x:0,y:0},this.numLayers=0,this.zoom=1,this.translation={x:0,y:0},this.animating=!1,this.targetZoom=1,this.targetTranslation={x:0,y:0},this.startMousePos={x:0,y:0},this.lastMousePos={x:0,y:0},this.selectedBlockPtrs=new Set,this.lastSelectedBlockPtr=0,this.nav={visited:[],currentIndex:-1,siblings:[]},this.highlightedInstructions=[],this.instructionPalette=s.instructionPalette??[0,1,2,3,4].map(n=>`var(--ig-highlight-${n})`),this.blocks=e.mir.blocks.map(n=>{let r={ptr:n.ptr,id:n.id,mir:n,lir:e.lir.blocks.find(d=>d.id===n.id)??null,preds:[],succs:[],el:void 0,size:{x:0,y:0},layer:-1,loopID:-1,layoutNode:void 0};if(r.mir.attributes.includes("loopheader")){let d=r;d.loopHeight=0,d.parentLoop=null,d.outgoingEdges=[]}return k(r.ptr,"blocks must always have non-null ptrs"),r}),this.blocksByID=new Map,this.blocksByPtr=new Map,this.insPtrsByID=new Map,this.insIDsByPtr=new Map,this.loops=[];for(let n of this.blocks){this.blocksByID.set(n.id,n),this.blocksByPtr.set(n.ptr,n);for(let r of n.mir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id);if(n.lir)for(let r of n.lir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id)}for(let n of this.blocks)if(n.preds=n.mir.predecessors.map(r=>L(this.blocksByID.get(r))),n.succs=n.mir.successors.map(r=>L(this.blocksByID.get(r))),_(n)){let r=n.preds.filter(d=>d.mir.attributes.includes("backedge"));k(r.length===1),n.backedge=r[0]}for(let n of this.blocks)n.el=this.renderBlock(n);for(let n of this.blocks)n.size={x:n.el.clientWidth,y:n.el.clientHeight};let[c,l,o]=this.layout();this.render(c,l,o),this.addEventListeners()}layout(){let[t,e]=this.findLayoutRoots();w.log("Layout roots:",t.map(l=>l.id));for(let l of[...t,...e]){let o=l;o.loopHeight=0,o.parentLoop=null,o.outgoingEdges=[],Object.defineProperty(o,"backedge",{get(){throw new Error("Accessed .backedge on a pseudo loop header! Don't do that.")},configurable:!0})}for(let l of t)w.group("findLoops"),this.findLoops(l),w.groupEnd();for(let l of t)w.group("layer"),this.layer(l),w.groupEnd();for(let l of e)l.layer=0,l.loopID=l.id;let s=this.makeLayoutNodes();this.straightenEdges(s);let i=this.finagleJoints(s),c=this.verticalize(s,i);return[s,c,i]}findLayoutRoots(){let t=[],e=[],s=this.blocks.filter(i=>i.preds.length===0);for(let i of s){let c=i;if(i.mir.attributes.includes("osr")){k(i.succs.length>0);let l=i.succs[0];c=l;for(let o=0;;o++){if(o>=1e7)throw new Error("likely infinite loop");let n=c.preds.filter(r=>!r.mir.attributes.includes("osr")&&!r.mir.attributes.includes("backedge"));if(n.length===0)break;c=n[0]}c!==l?e.push(i):c=i}t.includes(c)||t.push(c)}return[t,e]}findLoops(t,e=null){if(e===null&&(e=[t.id]),!(t.loopID>=0)){if(w.log("block:",t.id,t.mir.loopDepth,"loopIDsByDepth:",e),w.log(t.mir.attributes),_(t)){let s=e[e.length-1],i=$(this.blocksByID.get(s));t.parentLoop=i,e=[...e,t.id],w.log("Block",t.id,"is true loop header, loopIDsByDepth is now",e)}if(t.mir.loopDepth>e.length-1&&(t.mir.loopDepth=e.length-1,w.log("Block",t.id,"has been forced back to loop depth",t.mir.loopDepth)),t.mir.loopDepth<e.length-1&&(e=e.slice(0,t.mir.loopDepth+1),w.log("Block",t.id,"has low loop depth, therefore we exited a loop. loopIDsByDepth:",e)),t.loopID=e[t.mir.loopDepth],!t.mir.attributes.includes("backedge"))for(let s of t.succs)this.findLoops(s,e)}}layer(t,e=0){if(w.log("block",t.id,"layer",e),t.mir.attributes.includes("backedge")){t.layer=t.succs[0].layer;return}if(e<=t.layer)return;t.layer=Math.max(t.layer,e),this.numLayers=Math.max(t.layer+1,this.numLayers);let s=$(this.blocksByID.get(t.loopID));for(;s;)s.loopHeight=Math.max(s.loopHeight,t.layer-s.layer+1),s=s.parentLoop;for(let i of t.succs)i.mir.loopDepth<t.mir.loopDepth?$(this.blocksByID.get(t.loopID)).outgoingEdges.push(i):this.layer(i,e+1);if(_(t))for(let i of t.outgoingEdges)this.layer(i,e+t.loopHeight)}makeLayoutNodes(){w.group("makeLayoutNodes");function t(o,n,r){o.dstNodes[n]=r,r.srcNodes.includes(o)||r.srcNodes.push(o),w.log("connected",o.id,"to",r.id)}let e;{let o={};for(let n of this.blocks)o[n.layer]||(o[n.layer]=[]),o[n.layer].push(n);e=Object.entries(o).map(([n,r])=>[Number(n),r]).sort((n,r)=>n[0]-r[0]).map(([n,r])=>r)}let s=0,i=e.map(()=>[]),c=[],l=new Map;for(let[o,n]of e.entries()){w.group("layer",o,"blocks",n.map(u=>u.id));let r=[];for(let u of n)for(let h=c.length-1;h>=0;h--){let p=c[h];p.dstBlock===u&&(r.unshift(p),c.splice(h,1))}let d=new Map;for(let u of c){let h,p=d.get(u.dstBlock.id);if(p)t(u.src,u.srcPort,p),h=p;else{let m={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:u.dstBlock,jointOffsets:[],flags:0};t(u.src,u.srcPort,m),i[o].push(m),d.set(u.dstBlock.id,m),h=m,w.log("Created dummy",h.id,"on the way to block",u.dstBlock.id)}u.src=h,u.srcPort=0}let b=[];for(let u of n){let h=$(this.blocksByID.get(u.loopID));for(;_(h);){let p=b.find(f=>f.loopID===h.id);p?p.block=u:b.push({loopID:h.id,block:u});let m=h.parentLoop;if(!m)break;h=m}}let g=[];for(let u of n){let h={id:s++,pos:{x:P,y:P},size:u.size,block:u,srcNodes:[],dstNodes:[],jointOffsets:[],flags:0};for(let p of r)p.dstBlock===u&&t(p.src,p.srcPort,h);i[o].push(h),u.layoutNode=h;for(let p of b.filter(m=>m.block===u)){let m=$(this.blocksByID.get(p.loopID)).backedge,f={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:m,jointOffsets:[],flags:0},y=l.get(m);y?t(f,0,y):(f.flags|=U,t(f,0,m.layoutNode)),i[o].push(f),l.set(m,f)}if(u.mir.attributes.includes("backedge"))t(u.layoutNode,0,u.succs[0].layoutNode);else for(let[p,m]of u.succs.entries())m.mir.attributes.includes("backedge")?g.push({src:h,srcPort:p,dstBlock:m}):c.push({src:h,srcPort:p,dstBlock:m})}for(let u of g){let h=L(l.get(u.dstBlock));t(u.src,u.srcPort,h)}w.groupEnd()}w.log("Pruning backedge dummies");{let o=[];for(let r of At(i))r.srcNodes.length===0&&o.push(r);let n=new Set;for(let r of o){let d=r;for(;d.block===null&&d.srcNodes.length===0;)Ot(d),n.add(d),k(d.dstNodes.length===1),d=d.dstNodes[0]}for(let r of i)for(let d=r.length-1;d>=0;d--)n.has(r[d])&&r.splice(d,1)}w.log("Marking leftmost and rightmost dummies");for(let o of i){for(let n=0;n<o.length&&o[n].block===null;n++)o[n].flags|=j;for(let n=o.length-1;n>=0&&o[n].block===null;n--)o[n].flags|=lt}w.log("Verifying integrity of all nodes");for(let o of i)for(let n of o){n.block?k(n.dstNodes.length===n.block.succs.length,`expected node ${n.id} for block ${n.block.id} to have ${n.block.succs.length} destination nodes, but got ${n.dstNodes.length} instead`):k(n.dstNodes.length===1,`expected dummy node ${n.id} to have only one destination node, but got ${n.dstNodes.length} instead`);for(let r=0;r<n.dstNodes.length;r++)k(n.dstNodes[r]!==void 0,`dst slot ${r} of node ${n.id} was undefined`)}return w.groupEnd(),i}straightenEdges(t){let e=g=>{for(let u=0;u<g.length-1;u++){let h=g[u],p=g[u+1],m=h.block===null&&p.block!==null,f=h.pos.x+h.size.x+(m?I:0)+C;p.pos.x=Math.max(p.pos.x,f)}},s=()=>{for(let g of t)for(let u of g){if(u.block===null)continue;let h=u.block.loopID!==null?$(this.blocksByID.get(u.block.loopID)):null;if(h){let p=h.layoutNode;u.pos.x=Math.max(u.pos.x,p.pos.x)}}},i=()=>{let g=new Map;for(let u of Q(t)){let h=u.dstBlock,p=u.pos.x;g.set(h,Math.max(g.get(h)??0,p))}for(let u of Q(t)){let h=u.dstBlock,p=g.get(h);k(p,`no position for backedge ${h.id}`),u.pos.x=p}for(let u of t)e(u)},c=()=>{let g=new Map;for(let u of t){let h=0,p=0;for(;h<u.length;h++)if(!(u[h].flags&j)){p=u[h].pos.x;break}for(h-=1,p-=C+I;h>=0;h--){let m=u[h];k(m.block===null&&m.flags&j);let f=p;for(let y of m.srcNodes){let v=y.pos.x+y.dstNodes.indexOf(m)*S;v<f&&(f=v)}m.pos.x=f,p=m.pos.x-C,g.set(m.dstBlock,Math.min(g.get(m.dstBlock)??1/0,f))}}for(let u of Q(t)){if(!(u.flags&j))continue;let h=g.get(u.dstBlock);k(h,`no position for run to block ${u.dstBlock.id}`),u.pos.x=h}},l=()=>{for(let g=0;g<t.length-1;g++){let u=t[g];e(u);let h=-1;for(let p of u)for(let[m,f]of p.dstNodes.entries()){let y=t[g+1].indexOf(f);if(y>h&&f.srcNodes[0]===p){let v=I+S*m,B=I,Z=f.pos.x;f.pos.x=Math.max(f.pos.x,p.pos.x+v-B),f.pos.x!==Z&&(h=y)}}}},o=()=>{for(let g of t){for(let u=g.length-1;u>=0;u--){let h=g[u];if(!h.block||h.block.mir.attributes.includes("backedge"))continue;let p=[];for(let m of h.srcNodes){let f=I+m.dstNodes.indexOf(h)*S,y=I;p.push(m.pos.x+f-(h.pos.x+y))}for(let[m,f]of h.dstNodes.entries()){if(f.block===null&&f.dstBlock.mir.attributes.includes("backedge"))continue;let y=I+m*S,v=I;p.push(f.pos.x+v-(h.pos.x+y))}if(!p.includes(0)){p=p.filter(m=>m>0).sort((m,f)=>m-f);for(let m of p){let f=!1;for(let y=u+1;y<g.length;y++){let v=g[y];if(v.flags<)continue;let B=h.pos.x+m,Z=h.pos.x+m+h.size.x,ht=v.pos.x-C,pt=v.pos.x+v.size.x+C;Z>=ht&&B<=pt&&(f=!0)}if(!f){h.pos.x+=m;break}}}}e(g)}},n=()=>{for(let g=t.length-1;g>=0;g--){let u=t[g];e(u);for(let h of u)for(let p of h.srcNodes){if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}},r=()=>{for(let g=0;g<t.length;g++){let u=t[g];e(u);for(let h of u){if(h.dstNodes.length===0)continue;let p=h.dstNodes[0];if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}};function d(g,u){let h=[];for(let p=0;p<u;p++)for(let m of g)h.push(m);return h}let b=[...d([l,s,i],Tt),i,...d([n,r],$t),o,i,c];k(b.length<=(at.initial??1/0),`STOP_AT_PASS was too small - should be at least ${b.length}`),w.group("Running passes");for(let[g,u]of b.entries())g<at&&(w.log(u.name??u.toString()),u());w.groupEnd()}finagleJoints(t){let e=[];for(let s of t){let i=[];for(let r of s)if(r.jointOffsets=new Array(r.dstNodes.length).fill(0),!r.block?.mir.attributes.includes("backedge"))for(let[d,b]of r.dstNodes.entries()){let g=r.pos.x+I+S*d,u=b.pos.x+I;Math.abs(u-g)<2*T||i.push({x1:g,x2:u,src:r,srcPort:d,dst:b})}i.sort((r,d)=>r.x1-d.x1);let c=[],l=[];t:for(let r of i){let d=r.x2-r.x1>=0?c:l,b=null;for(let g=d.length-1;g>=0;g--){let u=d[g],h=!1;for(let p of u){if(r.dst===p.dst){u.push(r);continue t}let m=Math.min(r.x1,r.x2),f=Math.max(r.x1,r.x2),y=Math.min(p.x1,p.x2),v=Math.max(p.x1,p.x2);if(f>=y&&m<=v){h=!0;break}}if(h)break;b=u}b?b.push(r):d.push([r])}let o=Math.max(0,c.length+l.length-1)*it,n=-o/2;for(let r of[...c.reverse(),...l]){for(let d of r)d.src.jointOffsets[d.srcPort]=n;n+=it}e.push(o)}return k(e.length===t.length),e}verticalize(t,e){let s=new Array(t.length),i=P;for(let c=0;c<t.length;c++){let l=t[c],o=0;for(let n of l)n.pos.y=i,o=Math.max(o,n.size.y);s[c]=o,i+=o+H+e[c]+H}return s}renderBlock(t){let e=document.createElement("div");this.graphContainer.appendChild(e),e.classList.add("ig-block","ig-bg-white");for(let o of t.mir.attributes)e.classList.add(`ig-block-att-${o}`);e.setAttribute("data-ig-block-ptr",`${t.ptr}`),e.setAttribute("data-ig-block-id",`${t.id}`);let s="";t.mir.attributes.includes("loopheader")?s=" (loop header)":t.mir.attributes.includes("backedge")?s=" (backedge)":t.mir.attributes.includes("splitedge")&&(s=" (split edge)");let i=document.createElement("div");i.classList.add("ig-block-header"),i.innerText=`Block ${t.id}${s}`,e.appendChild(i);let c=document.createElement("div");c.classList.add("ig-instructions"),e.appendChild(c);let l=document.createElement("table");if(t.lir){l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + ${this.sampleCounts?` + <col style="width: 1px"> + <col style="width: 1px"> + `:""} + </colgroup> + ${this.sampleCounts?` + <thead> + <tr> + <th></th> + <th></th> + <th class="ig-f6">Total</th> + <th class="ig-f6">Self</th> + </tr> + </thead> + `:""} + `;for(let o of t.lir.instructions)l.appendChild(this.renderLIRInstruction(o))}else{l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + <col style="width: 1px"> + </colgroup> + `;for(let o of t.mir.instructions)l.appendChild(this.renderMIRInstruction(o))}if(c.appendChild(l),t.succs.length===2)for(let[o,n]of[1,0].entries()){let r=document.createElement("div");r.innerText=`${n}`,r.classList.add("ig-edge-label"),r.style.left=`${I+S*o}px`,e.appendChild(r)}return i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",o=>{o.stopPropagation(),o.shiftKey||this.selectedBlockPtrs.clear(),this.setSelection([],t.ptr)}),e}render(t,e,s){for(let n of t)for(let r of n)if(r.block!==null){let d=r.block;d.el.style.left=`${r.pos.x}px`,d.el.style.top=`${r.pos.y}px`}let i=0,c=0;for(let n of t)for(let r of n)i=Math.max(i,r.pos.x+r.size.x+P),c=Math.max(c,r.pos.y+r.size.y+P);let l=document.createElementNS("http://www.w3.org/2000/svg","svg");this.graphContainer.appendChild(l);let o=(n,r)=>{for(let d of n)i=Math.max(i,d+P);for(let d of r)c=Math.max(c,d+P)};for(let n=0;n<t.length;n++){let r=t[n];for(let d of r){d.block||k(d.dstNodes.length===1,`dummy nodes must have exactly one destination, but dummy ${d.id} had ${d.dstNodes.length}`),k(d.dstNodes.length===d.jointOffsets.length,"must have a joint offset for each destination");for(let[b,g]of d.dstNodes.entries()){let u=d.pos.x+I+S*b,h=d.pos.y+d.size.y;if(d.block?.mir.attributes.includes("backedge")){let p=d.block.succs[0],m=d.pos.x,f=d.pos.y+D,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=jt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(d.flags&U){let p=L(g.block),m=d.pos.x+I,f=d.pos.y+D+T,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=zt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(g.block===null&&g.dstBlock.mir.attributes.includes("backedge")){let p=g.pos.x+I,m=g.pos.y+(g.flags&U?D+T:0);if(d.block===null){let f=h-H,y=Ft(u,h,p,m,f,!1);l.appendChild(y),o([u,p],[h,m,f])}else{let f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=_t(u,h,p,m,f);l.appendChild(y),o([u,p],[h,m,f])}}else{let p=g.pos.x+I,m=g.pos.y,f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=Rt(u,h,p,m,f,g.block!==null);l.appendChild(y),o([u,p],[h,m,f])}}}}if(l.setAttribute("width",`${i}`),l.setAttribute("height",`${c}`),this.size={x:i,y:c},+dt)for(let n of t)for(let r of n){let d=document.createElement("div");d.innerHTML=`${r.id}<br><- ${r.srcNodes.map(b=>b.id)}<br>-> ${r.dstNodes.map(b=>b.id)}<br>${r.flags}`,d.style.position="absolute",d.style.border="1px solid black",d.style.backgroundColor="white",d.style.left=`${r.pos.x}px`,d.style.top=`${r.pos.y}px`,d.style.whiteSpace="nowrap",this.graphContainer.appendChild(d)}this.updateHighlightedInstructions(),this.updateHotness()}renderMIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-mir","ig-can-flash",...t.attributes.map(o=>`ig-ins-att-${o}`)),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=`v${t.id}`,s.appendChild(i);let c=document.createElement("td");c.innerHTML=e.replace(/(v)(\d+)/g,(o,n,r)=>`<span class="ig-use ig-highlightable" data-ig-use="${r}">${n}${r}</span>`),s.appendChild(c);let l=document.createElement("td");return l.classList.add("ig-ins-type"),l.innerText=t.type==="None"?"":t.type,s.appendChild(l),i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),c.querySelectorAll(".ig-use").forEach(o=>{o.addEventListener("pointerdown",n=>{n.preventDefault(),n.stopPropagation()}),o.addEventListener("click",n=>{let r=parseInt(L(o.getAttribute("data-ig-use")),10);this.jumpToInstruction(r,{zoom:1})})}),s}renderLIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-lir","ig-hotness"),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=String(t.id),s.appendChild(i);let c=document.createElement("td");if(c.innerText=e,s.appendChild(c),this.sampleCounts){let l=this.sampleCounts?.totalLineHits.get(t.id)??0,o=this.sampleCounts?.selfLineHits.get(t.id)??0,n=document.createElement("td");n.classList.add("ig-ins-samples"),n.classList.toggle("ig-text-dim",l===0),n.innerText=`${l}`,n.title="Color by total count",s.appendChild(n);let r=document.createElement("td");r.classList.add("ig-ins-samples"),r.classList.toggle("ig-text-dim",o===0),r.innerText=`${o}`,r.title="Color by self count",s.appendChild(r);for(let[d,b]of[n,r].entries())b.addEventListener("pointerdown",g=>{g.preventDefault(),g.stopPropagation()}),b.addEventListener("click",()=>{k(d===O||d===W),this.heatmapMode=d,this.updateHotness()})}return i.addEventListener("pointerdown",l=>{l.preventDefault(),l.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),s}renderSelection(){this.graphContainer.querySelectorAll(".ig-block").forEach(t=>{let e=parseInt(L(t.getAttribute("data-ig-block-ptr")),10);t.classList.toggle("ig-selected",this.selectedBlockPtrs.has(e)),t.classList.toggle("ig-last-selected",this.lastSelectedBlockPtr===e)})}removeNonexistentHighlights(){this.highlightedInstructions=this.highlightedInstructions.filter(t=>this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`))}updateHighlightedInstructions(){for(let t of this.highlightedInstructions)k(this.highlightedInstructions.filter(e=>e.ptr===t.ptr).length===1,`instruction ${t.ptr} was highlighted more than once`);this.graphContainer.querySelectorAll(".ig-ins, .ig-use").forEach(t=>{Vt(t)});for(let t of this.highlightedInstructions){let e=this.instructionPalette[t.paletteColor%this.instructionPalette.length],s=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`);if(s){ct(s,e);let i=this.insIDsByPtr.get(t.ptr);this.graphContainer.querySelectorAll(`.ig-use[data-ig-use="${i}"]`).forEach(c=>{ct(c,e)})}}}updateHotness(){this.graphContainer.querySelectorAll(".ig-ins-lir").forEach(t=>{k(t.classList.contains("ig-hotness"));let e=parseInt(L(t.getAttribute("data-ig-ins-id")),10),s=0;this.sampleCounts&&(s=((this.heatmapMode===O?this.sampleCounts.totalLineHits:this.sampleCounts.selfLineHits).get(e)??0)/this.maxSampleCounts[this.heatmapMode]),t.style.setProperty("--ig-hotness",`${s}`)})}addEventListeners(){this.viewport.addEventListener("wheel",e=>{e.preventDefault();let s=this.zoom;if(e.ctrlKey){s=Math.max(Ht,Math.min(Ct,this.zoom*Math.pow(Mt,-e.deltaY*Et)));let c=s/this.zoom-1;this.zoom=s;let{x:l,y:o}=this.viewport.getBoundingClientRect(),n=e.clientX-l-this.translation.x,r=e.clientY-o-this.translation.y;this.translation.x-=n*c,this.translation.y-=r*c}else this.translation.x-=e.deltaX,this.translation.y-=e.deltaY;let i=this.clampTranslation(this.translation,s);this.translation.x=i.x,this.translation.y=i.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerdown",e=>{e.pointerType==="mouse"&&!(e.button===0||e.button===1)||(e.preventDefault(),this.viewport.setPointerCapture(e.pointerId),this.startMousePos={x:e.clientX,y:e.clientY},this.lastMousePos={x:e.clientX,y:e.clientY},this.animating=!1)}),this.viewport.addEventListener("pointermove",e=>{if(!this.viewport.hasPointerCapture(e.pointerId))return;let s=e.clientX-this.lastMousePos.x,i=e.clientY-this.lastMousePos.y;this.translation.x+=s,this.translation.y+=i,this.lastMousePos={x:e.clientX,y:e.clientY};let c=this.clampTranslation(this.translation,this.zoom);this.translation.x=c.x,this.translation.y=c.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerup",e=>{this.viewport.releasePointerCapture(e.pointerId);let s=2,i=this.startMousePos.x-e.clientX,c=this.startMousePos.y-e.clientY;Math.abs(i)<=s&&Math.abs(c)<=s&&this.setSelection([]),this.animating=!1}),new ResizeObserver(e=>{k(e.length===1);let s=e[0].contentRect;this.viewportSize.x=s.width,this.viewportSize.y=s.height}).observe(this.viewport)}setSelection(t,e=0){this.setSelectionRaw(t,e),e?this.nav={visited:[e],currentIndex:0,siblings:[e]}:this.nav={visited:[],currentIndex:-1,siblings:[]}}setSelectionRaw(t,e){this.selectedBlockPtrs.clear();for(let s of[...t,e])this.blocksByPtr.has(s)&&this.selectedBlockPtrs.add(s);this.lastSelectedBlockPtr=this.blocksByPtr.has(e)?e:0,this.renderSelection()}navigate(t){let e=this.lastSelectedBlockPtr;if(t==="down"||t==="up")if(e){let s=L(this.blocksByPtr.get(e)),i=(t==="down"?s.succs:s.preds).map(l=>l.ptr);s.ptr!==this.nav.visited[this.nav.currentIndex]&&(this.nav.visited=[s.ptr],this.nav.currentIndex=0);let c=this.nav.currentIndex+(t==="down"?1:-1);if(0<=c&&c<this.nav.visited.length)this.nav.currentIndex=c,this.nav.siblings=i;else{let l=i[0];l!==void 0&&(t==="down"?(this.nav.visited.push(l),this.nav.currentIndex+=1,k(this.nav.currentIndex===this.nav.visited.length-1)):(this.nav.visited.unshift(l),k(this.nav.currentIndex===0)),this.nav.siblings=i)}this.setSelectionRaw([],this.nav.visited[this.nav.currentIndex])}else{let s=[...this.blocks].sort((n,r)=>n.id-r.id),i=s.filter(n=>n.preds.length===0),c=s.filter(n=>n.succs.length===0),l=t==="down"?i:c,o=l[0];k(o),this.setSelectionRaw([],o.ptr),this.nav={visited:[o.ptr],currentIndex:0,siblings:l.map(n=>n.ptr)}}else if(e!==void 0){let s=this.nav.siblings.indexOf(e);k(s>=0,"currently selected node should be in siblings array");let i=s+(t==="right"?1:-1);0<=i&&i<this.nav.siblings.length&&this.setSelectionRaw([],this.nav.siblings[i])}k(this.nav.visited.length===0||this.nav.siblings.includes(this.nav.visited[this.nav.currentIndex]),"expected currently visited node to be in the siblings array"),k(this.lastSelectedBlockPtr===0||this.nav.siblings.includes(this.lastSelectedBlockPtr),"expected currently selected block to be in siblings array")}toggleInstructionHighlight(t,e){this.removeNonexistentHighlights();let s=this.highlightedInstructions.findIndex(c=>c.ptr===t),i=s>=0;if(e!==void 0&&(i=!e),i)s>=0&&this.highlightedInstructions.splice(s,1);else if(s<0){let c=0;for(;;){if(this.highlightedInstructions.find(l=>l.paletteColor===c)){c+=1;continue}break}this.highlightedInstructions.push({ptr:t,paletteColor:c})}this.updateHighlightedInstructions()}clampTranslation(t,e){let s=z-this.size.x*e,i=this.viewportSize.x-z,c=z-this.size.y*e,l=this.viewportSize.y-z,o=q(t.x,s,i),n=q(t.y,c,l);return{x:o,y:n}}updatePanAndZoom(){let t=this.clampTranslation(this.translation,this.zoom);this.graphContainer.style.transform=`translate(${t.x}px, ${t.y}px) scale(${this.zoom})`}graph2viewport(t,e=this.translation,s=this.zoom){return{x:t.x*s+e.x,y:t.y*s+e.y}}viewport2graph(t,e=this.translation,s=this.zoom){return{x:(t.x-e.x)/s,y:(t.y-e.y)/s}}async goToGraphCoordinates(t,{zoom:e=this.zoom,animate:s=!0}){let i={x:-t.x*e,y:-t.y*e};if(!s){this.animating=!1,this.translation.x=i.x,this.translation.y=i.y,this.zoom=e,this.updatePanAndZoom(),await new Promise(l=>setTimeout(l,0));return}if(this.targetTranslation=i,this.targetZoom=e,this.animating)return;this.animating=!0;let c=performance.now();for(;this.animating;){let l=await new Promise(h=>requestAnimationFrame(h)),o=(l-c)/1e3;c=l;let n=1,r=.01,d=1e-6,b=this.targetTranslation.x-this.translation.x,g=this.targetTranslation.y-this.translation.y,u=this.targetZoom-this.zoom;if(this.translation.x=R(this.translation.x,this.targetTranslation.x,d,o),this.translation.y=R(this.translation.y,this.targetTranslation.y,d,o),this.zoom=R(this.zoom,this.targetZoom,d,o),this.updatePanAndZoom(),Math.abs(b)<=n&&Math.abs(g)<=n&&Math.abs(u)<=r){this.translation.x=this.targetTranslation.x,this.translation.y=this.targetTranslation.y,this.zoom=this.targetZoom,this.animating=!1,this.updatePanAndZoom();break}}await new Promise(l=>setTimeout(l,0))}jumpToBlock(t,{zoom:e=this.zoom,animate:s=!0,viewportPos:i}={}){let c=this.blocksByPtr.get(t);if(!c)return Promise.resolve();let l;return i?l={x:c.layoutNode.pos.x-i.x/e,y:c.layoutNode.pos.y-i.y/e}:l=this.graphPosToCenterRect(c.layoutNode.pos,c.layoutNode.size,e),this.goToGraphCoordinates(l,{zoom:e,animate:s})}async jumpToInstruction(t,{zoom:e=this.zoom,animate:s=!0}){let i=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-id="${t}"]`);if(!i)return;let c=i.getBoundingClientRect(),l=this.graphContainer.getBoundingClientRect(),o=(c.x-l.x)/this.zoom,n=(c.y-l.y)/this.zoom,r=c.width/this.zoom,d=c.height/this.zoom,b=this.graphPosToCenterRect({x:o,y:n},{x:r,y:d},e);i.classList.add("ig-flash"),await this.goToGraphCoordinates(b,{zoom:e,animate:s}),i.classList.remove("ig-flash")}graphPosToCenterRect(t,e,s){let i=this.viewportSize.x/s,c=this.viewportSize.y/s,l=Math.max(20/s,(i-e.x)/2),o=Math.max(20/s,(c-e.y)/2),n=t.x-l,r=t.y-o;return{x:n,y:r}}exportState(){let t={translation:this.translation,zoom:this.zoom,heatmapMode:this.heatmapMode,highlightedInstructions:this.highlightedInstructions,selectedBlockPtrs:this.selectedBlockPtrs,lastSelectedBlockPtr:this.lastSelectedBlockPtr,viewportPosOfSelectedBlock:void 0};return this.lastSelectedBlockPtr&&(t.viewportPosOfSelectedBlock=this.graph2viewport(L(this.blocksByPtr.get(this.lastSelectedBlockPtr)).layoutNode.pos)),t}restoreState(t,e){this.translation.x=t.translation.x,this.translation.y=t.translation.y,this.zoom=t.zoom,this.heatmapMode=t.heatmapMode,this.highlightedInstructions=t.highlightedInstructions,this.setSelection(Array.from(t.selectedBlockPtrs),t.lastSelectedBlockPtr),this.updatePanAndZoom(),this.updateHotness(),this.updateHighlightedInstructions(),e.preserveSelectedBlockPosition&&this.jumpToBlock(this.lastSelectedBlockPtr,{zoom:this.zoom,animate:!1,viewportPos:t.viewportPosOfSelectedBlock})}};function Ot(a){for(let t of a.dstNodes){let e=t.srcNodes.indexOf(a);k(e!==-1),t.srcNodes.splice(e,1)}}function*Q(a){for(let t of a)for(let e of t)e.block===null&&(yield e)}function*At(a){for(let t of a)for(let e of t)e.block===null&&e.dstBlock.mir.attributes.includes("backedge")&&(yield e)}function Rt(a,t,e,s,i,c,l=1){let o=T;k(t+o<=i&&i<s-o,`downward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i-o} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${e} ${i+o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,180);r.appendChild(b)}return r}function Ft(a,t,e,s,i,c,l=1){let o=T;k(s+o<=i&&i<=t-o,`upward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i+o} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${e} ${i-o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,0);r.appendChild(b)}return r}function zt(a,t,e,s,i=1){let c=T;k(t-c>=s&&a-c>=e,`to backedge: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, r = ${c} `,!0),i%2===1&&(a+=.5,s+=.5);let l="";l+=`M ${a} ${t} `,l+=`A ${c} ${c} 0 0 0 ${a-c} ${s} `,l+=`L ${e} ${s} `;let o=document.createElementNS("http://www.w3.org/2000/svg","g"),n=document.createElementNS("http://www.w3.org/2000/svg","path");n.setAttribute("d",l),n.setAttribute("fill","none"),n.setAttribute("stroke","black"),n.setAttribute("stroke-width",`${i} `),o.appendChild(n);let r=G(e,s,270);return o.appendChild(r),o}function _t(a,t,e,s,i,c=1){let l=T;k(t+l<=i&&a<=e&&s<=t,`block to backedge dummy: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${l} `,!0),c%2===1&&(a+=.5,e+=.5,i+=.5);let o="";o+=`M ${a} ${t} `,o+=`L ${a} ${i-l} `,o+=`A ${l} ${l} 0 0 0 ${a+l} ${i} `,o+=`L ${e-l} ${i} `,o+=`A ${l} ${l} 0 0 0 ${e} ${i-l} `,o+=`L ${e} ${s} `;let n=document.createElementNS("http://www.w3.org/2000/svg","g"),r=document.createElementNS("http://www.w3.org/2000/svg","path");return r.setAttribute("d",o),r.setAttribute("fill","none"),r.setAttribute("stroke","black"),r.setAttribute("stroke-width",`${c} `),n.appendChild(r),n}function jt(a,t,e,s,i=1){k(e<a&&s===t,`x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s} `,!0),i%2===1&&(t+=.5,s+=.5);let c="";c+=`M ${a} ${t} `,c+=`L ${e} ${s} `;let l=document.createElementNS("http://www.w3.org/2000/svg","g"),o=document.createElementNS("http://www.w3.org/2000/svg","path");o.setAttribute("d",c),o.setAttribute("fill","none"),o.setAttribute("stroke","black"),o.setAttribute("stroke-width",`${i} `),l.appendChild(o);let n=G(e,s,270);return l.appendChild(n),l}function G(a,t,e,s=5){let i=document.createElementNS("http://www.w3.org/2000/svg","path");return i.setAttribute("d",`M 0 0 L ${-s} ${s*1.5} L ${s} ${s*1.5} Z`),i.setAttribute("transform",`translate(${a}, ${t}) rotate(${e})`),i}function ct(a,t){a.classList.add("ig-highlight"),a.style.setProperty("--ig-highlight-color",t)}function Vt(a){a.classList.remove("ig-highlight"),a.style.setProperty("--ig-highlight-color","transparent")}var A=class{constructor(t,{func:e,pass:s=0,sampleCounts:i}){this.graph=null,this.func=e,this.passNumber=s,this.sampleCounts=i,this.keyPasses=[null,null,null,null];{let c=null;for(let[l,o]of e.passes.entries())o.mir.blocks.length>0&&(this.keyPasses[0]===null&&(this.keyPasses[0]=l),o.lir.blocks.length===0&&(this.keyPasses[1]=l)),o.lir.blocks.length>0&&(c?.lir.blocks.length===0&&(this.keyPasses[2]=l),this.keyPasses[3]=l),c=o}this.redundantPasses=[];{let c=null;for(let[l,o]of e.passes.entries()){if(c===null){c=o;continue}E(c.mir,o.mir)&&E(c.lir,o.lir)&&this.redundantPasses.push(l),c=o}}this.viewport=x("div",["ig-flex-grow-1","ig-overflow-hidden"],c=>{c.style.position="relative"}),this.sidebarLinks=e.passes.map((c,l)=>x("a",["ig-link-normal","ig-pv1","ig-ph2","ig-flex","ig-g2"],o=>{o.href="#",o.addEventListener("click",n=>{n.preventDefault(),this.switchPass(l)})},[x("div",["ig-w1","ig-tr","ig-f6","ig-text-dim"],o=>{o.style.paddingTop="0.08rem"},[`${l}`]),x("div",[this.redundantPasses.includes(l)&&"ig-text-dim"],()=>{},[c.name])])),this.container=x("div",["ig-absolute","ig-absolute-fill","ig-flex"],()=>{},[x("div",["ig-w5","ig-br","ig-flex-shrink-0","ig-overflow-y-auto","ig-bg-white"],()=>{},[...this.sidebarLinks]),this.viewport]),t.appendChild(this.container),this.keydownHandler=this.keydownHandler.bind(this),this.tweakHandler=this.tweakHandler.bind(this),window.addEventListener("keydown",this.keydownHandler),window.addEventListener("tweak",this.tweakHandler),this.update()}destroy(){this.container.remove(),window.removeEventListener("keydown",this.keydownHandler),window.removeEventListener("tweak",this.tweakHandler)}update(){for(let[s,i]of this.sidebarLinks.entries())i.classList.toggle("ig-bg-primary",this.passNumber===s);let t=this.graph?.exportState();this.viewport.innerHTML="",this.graph=null;let e=this.func.passes[this.passNumber];if(e)try{this.graph=new V(this.viewport,e,{sampleCounts:this.sampleCounts}),t&&this.graph.restoreState(t,{preserveSelectedBlockPosition:!0})}catch(s){this.viewport.innerHTML="An error occurred while laying out the graph. See console.",console.error(s)}}switchPass(t){this.passNumber=t,this.update()}keydownHandler(t){switch(t.key){case"w":case"s":this.graph?.navigate(t.key==="s"?"down":"up"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"a":case"d":this.graph?.navigate(t.key==="d"?"right":"left"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"f":for(let e=this.passNumber+1;e<this.func.passes.length;e++)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"r":for(let e=this.passNumber-1;e>=0;e--)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"1":case"2":case"3":case"4":{let e=["1","2","3","4"].indexOf(t.key),s=this.keyPasses[e];typeof s=="number"&&this.switchPass(s)}break;case"c":{let e=this.graph?.blocksByPtr.get(this.graph?.lastSelectedBlockPtr??-1);e&&this.graph?.jumpToBlock(e.ptr,{zoom:1})}break}}tweakHandler(){this.update()}};var M=new URL(window.location.toString()).searchParams,Gt=M.has("func")?parseInt(M.get("func"),10):void 0,ut=M.has("pass")?parseInt(M.get("pass"),10):void 0,Y=class{constructor(t){this.exportButton=null,this.ionjson=null,this.funcIndex=Gt??0,this.funcSelected=t.funcSelected,this.funcSelector=x("div",[],()=>{},["Function",x("input",["ig-w3"],e=>{e.type="number",e.min="1",e.addEventListener("input",()=>{this.switchFunc(parseInt(e.value,10)-1)})},[])," / ",x("span",["num-functions"])]),this.funcSelectorNone=x("div",[],()=>{},["No functions to display."]),this.funcName=x("div"),this.root=x("div",["ig-bb","ig-flex","ig-bg-white"],()=>{},[x("div",["ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center","ig-br","ig-hide-if-empty"],()=>{},[t.browse&&x("div",[],()=>{},[x("input",[],e=>{e.type="file",e.addEventListener("change",s=>{let i=s.target;i.files?.length&&this.fileSelected(i.files[0])})})]),this.funcSelector,this.funcSelectorNone]),x("div",["ig-flex-grow-1","ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center"],()=>{},[this.funcName,x("div",["ig-flex-grow-1"]),t.export&&x("div",[],()=>{},[x("button",[],e=>{this.exportButton=e,e.addEventListener("click",()=>{this.exportStandalone()})},["Export"])])])]),this.update()}async fileSelected(t){let e=JSON.parse(await t.text());this.ionjson=X(e),this.switchFunc(0),this.update()}switchIonJSON(t){this.ionjson=t,this.switchFunc(this.funcIndex)}switchFunc(t){t=Math.max(0,Math.min(this.numFunctions()-1,t)),this.funcIndex=isNaN(t)?0:t,this.funcSelected(this.ionjson?.functions[this.funcIndex]??null),this.update()}numFunctions(){return this.ionjson?.functions.length??0}update(){let t=0<=this.funcIndex&&this.funcIndex<this.numFunctions();this.funcSelector.hidden=this.numFunctions()<=1,this.funcSelectorNone.hidden=!(this.ionjson&&this.numFunctions()===0);let e=this.funcSelector.querySelector("input");e.max=`${this.numFunctions()}`,e.value=`${this.funcIndex+1}`,this.funcSelector.querySelector(".num-functions").innerHTML=`${this.numFunctions()}`,this.funcName.hidden=!t,this.funcName.innerText=`${this.ionjson?.functions[this.funcIndex].name??""}`,this.exportButton&&(this.exportButton.disabled=!this.ionjson||!t)}async exportStandalone(){let t=L(this.ionjson),e=t.functions[this.funcIndex].name,s={version:1,functions:[t.functions[this.funcIndex]]},c=(await(await fetch("./standalone.html")).text()).replace(/\{\{\s*IONJSON\s*\}\}/,JSON.stringify(s)),l=URL.createObjectURL(new Blob([c],{type:"text/html;charset=utf-8"})),o=document.createElement("a");o.href=l,o.download=`iongraph-${e}.html`,document.body.appendChild(o),o.click(),o.remove(),URL.revokeObjectURL(l)}},tt=class{constructor(){this.menuBar=new Y({browse:!0,export:!0,funcSelected:t=>this.switchFunc(t)}),this.func=null,this.sampleCountsFromFile=void 0,this.graph=null,this.loadStuffFromQueryParams(),this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer]),this.update()}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut,sampleCounts:this.sampleCountsFromFile}))}loadStuffFromQueryParams(){(async()=>{let t=M.get("file");if(t){let s=await(await fetch(t)).json(),i=X(s);this.menuBar.switchIonJSON(i)}})(),(async()=>{let t=M.get("sampleCounts");if(t){let s=await(await fetch(t)).json();this.sampleCountsFromFile={selfLineHits:new Map(s.selfLineHits),totalLineHits:new Map(s.totalLineHits)},this.update()}})()}switchFunc(t){this.func=t,this.update()}},et=class{constructor(){this.menuBar=new Y({funcSelected:t=>this.switchFunc(t)}),this.func=null,this.graph=null,this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer])}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut}))}setIonJSON(t){this.menuBar.switchIonJSON(t)}switchFunc(t){this.func=t,this.update()}};return yt(Yt);})(); +</script> + <script>window.__exportedIonJSON = {{ IONJSON }}</script> + <script> + const ui = new iongraph.StandaloneUI(); + document.body.appendChild(ui.root); + ui.setIonJSON(window.__exportedIonJSON); + </script> +</body> diff --git a/tool/zjit_iongraph.rb b/tool/zjit_iongraph.rb new file mode 100755 index 0000000000..0cb7701614 --- /dev/null +++ b/tool/zjit_iongraph.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +require 'json' +require 'logger' + +LOGGER = Logger.new($stderr) + +def run_ruby *cmd + # Find the first --zjit* option and add --zjit-dump-hir-iongraph after it + zjit_index = cmd.find_index { |arg| arg.start_with?("--zjit") } + raise "No --zjit option found in command" unless zjit_index + cmd.insert(zjit_index + 1, "--zjit-dump-hir-iongraph") + pid = Process.spawn(*cmd) + _, status = Process.wait2(pid) + if status.exitstatus != 0 + LOGGER.warn("Command failed with exit status #{status.exitstatus}") + end + pid +end + +usage = "Usage: zjit_iongraph.rb <path_to_ruby> <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +pid = run_ruby(RUBY, *OPTIONS) +functions = Dir["/tmp/zjit-iongraph-#{pid}/fun*.json"].map do |path| + JSON.parse(File.read(path)) +end + +if functions.empty? + LOGGER.warn("No iongraph functions found for PID #{pid}") +end + +json = JSON.dump({version: 1, functions: functions}) +# Get zjit_iongraph.html from the sibling file next to this script +html = File.read(File.join(File.dirname(__FILE__), "zjit_iongraph.html")) +html.sub!("{{ IONJSON }}", json) +output_path = "zjit_iongraph_#{pid}.html" +File.write(output_path, html) +puts "Wrote iongraph to #{output_path}" |