diff options
Diffstat (limited to 'tool')
356 files changed, 24530 insertions, 15296 deletions
diff --git a/tool/annocheck/Dockerfile b/tool/annocheck/Dockerfile new file mode 100644 index 0000000000..d1fb1839c9 --- /dev/null +++ b/tool/annocheck/Dockerfile @@ -0,0 +1,4 @@ +FROM ghcr.io/ruby/fedora:latest + +RUN dnf -y install annobin-annocheck +WORKDIR /work diff --git a/tool/annocheck/Dockerfile-copy b/tool/annocheck/Dockerfile-copy new file mode 100644 index 0000000000..d437f27387 --- /dev/null +++ b/tool/annocheck/Dockerfile-copy @@ -0,0 +1,6 @@ +FROM ghcr.io/ruby/fedora:latest +ARG IN_DIR + +RUN dnf -y install annobin-annocheck +COPY ${IN_DIR} /work +WORKDIR /work diff --git a/tool/auto-style.rb b/tool/auto-style.rb new file mode 100755 index 0000000000..3b93c8c317 --- /dev/null +++ b/tool/auto-style.rb @@ -0,0 +1,284 @@ +#!/usr/bin/env ruby +# Usage: +# auto-style.rb oldrev newrev [pushref] + +require 'shellwords' +require 'tmpdir' +ENV['LC_ALL'] = 'C' + +class Git + attr_reader :depth + + def initialize(oldrev, newrev, branch = nil) + @oldrev = oldrev + @newrev = !newrev || newrev.empty? ? 'HEAD' : newrev + @branch = branch + + return unless oldrev + + # GitHub may not fetch github.event.pull_request.base.sha at checkout + git('log', '--format=%H', '-1', @oldrev, out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=1', 'origin', @oldrev) + git('log', '--format=%H', '-1', "#@newrev~99", out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=100', 'origin', @newrev) + + with_clean_env do + @revs = {} + IO.popen(['git', 'log', '--format=%H %s', "#{@oldrev}..#{@newrev}"]) do |f| + f.each do |line| + line.chomp! + rev, subj = line.split(' ', 2) + @revs[rev] = subj + end + end + @depth = @revs.size + end + end + + # ["foo/bar.c", "baz.h", ...] + def updated_paths + with_clean_env do + IO.popen(['git', 'diff', '--name-only', @oldrev, @newrev], &:readlines).each(&:chomp!) + end + end + + # [0, 1, 4, ...] + def updated_lines(file) # NOTE: This doesn't work well on pull requests, so not used anymore + lines = [] + revs = @revs.map {|rev, subj| rev unless subj.start_with?("Revert ")}.compact + revs_pattern = /\A(?:#{revs.join('|')}) / + with_clean_env { IO.popen(['git', 'blame', '-l', '--', file], &:readlines) }.each_with_index do |line, index| + if revs_pattern =~ line + lines << index + end + end + lines + end + + def commit(log, *files) + git('add', *files) + git('commit', '-m', log) + end + + def push + git('push', 'origin', @branch) if @branch + end + + def diff + git('--no-pager', 'diff') + end + + private + + def git(*args, **opts) + cmd = ['git', *args] + puts "+ #{cmd.shelljoin}" + ret = with_clean_env { system(*cmd, **opts) } + unless ret or opts[:err] + abort "Failed to run: #{cmd}" + end + ret + end + + def with_clean_env + git_dir = ENV.delete('GIT_DIR') # this overcomes '-C' or pwd + yield + ensure + ENV['GIT_DIR'] = git_dir if git_dir + end +end + +DEFAULT_GEM_LIBS = %w[ + bundler + cmath + csv + e2mmap + fileutils + forwardable + ipaddr + irb + logger + matrix + mutex_m + ostruct + prime + rdoc + rexml + rss + scanf + shell + sync + thwait + tracer + webrick +] + +DEFAULT_GEM_EXTS = %w[ + bigdecimal + date + dbm + digest + etc + fcntl + fiddle + gdbm + io/console + io/nonblock + json + openssl + psych + racc + sdbm + stringio + strscan + zlib +] + +IGNORED_FILES = [ + # default gems whose master is GitHub + %r{\Abin/(?!erb)\w+\z}, + *(DEFAULT_GEM_LIBS + DEFAULT_GEM_EXTS).flat_map { |lib| + [ + %r{\Alib/#{lib}/}, + %r{\Alib/#{lib}\.gemspec\z}, + %r{\Alib/#{lib}\.rb\z}, + %r{\Atest/#{lib}/}, + ] + }, + *DEFAULT_GEM_EXTS.flat_map { |ext| + [ + %r{\Aext/#{ext}/}, + %r{\Atest/#{ext}/}, + ] + }, + + # vendoring (ccan) + %r{\Accan/}, + + # vendoring (io/) + %r{\Aext/io/}, + + # vendoring (nkf) + %r{\Aext/nkf/nkf-utf8/}, + + # vendoring (onigmo) + %r{\Aenc/}, + %r{\Ainclude/ruby/onigmo\.h\z}, + %r{\Areg.+\.(c|h)\z}, + + # explicit or implicit `c-file-style: "linux"` + %r{\Aaddr2line\.c\z}, + %r{\Amissing/}, + %r{\Astrftime\.c\z}, + %r{\Avsnprintf\.c\z}, + + # to respect the original statements of licenses + %r{\ALEGAL\z}, + + # trailing spaces could be intentional in TRICK code + %r{\Asample/trick[^/]*/}, +] + +DIFFERENT_STYLE_FILES = %w[ + addr2line.c io_buffer.c prism*.c scheduler.c +] + +def adjust_styles(files) + trailing = eofnewline = expandtab = indent = false + + edited_files = files.select do |f| + src = File.binread(f) rescue next + eofnewline = eofnewline0 = true if src.sub!(/(?<!\A|\n)\z/, "\n") + + trailing0 = false + expandtab0 = false + indent0 = false + + src.gsub!(/^.*$/).with_index do |line, lineno| + trailing = trailing0 = true if line.sub!(/[ \t]+$/, '') + line + end + + if f.end_with?('.c') || f.end_with?('.h') || f == 'insns.def' + # If and only if unedited lines did not have tab indentation, prevent introducing tab indentation to the file. + expandtab_allowed = src.each_line.with_index.all? do |line, lineno| + !line.start_with?("\t") + end + + if expandtab_allowed + src.gsub!(/^.*$/).with_index do |line, lineno| + if line.start_with?("\t") # last-committed line with hard tabs + expandtab = expandtab0 = true + line.sub(/\A\t+/) { |tabs| ' ' * (8 * tabs.length) } + else + line + end + end + end + end + + if File.fnmatch?("*.[chy]", f, File::FNM_PATHNAME) && + !DIFFERENT_STYLE_FILES.any? {|pat| File.fnmatch?(pat, f, File::FNM_PATHNAME)} + indent0 = true if src.gsub!(/^\w+\([^\n]*?\)\K[ \t]*(?=\{( *\\)?$)/, '\1' "\n") + indent0 = true if src.gsub!(/^([ \t]*)\}\K[ \t]*(?=else\b.*?( *\\)?$)/, '\2' "\n" '\1') + indent0 = true if src.gsub!(/^[ \t]*\}\n\K\n+(?=[ \t]*else\b)/, '') + indent ||= indent0 + end + + if trailing0 or eofnewline0 or expandtab0 or indent0 + File.binwrite(f, src) + true + end + end + if edited_files.empty? + return + else + msg = [('remove trailing spaces' if trailing), + ('append newline at EOF' if eofnewline), + ('expand tabs' if expandtab), + ('adjust indents' if indent), + ].compact + message = "* #{msg.join(', ')}. [ci skip]" + if expandtab + message += "\nPlease consider using misc/expand_tabs.rb as a pre-commit hook." + end + return message, edited_files + end +end + +oldrev, newrev, pushref = ARGV +if (dry_run = oldrev == '-n') or oldrev == '--' + _, *updated_files = ARGV + git = Git.new(nil, nil) +else + unless dry_run = pushref.nil? + branch = IO.popen(['git', 'rev-parse', '--symbolic', '--abbrev-ref', pushref], &:read).strip + end + git = Git.new(oldrev, newrev, branch) + + updated_files = git.updated_paths +end + +files = updated_files.select {|l| + /^\d/ !~ l and /\.bat\z/ !~ l and + (/\A(?:config|[Mm]akefile|GNUmakefile|README)/ =~ File.basename(l) or + /\A\z|\.(?:[chsy]|\d+|e?rb|tmpl|bas[eh]|z?sh|in|ma?k|def|src|trans|rdoc|ja|en|el|sed|awk|p[ly]|scm|mspec|html|rs)\z/ =~ File.extname(l)) +} +files.select! {|n| File.file?(n) } +files.reject! do |f| + IGNORED_FILES.any? { |re| f.match(re) } +end + +if files.empty? + puts "No files are an auto-style target:\n#{updated_files.join("\n")}" +elsif !(message, edited_files = adjust_styles(files)) + puts "All edited lines are formatted well:\n#{files.join("\n")}" +else + if dry_run + git.diff + abort message + else + git.commit(message, *edited_files) + git.push + end +end diff --git a/tool/auto_review_pr.rb b/tool/auto_review_pr.rb new file mode 100755 index 0000000000..07c98c7e0a --- /dev/null +++ b/tool/auto_review_pr.rb @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'uri' +require_relative './sync_default_gems' + +class GitHubAPIClient + def initialize(token) + @token = token + end + + def get(path) + response = Net::HTTP.get_response(URI("https://api.github.com#{path}"), { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end + + def post(path, body = {}) + body = JSON.dump(body) + response = Net::HTTP.post(URI("https://api.github.com#{path}"), body, { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + 'Content-Type' => 'application/json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end +end + +class AutoReviewPR + REPO = 'ruby/ruby' + + COMMENT_USER = 'github-actions[bot]' + COMMENT_PREFIX = 'The following files are maintained in the following upstream repositories:' + COMMENT_SUFFIX = 'Please file a pull request to the above instead. Thank you!' + + def initialize(client) + @client = client + end + + def review(pr_number) + # Fetch the list of files changed by the PR + changed_files = @client.get("/repos/#{REPO}/pulls/#{pr_number}/files").map { it.fetch(:filename) } + + # Build a Hash: { upstream_repo => files, ... } + upstream_repos = SyncDefaultGems::Repository.group(changed_files) + upstream_repos.delete(nil) # exclude no-upstream files + upstream_repos.delete('prism') if changed_files.include?('prism_compile.c') # allow prism changes in this case + if upstream_repos.empty? + puts "Skipped: The PR ##{pr_number} doesn't have upstream repositories." + return + end + + # Check if the PR is already reviewed + existing_comments = @client.get("/repos/#{REPO}/issues/#{pr_number}/comments") + existing_comments.map! { [it.fetch(:user).fetch(:login), it.fetch(:body)] } + if existing_comments.any? { |user, comment| user == COMMENT_USER && comment.start_with?(COMMENT_PREFIX) } + puts "Skipped: The PR ##{pr_number} already has an automated review comment." + return + end + + # Post a comment + comment = format_comment(upstream_repos) + result = @client.post("/repos/#{REPO}/issues/#{pr_number}/comments", { body: comment }) + puts "Success: #{JSON.pretty_generate(result)}" + end + + private + + # upstream_repos: { upstream_repo => files, ... } + def format_comment(upstream_repos) + comment = +'' + comment << "#{COMMENT_PREFIX}\n\n" + + upstream_repos.each do |upstream_repo, files| + comment << "* https://github.com/ruby/#{upstream_repo}\n" + files.each do |file| + comment << " * #{file}\n" + end + end + + comment << "\n#{COMMENT_SUFFIX}" + comment + end +end + +pr_number = ARGV[0] || abort("Usage: #{$0} <pr_number>") +client = GitHubAPIClient.new(ENV.fetch('GITHUB_TOKEN')) + +AutoReviewPR.new(client).review(pr_number) diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb new file mode 100644 index 0000000000..91ac5628f1 --- /dev/null +++ b/tool/bundler/dev_gems.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "test-unit", "~> 3.0" +gem "test-unit-ruby-core" +gem "rake", "~> 13.1" +gem "rb_sys" + +gem "turbo_tests", "~> 2.2.3" +gem "parallel_tests", "~> 4.10.1" +gem "parallel", "~> 1.19" +gem "rspec-core", "~> 3.12" +gem "rspec-expectations", "~> 3.12" +gem "rspec-mocks", "~> 3.12" +gem "rubygems-generate_index", "~> 1.1" + +group :doc do + gem "ronn-ng", "~> 0.10.1", platform: :ruby +end diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock new file mode 100644 index 0000000000..832127fb4c --- /dev/null +++ b/tool/bundler/dev_gems.rb.lock @@ -0,0 +1,132 @@ +GEM + remote: https://rubygems.org/ + specs: + compact_index (0.15.0) + diff-lcs (1.6.2) + kramdown (2.5.1) + rexml (>= 3.3.9) + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + mini_portile2 (2.8.9) + mustache (1.1.1) + nokogiri (1.19.0) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.19.0-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-java) + racc (~> 1.4) + nokogiri (1.19.0-x64-mingw-ucrt) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-gnu) + racc (~> 1.4) + parallel (1.27.0) + parallel_tests (4.10.1) + parallel + power_assert (3.0.1) + racc (1.8.1) + racc (1.8.1-java) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rexml (3.4.4) + ronn-ng (0.10.1) + kramdown (~> 2, >= 2.1) + kramdown-parser-gfm (~> 1, >= 1.0.1) + mustache (~> 1) + nokogiri (~> 1, >= 1.14.3) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + test-unit (3.7.7) + power_assert + test-unit-ruby-core (1.0.14) + test-unit (>= 3.7.2) + turbo_tests (2.2.5) + parallel_tests (>= 3.3.0, < 5) + rspec (>= 3.10) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm-linux + arm64-darwin + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86-linux + x86_64-darwin + x86_64-linux + +DEPENDENCIES + parallel (~> 1.19) + parallel_tests (~> 4.10.1) + rake (~> 13.1) + rb_sys + ronn-ng (~> 0.10.1) + rspec-core (~> 3.12) + rspec-expectations (~> 3.12) + rspec-mocks (~> 3.12) + rubygems-generate_index (~> 1.1) + test-unit (~> 3.0) + test-unit-ruby-core + turbo_tests (~> 2.2.3) + +CHECKSUMS + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + kramdown (2.5.1) sha256=87bbb6abd9d3cebe4fc1f33e367c392b4500e6f8fa19dd61c0972cf4afe7368c + kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 + mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289 + mustache (1.1.1) sha256=90891fdd50b53919ca334c8c1031eada1215e78d226d5795e523d6123a2717d0 + nokogiri (1.19.0) sha256=e304d21865f62518e04f2bf59f93bd3a97ca7b07e7f03952946d8e1c05f45695 + nokogiri (1.19.0-aarch64-linux-gnu) sha256=11a97ecc3c0e7e5edcf395720b10860ef493b768f6aa80c539573530bc933767 + nokogiri (1.19.0-arm-linux-gnu) sha256=572a259026b2c8b7c161fdb6469fa2d0edd2b61cd599db4bbda93289abefbfe5 + nokogiri (1.19.0-arm64-darwin) sha256=0811dfd936d5f6dd3f6d32ef790568bf29b2b7bead9ba68866847b33c9cf5810 + nokogiri (1.19.0-java) sha256=5f3a70e252be641d8a4099f7fb4cc25c81c632cb594eec9b4b8f2ca8be4374f3 + nokogiri (1.19.0-x64-mingw-ucrt) sha256=05d7ed2d95731edc9bef2811522dc396df3e476ef0d9c76793a9fca81cab056b + nokogiri (1.19.0-x86_64-darwin) sha256=1dad56220b603a8edb9750cd95798bffa2b8dd9dd9aa47f664009ee5b43e3067 + nokogiri (1.19.0-x86_64-linux-gnu) sha256=f482b95c713d60031d48c44ce14562f8d2ce31e3a9e8dd0ccb131e9e5a68b58c + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parallel_tests (4.10.1) sha256=df05458c691462b210f7a41fc2651d4e4e8a881e8190e6d1e122c92c07735d70 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + ronn-ng (0.10.1) sha256=4eeb0185c0fbfa889efed923b5b50e949cd869e7d82ac74138acd0c9c7165ec0 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + test-unit-ruby-core (1.0.14) sha256=d2e997796c9c5c5e8e31ac014f83a473ff5c2523a67cfa491b08893e12d43d22 + turbo_tests (2.2.5) sha256=3fa31497d12976d11ccc298add29107b92bda94a90d8a0a5783f06f05102509f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb index 84cb226330..a9b6fda11b 100644 --- a/tool/bundler/rubocop_gems.rb +++ b/tool/bundler/rubocop_gems.rb @@ -2,10 +2,12 @@ source "https://rubygems.org" -gem "rubocop", "~> 1.7" +gem "rubocop", ">= 1.52.1", "< 2" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" gem "test-unit" +gem "rb_sys" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock index ab8c259659..d0c3120e11 100644 --- a/tool/bundler/rubocop_gems.rb.lock +++ b/tool/bundler/rubocop_gems.rb.lock @@ -1,60 +1,159 @@ GEM remote: https://rubygems.org/ specs: - ast (2.4.2) - diff-lcs (1.4.4) - minitest (5.14.4) - parallel (1.19.2) - parser (3.0.1.0) + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) ast (~> 2.4.1) - power_assert (2.0.0) - rainbow (3.0.0) - rake (13.0.3) - rake-compiler (1.1.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) rake - regexp_parser (2.1.1) - rexml (3.2.5) - rspec (3.10.0) - rspec-core (~> 3.10.0) - rspec-expectations (~> 3.10.0) - rspec-mocks (~> 3.10.0) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-mocks (3.10.2) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-support (3.10.2) - rubocop (1.12.1) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.82.1) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) - parser (>= 3.0.0.0) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.2.0, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.48.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.4.1) - parser (>= 2.7.1.5) - ruby-progressbar (1.11.0) - test-unit (3.4.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + stringio (3.2.0) + test-unit (3.7.7) power_assert - unicode-display_width (2.0.0) + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) PLATFORMS - arm64-darwin-20 - universal-java-11 + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin x86_64-linux DEPENDENCIES - minitest + irb + minitest (~> 5.1) rake rake-compiler + rb_sys rspec - rubocop (~> 1.7) + rubocop (>= 1.52.1, < 2) test-unit +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.82.1) sha256=09f1a6a654a960eda767aebea33e47603080f8e9c9a3f019bf9b94c9cab5e273 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + BUNDLED WITH - 2.3.0.dev + 4.1.0.dev diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb index 1cd189742d..f7bc34cf5e 100644 --- a/tool/bundler/standard_gems.rb +++ b/tool/bundler/standard_gems.rb @@ -4,8 +4,10 @@ source "https://rubygems.org" gem "standard", "~> 1.0" -gem "minitest" +gem "minitest", "~> 5.1" +gem "irb" gem "rake" gem "rake-compiler" gem "rspec" gem "test-unit" +gem "rb_sys" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock index c4e3ee685e..f3792f8611 100644 --- a/tool/bundler/standard_gems.rb.lock +++ b/tool/bundler/standard_gems.rb.lock @@ -1,66 +1,179 @@ GEM remote: https://rubygems.org/ specs: - ast (2.4.2) - diff-lcs (1.4.4) - minitest (5.14.4) - parallel (1.19.2) - parser (3.0.1.0) + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) ast (~> 2.4.1) - power_assert (2.0.0) - rainbow (3.0.0) - rake (13.0.3) - rake-compiler (1.1.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) rake - regexp_parser (2.1.1) - rexml (3.2.5) - rspec (3.10.0) - rspec-core (~> 3.10.0) - rspec-expectations (~> 3.10.0) - rspec-mocks (~> 3.10.0) - rspec-core (3.10.1) - rspec-support (~> 3.10.0) - rspec-expectations (3.10.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-mocks (3.10.2) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.10.0) - rspec-support (3.10.2) - rubocop (1.11.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.81.7) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) - parser (>= 3.0.0.0) + parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml - rubocop-ast (>= 1.2.0, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.47.1, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.4.1) - parser (>= 2.7.1.5) - rubocop-performance (1.10.1) - rubocop (>= 0.90.0, < 2.0) - rubocop-ast (>= 0.4.0) - ruby-progressbar (1.11.0) - standard (1.0.4) - rubocop (= 1.11.0) - rubocop-performance (= 1.10.1) - test-unit (3.4.0) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + rubocop-performance (1.26.1) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (1.13.0) + standard (1.52.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.81.7) + standard-custom (~> 1.0.0) + standard-performance (~> 1.8) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.9.0) + lint_roller (~> 1.1) + rubocop-performance (~> 1.26.0) + stringio (3.2.0) + test-unit (3.7.7) power_assert - unicode-display_width (2.0.0) + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) PLATFORMS - arm64-darwin-20 - universal-java-11 + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin x86_64-linux DEPENDENCIES - minitest + irb + minitest (~> 5.1) rake rake-compiler + rb_sys rspec standard (~> 1.0) test-unit +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.81.7) sha256=6fb5cc298c731691e2a414fe0041a13eb1beed7bab23aec131da1bcc527af094 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834 + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + standard (1.52.0) sha256=ec050e63228e31fabe40da3ef96da7edda476f7acdf3e7c2ad47b6e153f6a076 + standard-custom (1.0.2) sha256=424adc84179a074f1a2a309bb9cf7cd6bfdb2b6541f20c6bf9436c0ba22a652b + standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + BUNDLED WITH - 2.3.0.dev + 4.1.0.dev diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb index 215d23183e..ddc19e2939 100644 --- a/tool/bundler/test_gems.rb +++ b/tool/bundler/test_gems.rb @@ -2,11 +2,17 @@ source "https://rubygems.org" -gem "rack", "2.0.8" -gem "webrick", "1.7.0" -gem "rack-test", "~> 1.1" -gem "artifice", "~> 0.6.0" -gem "compact_index", "~> 0.13.0" -gem "sinatra", "~> 2.0" -gem "rake", "13.0.1" +gem "rack", "~> 3.1" +gem "rack-test", "~> 2.1" +gem "compact_index", "~> 0.15.0" +gem "sinatra", "~> 4.1" +gem "rake", "~> 13.1" gem "builder", "~> 3.2" +gem "rb_sys" +gem "fiddle" +gem "rubygems-generate_index", "~> 1.1" +gem "concurrent-ruby" +gem "psych" +gem "etc", platforms: [:ruby, :windows] +gem "open3" +gem "shellwords" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock index f8fa01b59c..fdffc1f09d 100644 --- a/tool/bundler/test_gems.rb.lock +++ b/tool/bundler/test_gems.rb.lock @@ -1,43 +1,106 @@ GEM remote: https://rubygems.org/ specs: - artifice (0.6) - rack-test - builder (3.2.4) - compact_index (0.13.0) - mustermann (1.1.1) + base64 (0.3.0) + builder (3.3.0) + compact_index (0.15.0) + concurrent-ruby (1.3.6) + date (3.5.1) + date (3.5.1-java) + etc (1.4.6) + fiddle (1.1.8) + jar-dependencies (0.5.5) + logger (1.7.0) + mustermann (3.0.4) ruby2_keywords (~> 0.0.1) - rack (2.0.8) - rack-protection (2.0.8.1) - rack - rack-test (1.1.0) - rack (>= 1.0, < 3) - rake (13.0.1) - ruby2_keywords (0.0.2) - sinatra (2.0.8.1) - mustermann (~> 1.0) - rack (~> 2.0) - rack-protection (= 2.0.8.1) + open3 (0.2.1) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + rack (3.2.4) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + ruby2_keywords (0.0.5) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + shellwords (0.2.2) + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) tilt (~> 2.0) - tilt (2.0.10) - webrick (1.7.0) + stringio (3.2.0) + tilt (2.6.1) PLATFORMS java ruby - universal-java-11 - x64-mingw32 + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin x86_64-linux DEPENDENCIES - artifice (~> 0.6.0) builder (~> 3.2) - compact_index (~> 0.13.0) - rack (= 2.0.8) - rack-test (~> 1.1) - rake (= 13.0.1) - sinatra (~> 2.0) - webrick (= 1.7.0) + compact_index (~> 0.15.0) + concurrent-ruby + etc + fiddle + open3 + psych + rack (~> 3.1) + rack-test (~> 2.1) + rake (~> 13.1) + rb_sys + rubygems-generate_index (~> 1.1) + shellwords + sinatra (~> 4.1) + +CHECKSUMS + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + etc (1.4.6) sha256=0f7e9e7842ea5e3c3bd9bc81746ebb8c65ea29e4c42a93520a0d638129c7de01 + fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mustermann (3.0.4) sha256=85fadcb6b3c6493a8b511b42426f904b7f27b282835502233dd154daab13aa22 + open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + rack (3.2.4) sha256=5d74b6f75082a643f43c1e76b419c40f0e5527fcfee1e669ac1e6b73c0ccb6f6 + rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac + rack-session (2.1.1) sha256=0b6dc07dea7e4b583f58a48e8b806d4c9f1c6c9214ebc202ec94562cbea2e4e9 + rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94 + sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + tilt (2.6.1) sha256=35a99bba2adf7c1e362f5b48f9b581cce4edfba98117e34696dde6d308d84770 BUNDLED WITH - 2.3.0.dev + 4.1.0.dev diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb new file mode 100644 index 0000000000..8d12c5adde --- /dev/null +++ b/tool/bundler/vendor_gems.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "fileutils", "1.8.0" +gem "molinillo", github: "cocoapods/molinillo", ref: "1d62d7d5f448e79418716dc779a4909509ccda2a" +gem "net-http", "0.7.0" # net-http-0.8.0 is broken with JRuby +gem "net-http-persistent", "4.0.6" +gem "net-protocol", "0.2.2" +gem "optparse", "0.8.0" +gem "pub_grub", github: "jhawthorn/pub_grub", ref: "df6add45d1b4d122daff2f959c9bd1ca93d14261" +gem "resolv", "0.6.2" +gem "securerandom", "0.4.1" +gem "timeout", "0.4.4" +gem "thor", "1.4.0" +gem "tsort", "0.2.0" +gem "uri", "1.1.1" diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock new file mode 100644 index 0000000000..cc7886e60b --- /dev/null +++ b/tool/bundler/vendor_gems.rb.lock @@ -0,0 +1,75 @@ +GIT + remote: https://github.com/cocoapods/molinillo.git + revision: 1d62d7d5f448e79418716dc779a4909509ccda2a + ref: 1d62d7d5f448e79418716dc779a4909509ccda2a + specs: + molinillo (0.8.0) + +GIT + remote: https://github.com/jhawthorn/pub_grub.git + revision: df6add45d1b4d122daff2f959c9bd1ca93d14261 + ref: df6add45d1b4d122daff2f959c9bd1ca93d14261 + specs: + pub_grub (0.5.0) + +GEM + remote: https://rubygems.org/ + specs: + connection_pool (2.5.4) + fileutils (1.8.0) + net-http (0.7.0) + uri + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + net-protocol (0.2.2) + timeout + optparse (0.8.0) + resolv (0.6.2) + securerandom (0.4.1) + thor (1.4.0) + timeout (0.4.4) + tsort (0.2.0) + uri (1.1.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + fileutils (= 1.8.0) + molinillo! + net-http (= 0.7.0) + net-http-persistent (= 4.0.6) + net-protocol (= 0.2.2) + optparse (= 0.8.0) + pub_grub! + resolv (= 0.6.2) + securerandom (= 0.4.1) + thor (= 1.4.0) + timeout (= 0.4.4) + tsort (= 0.2.0) + uri (= 1.1.1) + +CHECKSUMS + connection_pool (2.5.4) sha256=e9e1922327416091f3f6542f5f4446c2a20745276b9aa796dd0bb2fd0ea1e70a + fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02 + molinillo (0.8.0) + net-http (0.7.0) sha256=4db7d9f558f8ffd4dcf832d0aefd02320c569c7d4f857def49e585069673a425 + net-http-persistent (4.0.6) sha256=2abb3a04438edf6cb9e0e7e505969605f709eda3e3c5211beadd621a2c84dd5d + net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 + optparse (0.8.0) sha256=ef6b7fbaf7ec331474f325bc08dd5622e6e1e651007a5341330ee4b08ce734f0 + pub_grub (0.5.0) + resolv (0.6.2) sha256=61efe545cedddeb1b14f77e51f85c85ca66af5098fdbf567fadf32c34590fb14 + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + thor (1.4.0) sha256=8763e822ccb0f1d7bee88cde131b19a65606657b847cc7b7b4b82e772bcd8a3d + timeout (0.4.4) sha256=f0f6f970104b82427cd990680f539b6bbb8b1e55efa913a55c6492935e4e0edb + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + +BUNDLED WITH + 4.0.0.dev diff --git a/tool/checksum.rb b/tool/checksum.rb index bcc60ee14a..8f2d1d97d0 100755 --- a/tool/checksum.rb +++ b/tool/checksum.rb @@ -36,9 +36,7 @@ class Checksum end def update! - open(@checksum, "wb") {|f| - f.puts("src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") - } + File.binwrite(@checksum, "src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") end def update diff --git a/tool/ci_functions.sh b/tool/ci_functions.sh deleted file mode 100644 index 7066bbe4ec..0000000000 --- a/tool/ci_functions.sh +++ /dev/null @@ -1,29 +0,0 @@ -# -*- BASH -*- -# Manage functions used on a CI. -# Run `. tool/ci_functions.sh` to use it. - -# Create options with patterns `-n !/name1/ -n !/name2/ ..` to exclude the test -# method names by the method names `name1 name2 ..`. -# See `ruby tool/test/runner.rb --help` `-n` option. -function ci_to_excluded_test_opts { - local tests_str="${1}" - # Use the backward matching `!/name$/`, as the perfect matching doesn't work. - # https://bugs.ruby-lang.org/issues/16936 - ruby <<EOF - opts = "${tests_str}".split.map { |test| "-n \!/#{test}\$$/" } - puts opts.join(' ') -EOF - return 0 -} - -# Create options with patterns `-n name1 -n name2 ..` to include the test -# method names by the method names `name1 name2 ..`. -# See `ruby tool/test/runner.rb --help` `-n` option. -function ci_to_included_test_opts { - local tests_str="${1}" - ruby <<EOF - opts = "${tests_str}".split.map { |test| "-n #{test}" } - puts opts.join(' ') -EOF - return 0 -} diff --git a/tool/commit-email.rb b/tool/commit-email.rb new file mode 100755 index 0000000000..c887f8783e --- /dev/null +++ b/tool/commit-email.rb @@ -0,0 +1,372 @@ +#!/usr/bin/env ruby + +require "optparse" +require "nkf" +require "shellwords" + +CommitEmailInfo = Struct.new( + :author, + :author_email, + :revision, + :entire_sha256, + :date, + :log, + :branch, + :diffs, + :added_files, :deleted_files, :updated_files, + :added_dirs, :deleted_dirs, :updated_dirs, +) + +class GitInfoBuilder + GitCommandFailure = Class.new(RuntimeError) + + def initialize(repo_path) + @repo_path = repo_path + end + + def build(oldrev, newrev, refname) + diffs = build_diffs(oldrev, newrev) + + info = CommitEmailInfo.new + info.author = git_show(newrev, format: '%an') + info.author_email = normalize_email(git_show(newrev, format: '%aE')) + info.revision = newrev[0...10] + info.entire_sha256 = newrev + info.date = Time.at(Integer(git_show(newrev, format: '%at'))) + info.log = git_show(newrev, format: '%B') + info.branch = git('rev-parse', '--symbolic', '--abbrev-ref', refname).strip + info.diffs = diffs + info.added_files = find_files(diffs, status: :added) + info.deleted_files = find_files(diffs, status: :deleted) + info.updated_files = find_files(diffs, status: :modified) + info.added_dirs = [] # git does not deal with directory + info.deleted_dirs = [] # git does not deal with directory + info.updated_dirs = [] # git does not deal with directory + info + end + + private + + # Force git-svn email address to @ruby-lang.org to avoid email bounce by invalid email address. + def normalize_email(email) + if email.match(/\A[^@]+@\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/) # git-svn + svn_user, _ = email.split('@', 2) + "#{svn_user}@ruby-lang.org" + else + email + end + end + + def find_files(diffs, status:) + files = [] + diffs.each do |path, values| + if values.keys.first == status + files << path + end + end + files + end + + # SVN version: + # { + # "filename" => { + # "[modified|added|deleted|copied|property_changed]" => { + # type: "[modified|added|deleted|copied|property_changed]", + # body: "diff body", # not implemented because not used + # added: Integer, + # deleted: Integer, + # } + # } + # } + def build_diffs(oldrev, newrev) + diffs = {} + + numstats = git('diff', '--numstat', oldrev, newrev).lines.map { |l| l.strip.split("\t", 3) } + git('diff', '--name-status', oldrev, newrev).each_line do |line| + status, path, _newpath = line.strip.split("\t", 3) + diff = build_diff(path, numstats) + + case status + when 'A' + diffs[path] = { added: { type: :added, **diff } } + when 'M' + diffs[path] = { modified: { type: :modified, **diff } } + when 'C' + diffs[path] = { copied: { type: :copied, **diff } } + when 'D' + diffs[path] = { deleted: { type: :deleted, **diff } } + when /\AR/ # R100 (which does not exist in git.ruby-lang.org's git 2.1.4) + # TODO: implement something + else + $stderr.puts "unexpected git diff status: #{status}" + end + end + + diffs + end + + def build_diff(path, numstats) + diff = { added: 0, deleted: 0 } # :body not implemented because not used + line = numstats.find { |(_added, _deleted, file, *)| file == path } + return diff if line.nil? + + added, deleted, _ = line + if added + diff[:added] = Integer(added) + end + if deleted + diff[:deleted] = Integer(deleted) + end + diff + end + + def git_show(revision, format:) + git('show', '--no-show-signature', "--pretty=#{format}", '--no-patch', revision).strip + end + + def git(*args) + command = ['git', '-C', @repo_path, *args] + output = with_gitenv { IO.popen(command, external_encoding: 'UTF-8', &:read) } + unless $?.success? + raise GitCommandFailure, "failed to execute '#{command.join(' ')}':\n#{output}" + end + output + end + + def with_gitenv + orig = ENV.to_h.dup + begin + ENV.delete('GIT_DIR') + yield + ensure + ENV.replace(orig) + end + end +end + +CommitEmailOptions = Struct.new(:error_to, :viewer_uri) + +CommitEmail = Module.new +class << CommitEmail + SENDMAIL = ENV.fetch('SENDMAIL', '/usr/sbin/sendmail') + private_constant :SENDMAIL + + def parse(args) + options = CommitEmailOptions.new + + opts = OptionParser.new do |opts| + opts.separator('') + + opts.on('-e', '--error-to [TO]', + 'Add [TO] to to address when error is occurred') do |to| + options.error_to = to + end + + opts.on('--viewer-uri [URI]', + 'Use [URI] as URI of revision viewer') do |uri| + options.viewer_uri = uri + end + + opts.on_tail('--help', 'Show this message') do + puts opts + exit + end + end + + return opts.parse(args), options + end + + def main(repo_path, to, rest) + args, options = parse(rest) + + infos = args.each_slice(3).flat_map do |oldrev, newrev, refname| + revisions = IO.popen(['git', 'log', '--no-show-signature', '--reverse', '--pretty=%H', "#{oldrev}^..#{newrev}"], &:read).lines.map(&:strip) + revisions[0..-2].zip(revisions[1..-1]).map do |old, new| + GitInfoBuilder.new(repo_path).build(old, new, refname) + end + end + + infos.each do |info| + next if info.branch.start_with?('notes/') + puts "#{info.branch}: #{info.revision} (#{info.author})" + + from = make_from(name: info.author, email: "noreply@ruby-lang.org") + sendmail(to, from, make_mail(to, from, info, viewer_uri: options.viewer_uri)) + end + end + + def sendmail(to, from, mail) + IO.popen([*SENDMAIL.shellsplit, to], 'w') do |f| + f.print(mail) + end + unless $?.success? + raise "Failed to run `#{SENDMAIL} #{to}` with: '#{mail}'" + end + end + + private + + def b_encode(str) + NKF.nkf('-WwM', str) + end + + def make_body(info, viewer_uri:) + body = +'' + body << "#{info.author}\t#{format_time(info.date)}\n" + body << "\n" + body << " New Revision: #{info.revision}\n" + body << "\n" + body << " #{viewer_uri}#{info.revision}\n" + body << "\n" + body << " Log:\n" + body << info.log.lstrip.gsub(/^\t*/, ' ').rstrip + body << "\n\n" + body << added_dirs(info) + body << added_files(info) + body << deleted_dirs(info) + body << deleted_files(info) + body << modified_dirs(info) + body << modified_files(info) + [body.rstrip].pack('M') + end + + def format_time(time) + time.strftime('%Y-%m-%d %X %z (%a, %d %b %Y)') + end + + def changed_items(title, type, items) + rv = '' + unless items.empty? + rv << " #{title} #{type}:\n" + rv << items.collect {|item| " #{item}\n"}.join('') + end + rv + end + + def changed_files(title, files) + changed_items(title, 'files', files) + end + + def added_files(info) + changed_files('Added', info.added_files) + end + + def deleted_files(info) + changed_files('Removed', info.deleted_files) + end + + def modified_files(info) + changed_files('Modified', info.updated_files) + end + + def changed_dirs(title, files) + changed_items(title, 'directories', files) + end + + def added_dirs(info) + changed_dirs('Added', info.added_dirs) + end + + def deleted_dirs(info) + changed_dirs('Removed', info.deleted_dirs) + end + + def modified_dirs(info) + changed_dirs('Modified', info.updated_dirs) + end + + def changed_dirs_info(info, uri) + (info.added_dirs.collect do |dir| + " Added: #{dir}\n" + end + info.deleted_dirs.collect do |dir| + " Deleted: #{dir}\n" + end + info.updated_dirs.collect do |dir| + " Modified: #{dir}\n" + end).join("\n") + end + + def diff_info(info, uri) + info.diffs.collect do |key, values| + [ + key, + values.collect do |type, value| + case type + when :added + rev = "?revision=#{info.revision}&view=markup" + when :modified, :property_changed + prev_revision = (info.revision.is_a?(Integer) ? info.revision - 1 : "#{info.revision}^") + rev = "?r1=#{info.revision}&r2=#{prev_revision}&diff_format=u" + when :deleted, :copied + rev = '' + else + raise "unknown diff type: #{value[:type]}" + end + + link = [uri, key.sub(/ .+/, '') || ''].join('/') + rev + + desc = '' + + [desc, link] + end + ] + end + end + + def make_header(to, from, info) + <<~EOS + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: #{from} + To: #{to} + Subject: #{make_subject(info)} + EOS + end + + def make_subject(info) + subject = +'' + subject << "#{info.revision}" + subject << " (#{info.branch})" + subject << ': ' + subject << info.log.lstrip.lines.first.to_s.strip + b_encode(subject) + end + + # https://tools.ietf.org/html/rfc822#section-4.1 + # https://tools.ietf.org/html/rfc822#section-6.1 + # https://tools.ietf.org/html/rfc822#appendix-D + # https://tools.ietf.org/html/rfc2047 + def make_from(name:, email:) + if name.ascii_only? + escaped_name = name.gsub(/["\\\n]/) { |c| "\\#{c}" } + %Q["#{escaped_name}" <#{email}>] + else + escaped_name = "=?UTF-8?B?#{NKF.nkf('-WwMB', name)}?=" + %Q[#{escaped_name} <#{email}>] + end + end + + def make_mail(to, from, info, viewer_uri:) + make_header(to, from, info) + make_body(info, viewer_uri: viewer_uri) + end +end + +repo_path, to, *rest = ARGV +begin + CommitEmail.main(repo_path, to, rest) +rescue StandardError => e + $stderr.puts "#{e.class}: #{e.message}" + $stderr.puts e.backtrace + + _, options = CommitEmail.parse(rest) + to = options.error_to + CommitEmail.sendmail(to, to, <<-MAIL) +From: #{to} +To: #{to} +Subject: Error + +#{$!.class}: #{$!.message} +#{$@.join("\n")} +MAIL + exit 1 +end diff --git a/tool/darwin-ar b/tool/darwin-ar new file mode 100755 index 0000000000..8b25425cfe --- /dev/null +++ b/tool/darwin-ar @@ -0,0 +1,6 @@ +#!/bin/bash +export LANG=C LC_ALL=C # Suppress localication +exec 2> >(exec grep -v \ + -e ' no symbols$' \ + >&2) +exec "$@" diff --git a/tool/darwin-cc b/tool/darwin-cc index 6eee96e435..42637022a4 100755 --- a/tool/darwin-cc +++ b/tool/darwin-cc @@ -2,5 +2,8 @@ exec 2> >(exec grep -v \ -e '^ld: warning: The [a-z0-9_][a-z0-9_]* architecture is deprecated for macOS' \ -e '^ld: warning: text-based stub file /System/Library/Frameworks/' \ + -e '^ld: warning: ignoring duplicate libraries:' \ + -e "warning: '\.debug_macinfo' is not currently supported:" \ + -e "note: while processing" \ >&2) exec "$@" diff --git a/tool/disable_ipv6.sh b/tool/disable_ipv6.sh new file mode 100755 index 0000000000..ce1cc0da68 --- /dev/null +++ b/tool/disable_ipv6.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex +sysctl -w net.ipv6.conf.all.disable_ipv6=1 +sysctl -w net.ipv6.conf.default.disable_ipv6=1 +sysctl -w net.ipv6.conf.lo.disable_ipv6=1 + +cat /etc/hosts +ruby -e "hosts = File.read('/etc/hosts').sub(/^::1\s*localhost.*$/, ''); File.write('/etc/hosts', hosts)" +cat /etc/hosts diff --git a/tool/downloader.rb b/tool/downloader.rb index e9ea00c4c4..39ebf44a83 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -1,74 +1,53 @@ # Used by configure and make to download or update mirrored Ruby and GCC -# files. This will use HTTPS if possible, falling back to HTTP. +# files. + +# -*- frozen-string-literal: true -*- require 'fileutils' require 'open-uri' require 'pathname' -begin - require 'net/https' -rescue LoadError - https = 'http' -else - https = 'https' - - # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old - # versions do not. so, patching OpenSSL::X509::Store#add_file instead. - class OpenSSL::X509::Store - alias orig_add_file add_file - def add_file(pems) - Array(pems).each do |pem| - if File.directory?(pem) - add_path pem - else - orig_add_file pem - end - end - end - end - # since open-uri internally checks ssl_ca_cert using File.directory?, - # allow to accept an array. - class <<File - alias orig_directory? directory? - def File.directory? files - files.is_a?(Array) ? false : orig_directory?(files) - end - end -end +verbose, $VERBOSE = $VERBOSE, nil +require 'net/https' +$VERBOSE = verbose class Downloader - def self.https=(https) - @@https = https - end - - def self.https? - @@https == 'https' + def self.find(dlname) + constants.find do |name| + return const_get(name) if dlname.casecmp(name.to_s) == 0 + end end - def self.https - @@https + def self.get_option(argv, options) + false end class GNU < self - def self.download(name, *rest) - if https? - begin - super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest) - rescue => e - STDERR.puts "Download failed (#{e.message}), try another URL" - super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) - end + Mirrors = %w[ + https://raw.githubusercontent.com/autotools-mirror/autoconf/refs/heads/master/build-aux/ + https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master + ] + + def self.download(name, *rest, **options) + Mirrors.each_with_index do |url, i| + super("#{url}/#{name}", name, *rest, **options) + rescue => e + raise if i + 1 == Mirrors.size # no more URLs + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" else - super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) + return end end end class RubyGems < self - def self.download(name, dir = nil, since = true, options = {}) + def self.download(name, dir = nil, since = true, **options) require 'rubygems' - options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) - super("https://rubygems.org/downloads/#{name}", name, dir, since, options) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true + end + super("https://rubygems.org/downloads/#{name}", name, dir, since, **options) end end @@ -78,16 +57,28 @@ class Downloader INDEX = {} # cache index file information across files in the same directory UNICODE_PUBLIC = "https://www.unicode.org/Public/" - def self.download(name, dir = nil, since = true, options = {}) - options = options.dup - unicode_beta = options.delete(:unicode_beta) + def self.get_option(argv, options) + case argv[0] + when '--unicode-beta' + options[:unicode_beta] = argv[1] + argv.shift(2) + true + when /\A--unicode-beta=(.*)/m + options[:unicode_beta] = $1 + argv.shift + true + else + super + end + end + + def self.download(name, dir = nil, since = true, unicode_beta: nil, **options) name_dir_part = name.sub(/[^\/]+$/, '') if unicode_beta == 'YES' if INDEX.size == 0 - index_options = options.dup - index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file + cache_save = false # TODO: make sure caching really doesn't work for index file index_data = File.read(under(dir, "index.html")) rescue nil - index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options) + index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, cache_save: cache_save, **options) INDEX[:index] = File.read(index_file) since = true unless INDEX[:index] == index_data end @@ -96,7 +87,7 @@ class Downloader beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] # make sure we always check for new versions of files, # because they can easily change in the beta period - super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options) + super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, **options) else index_file = Pathname.new(under(dir, name_dir_part+'index.html')) if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ @@ -104,7 +95,7 @@ class Downloader "Remove all files in this directory and in .downloaded-cache/ " + "because they may be leftovers from the beta period." end - super(UNICODE_PUBLIC+name, name, dir, since, options) + super(UNICODE_PUBLIC+name, name, dir, since, **options) end end end @@ -169,59 +160,53 @@ class Downloader # Example usage: # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', # 'UnicodeData.txt', 'enc/unicode/data' - def self.download(url, name, dir = nil, since = true, options = {}) - options = options.dup + def self.download(url, name, dir = nil, since = true, + cache_save: ENV["CACHE_SAVE"] != "no", cache_dir: nil, + ignore_http_client_errors: nil, + dryrun: nil, verbose: false, **options) url = URI(url) - dryrun = options.delete(:dryrun) - options.delete(:unicode_beta) # just to be on the safe side for gems and gcc - if name file = Pathname.new(under(dir, name)) else name = File.basename(url.path) end - cache_save = options.delete(:cache_save) { - ENV["CACHE_SAVE"] != "no" - } - cache = cache_file(url, name, options.delete(:cache_dir)) + cache = cache_file(url, name, cache_dir) file ||= cache if since.nil? and file.exist? - if $VERBOSE + if verbose $stdout.puts "#{file} already exists" $stdout.flush end - if cache_save - save_cache(cache, file, name) - end return file.to_path end if dryrun puts "Download #{url} into #{file}" return end - if link_cache(cache, file, name, $VERBOSE) + if link_cache(cache, file, name, verbose: verbose) return file.to_path end - if !https? and URI::HTTPS === url - warn "*** using http instead of https ***" - url.scheme = 'http' - url = URI(url.to_s) - end - if $VERBOSE + if verbose $stdout.print "downloading #{name} ... " $stdout.flush end + mtime = nil + options = options.merge(http_options(file, since.nil? ? true : since)) begin - data = with_retry(10) do - url.read(options.merge(http_options(file, since.nil? ? true : since))) - end + data = with_retry(10) {url.read(options)} rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified - if $VERBOSE + case http_error.message + when /^304 / # 304 Not Modified + if verbose $stdout.puts "#{name} not modified" $stdout.flush end return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error @@ -236,20 +221,22 @@ class Downloader return file.to_path end raise + else + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end end - mtime = nil dest = (cache_save && cache && !cache.exist? ? cache : file) dest.parent.mkpath + dest.unlink if dest.symlink? && !dest.exist? dest.open("wb", 0600) do |f| f.write(data) f.chmod(mode_for(data)) - mtime = data.meta["last-modified"] end if mtime - mtime = httpdate(mtime) dest.utime(mtime, mtime) end - if $VERBOSE + if verbose $stdout.puts "done" $stdout.flush end @@ -267,25 +254,38 @@ class Downloader dir ? File.join(dir, File.basename(name)) : name end + def self.default_cache_dir + if cache_dir = ENV['CACHE_DIR'] + return cache_dir unless cache_dir.empty? + end + ".downloaded-cache" + end + def self.cache_file(url, name, cache_dir = nil) case cache_dir when false return nil when nil - cache_dir = ENV['CACHE_DIR'] - if !cache_dir or cache_dir.empty? - cache_dir = ".downloaded-cache" - end + cache_dir = default_cache_dir end Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) end - def self.link_cache(cache, file, name, verbose = false) + def self.link_cache(cache, file, name, verbose: false) return false unless cache and cache.exist? return true if cache.eql?(file) if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] begin - file.make_symlink(cache.relative_path_from(file.parent)) + link = cache.relative_path_from(file.parent) + rescue ArgumentError + abs = cache.expand_path + link = abs.relative_path_from(file.parent.expand_path) + if link.to_s.count("/") > abs.to_s.count("/") + link = abs + end + end + begin + file.make_symlink(link) rescue SystemCallError else if verbose @@ -328,7 +328,7 @@ class Downloader times = 0 begin block.call - rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout => e + rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error times += 1 if times <= max_times @@ -343,50 +343,76 @@ class Downloader private_class_method :with_retry end -Downloader.https = https.freeze - if $0 == __FILE__ since = true options = {} + dl = nil + (args = []).singleton_class.__send__(:define_method, :downloader?) do |arg| + !dl and args.empty? and (dl = Downloader.find(arg)) + end until ARGV.empty? + if ARGV[0] == '--' + ARGV.shift + break if ARGV.empty? + ARGV.shift if args.downloader? ARGV[0] + args.concat(ARGV) + break + end + + if dl and dl.get_option(ARGV, options) + # the downloader dealt with the arguments, and should be removed + # from ARGV. + next + end + case ARGV[0] - when '-d' + when '-d', '--destdir' + ## -d, --destdir DIRECTORY Download into the directory destdir = ARGV[1] ARGV.shift - when '-p' - # strip directory names from the name to download, and add the - # prefix instead. + when '-p', '--prefix' + ## -p, --prefix Strip directory names from the name to download, + ## and add the prefix instead. prefix = ARGV[1] ARGV.shift - when '-e' + when '-e', '--exist', '--non-existent-only' + ## -e, --exist, --non-existent-only Skip already existent files. since = nil - when '-a' + when '-a', '--always' + ## -a, --always Download all files. since = false - when '-n', '--dryrun' + when '-u', '--update', '--if-modified' + ## -u, --update, --if-modified Download newer files only. + since = true + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Do not download actually. options[:dryrun] = true when '--cache-dir' + ## --cache-dir DIRECTORY Cache downloaded files in the directory. options[:cache_dir] = ARGV[1] ARGV.shift - when '--unicode-beta' - options[:unicode_beta] = ARGV[1] - ARGV.shift when /\A--cache-dir=(.*)/m options[:cache_dir] = $1 + when /\A--help\z/ + ## --help Print this message + puts "Usage: #$0 [options] relative-url..." + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ abort "#{$0}: unknown option #{ARGV[0]}" else - break + args << ARGV[0] unless args.downloader? ARGV[0] end ARGV.shift end - dl = Downloader.constants.find do |name| - ARGV[0].casecmp(name.to_s) == 0 - end unless ARGV.empty? - $VERBOSE = true + options[:verbose] = true if dl - dl = Downloader.const_get(dl) - ARGV.shift - ARGV.each do |name| + args.each do |name| dir = destdir if prefix name = name.sub(/\A\.\//, '') @@ -403,10 +429,10 @@ if $0 == __FILE__ end name = "#{prefix}/#{name}" end - dl.download(name, dir, since, options) + dl.download(name, dir, since, **options) end else - abort "usage: #{$0} url name" unless ARGV.size == 2 - Downloader.download(ARGV[0], ARGV[1], destdir, since, options) + abort "usage: #{$0} url name" unless args.size == 2 + Downloader.download(args[0], args[1], destdir, since, **options) end end diff --git a/tool/enc-case-folding.rb b/tool/enc-case-folding.rb new file mode 100755 index 0000000000..82fec7b625 --- /dev/null +++ b/tool/enc-case-folding.rb @@ -0,0 +1,416 @@ +#!/usr/bin/ruby +require 'stringio' + +# Usage (for case folding only): +# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +# $ ruby enc-case-folding.rb CaseFolding.txt -o casefold.h +# or (for case folding and case mapping): +# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +# $ wget http://www.unicode.org/Public/UNIDATA/UnicodeData.txt +# $ wget http://www.unicode.org/Public/UNIDATA/SpecialCasing.txt +# $ ruby enc-case-folding.rb -m . -o casefold.h +# using -d or --debug will include UTF-8 characters in comments for debugging + +class CaseFolding + module Util + module_function + + def hex_seq(v) + v.map { |i| "0x%04x" % i }.join(", ") + end + + def print_table_1(dest, type, mapping_data, data) + for k, v in data = data.sort + sk = (Array === k and k.length > 1) ? "{#{hex_seq(k)}}" : ("0x%04x" % k) + if type=='CaseUnfold_11' and v.length>1 + # reorder CaseUnfold_11 entries to avoid special treatment for U+03B9/U+03BC/U+A64B + item = mapping_data.map("%04X" % k[0]) + upper = item.upper if item + v = v.sort_by { |i| ("%04X"%i) == upper ? 0 : 1 } + end + ck = @debug ? ' /* ' + Array(k).pack("U*") + ' */' : '' + cv = @debug ? ' /* ' + Array(v).map{|c|[c].pack("U*")}.join(", ") + ' */' : '' + dest.print(" {#{sk}#{ck}, {#{v.length}#{mapping_data.flags(k, type, v)}, {#{hex_seq(v)}#{cv}}}},\n") + end + data + end + + def print_table(dest, type, mapping_data, data) + dest.print("static const #{type}_Type #{type}_Table[] = {\n") + i = 0 + ret = data.inject([]) do |a, (n, d)| + dest.print("#define #{n} (*(#{type}_Type (*)[#{d.size}])(#{type}_Table+#{i}))\n") + i += d.size + a.concat(print_table_1(dest, type, mapping_data, d)) + end + dest.print("};\n\n") + ret + end + end + + include Util + + attr_reader :fold, :fold_locale, :unfold, :unfold_locale, :version + + def load(filename) + pattern = /([0-9A-F]{4,6}); ([CFT]); ([0-9A-F]{4,6})(?: ([0-9A-F]{4,6}))?(?: ([0-9A-F]{4,6}))?;/ + + @fold = fold = {} + @unfold = unfold = [{}, {}, {}] + @debug = false + @version = nil + turkic = [] + + File.foreach(filename, mode: "rb") do |line| + @version ||= line[/-([0-9.]+).txt/, 1] + next unless res = pattern.match(line) + ch_from = res[1].to_i(16) + + if res[2] == 'T' + # Turkic case folding + turkic << ch_from + next + end + + # store folding data + ch_to = res[3..6].inject([]) do |a, i| + break a unless i + a << i.to_i(16) + end + fold[ch_from] = ch_to + + # store unfolding data + i = ch_to.length - 1 + (unfold[i][ch_to] ||= []) << ch_from + end + + # move locale dependent data to (un)fold_locale + @fold_locale = fold_locale = {} + @unfold_locale = unfold_locale = [{}, {}] + for ch_from in turkic + key = fold[ch_from] + i = key.length - 1 + unfold_locale[i][i == 0 ? key[0] : key] = unfold[i].delete(key) + fold_locale[ch_from] = fold.delete(ch_from) + end + self + end + + def range_check(code) + "#{code} <= MAX_CODE_VALUE && #{code} >= MIN_CODE_VALUE" + end + + def lookup_hash(key, type, data) + hash = "onigenc_unicode_#{key}_hash" + lookup = "onigenc_unicode_#{key}_lookup" + arity = Array(data[0][0]).size + gperf = %W"gperf -7 -k#{[*1..(arity*3)].join(',')} -F,-1 -c -j1 -i1 -t -T -E -C -H #{hash} -N #{lookup} -n" + argname = arity > 1 ? "codes" : "code" + argdecl = "const OnigCodePoint #{arity > 1 ? "*": ""}#{argname}" + n = 7 + m = (1 << n) - 1 + min, max = data.map {|c, *|c}.flatten.minmax + src = IO.popen(gperf, "r+") {|f| + f << "short\n%%\n" + data.each_with_index {|(k, _), i| + k = Array(k) + ks = k.map {|j| [(j >> n*2) & m, (j >> n) & m, (j) & m]}.flatten.map {|c| "\\x%.2x" % c}.join("") + f.printf "\"%s\", ::::/*%s*/ %d\n", ks, k.map {|c| "0x%.4x" % c}.join(","), i + } + f << "%%\n" + f.close_write + f.read + } + src.sub!(/^(#{hash})\s*\(.*?\).*?\n\{\n(.*)^\}/m) { + name = $1 + body = $2 + body.gsub!(/\(unsigned char\)str\[(\d+)\]/, "bits_#{arity > 1 ? 'at' : 'of'}(#{argname}, \\1)") + "#{name}(#{argdecl})\n{\n#{body}}" + } + src.sub!(/const short *\*\n^(#{lookup})\s*\(.*?\).*?\n\{\n(.*)^\}/m) { + name = $1 + body = $2 + body.sub!(/\benum\s+\{(\n[ \t]+)/, "\\&MIN_CODE_VALUE = 0x#{min.to_s(16)},\\1""MAX_CODE_VALUE = 0x#{max.to_s(16)},\\1") + body.gsub!(/(#{hash})\s*\(.*?\)/, "\\1(#{argname})") + body.gsub!(/\{"",-1}/, "-1") + body.gsub!(/\{"(?:[^"]|\\")+", *::::(.*)\}/, '\1') + body.sub!(/(\s+if\s)\(len\b.*\)/) do + "#$1(" << + (arity > 1 ? (0...arity).map {|i| range_check("#{argname}[#{i}]")}.join(" &&\n ") : range_check(argname)) << + ")" + end + v = nil + body.sub!(/(if\s*\(.*MAX_HASH_VALUE.*\)\n([ \t]*))\{(.*?)\n\2\}/m) { + pre = $1 + indent = $2 + s = $3 + s.sub!(/const char *\* *(\w+)( *= *wordlist\[\w+\]).\w+/, 'short \1 = wordlist[key]') + v = $1 + s.sub!(/\bif *\(.*\)/, "if (#{v} >= 0 && code#{arity}_equal(#{argname}, #{key}_Table[#{v}].from))") + "#{pre}{#{s}\n#{indent}}" + } + body.sub!(/\b(return\s+&)([^;]+);/, '\1'"#{key}_Table[#{v}].to;") + "static const #{type} *\n#{name}(#{argdecl})\n{\n#{body}}" + } + src + end + + def display(dest, mapping_data) + # print the header + dest.print("/* DO NOT EDIT THIS FILE. */\n") + dest.print("/* Generated by enc-case-folding.rb */\n\n") + + versions = version.scan(/\d+/) + dest.print("#if defined ONIG_UNICODE_VERSION_STRING && !( \\\n") + %w[MAJOR MINOR TEENY].zip(versions) do |n, v| + dest.print(" ONIG_UNICODE_VERSION_#{n} == #{v} && \\\n") + end + dest.print(" 1)\n") + dest.print("# error ONIG_UNICODE_VERSION_STRING mismatch\n") + dest.print("#endif\n") + dest.print("#define ONIG_UNICODE_VERSION_STRING #{version.dump}\n") + %w[MAJOR MINOR TEENY].zip(versions) do |n, v| + dest.print("#define ONIG_UNICODE_VERSION_#{n} #{v}\n") + end + dest.print("\n") + + # print folding data + + # CaseFold + CaseFold_Locale + name = "CaseFold_11" + data = print_table(dest, name, mapping_data, "CaseFold"=>fold, "CaseFold_Locale"=>fold_locale) + dest.print lookup_hash(name, "CodePointList3", data) + + # print unfolding data + + # CaseUnfold_11 + CaseUnfold_11_Locale + name = "CaseUnfold_11" + data = print_table(dest, name, mapping_data, name=>unfold[0], "#{name}_Locale"=>unfold_locale[0]) + dest.print lookup_hash(name, "CodePointList3", data) + + # CaseUnfold_12 + CaseUnfold_12_Locale + name = "CaseUnfold_12" + data = print_table(dest, name, mapping_data, name=>unfold[1], "#{name}_Locale"=>unfold_locale[1]) + dest.print lookup_hash(name, "CodePointList2", data) + + # CaseUnfold_13 + name = "CaseUnfold_13" + data = print_table(dest, name, mapping_data, name=>unfold[2]) + dest.print lookup_hash(name, "CodePointList2", data) + + # TitleCase + dest.print mapping_data.specials_output + end + + def debug! + @debug = true + end + + def self.load(*args) + new.load(*args) + end +end + +class MapItem + attr_accessor :upper, :lower, :title, :code + + def initialize(code, upper, lower, title) + @code = code + @upper = upper unless upper == '' + @lower = lower unless lower == '' + @title = title unless title == '' + end +end + +class CaseMapping + attr_reader :filename, :version + + def initialize(mapping_directory) + @mappings = {} + @specials = [] + @specials_length = 0 + @version = nil + File.foreach(File.join(mapping_directory, 'UnicodeData.txt'), mode: "rb") do |line| + next if line =~ /^</ + code, _, _, _, _, _, _, _, _, _, _, _, upper, lower, title = line.chomp.split ';' + unless upper and lower and title and (upper+lower+title)=='' + @mappings[code] = MapItem.new(code, upper, lower, title) + end + end + + @filename = File.join(mapping_directory, 'SpecialCasing.txt') + File.foreach(@filename, mode: "rb") do |line| + @version ||= line[/-([0-9.]+).txt/, 1] + line.chomp! + line, comment = line.split(/ *#/) + next if not line or line == '' + code, lower, title, upper, conditions = line.split(/ *; */) + unless conditions + item = @mappings[code] + item.lower = lower + item.title = title + item.upper = upper + end + end + end + + def map (from) + @mappings[from] + end + + def flags(from, type, to) + # types: CaseFold_11, CaseUnfold_11, CaseUnfold_12, CaseUnfold_13 + flags = "" + from = Array(from).map {|i| "%04X" % i}.join(" ") + to = Array(to).map {|i| "%04X" % i}.join(" ") + item = map(from) + specials = [] + case type + when 'CaseFold_11' + flags += '|F' + if item + flags += '|U' if to==item.upper + flags += '|D' if to==item.lower + unless item.upper == item.title + if item.code == item.title + flags += '|IT' + swap = case item.code + when '01C5' then '0064 017D' + when '01C8' then '006C 004A' + when '01CB' then '006E 004A' + when '01F2' then '0064 005A' + else # Greek + to.split(' ').first + ' 0399' + end + specials << swap + else + flags += '|ST' + specials << item.title + end + end + unless item.lower.nil? or item.lower==from or item.lower==to + specials << item.lower + flags += '|SL' + end + unless item.upper.nil? or item.upper==from or item.upper==to + specials << item.upper + flags += '|SU' + end + end + when 'CaseUnfold_11' + to = to.split(/ /) + if item + case to.first + when item.upper then flags += '|U' + when item.lower then flags += '|D' + else + raise "Unpredicted case 0 in enc/unicode/case_folding.rb. Please contact https://bugs.ruby-lang.org/." + end + unless item.upper == item.title + if item.code == item.title + flags += '|IT' # was unpredicted case 1 + elsif item.title==to[1] + flags += '|ST' + else + raise "Unpredicted case 2 in enc/unicode/case_folding.rb. Please contact https://bugs.ruby-lang.org/." + end + end + end + end + unless specials.empty? + flags += "|I(#{@specials_length})" + @specials_length += specials.map { |s| s.split(/ /).length }.reduce(:+) + @specials << specials + end + flags + end + + def debug! + @debug = true + end + + def specials_output + "static const OnigCodePoint CaseMappingSpecials[] = {\n" + + @specials.map do |sps| + ' ' + sps.map do |sp| + chars = sp.split(/ /) + ct = ' /* ' + Array(chars).map{|c|[c.to_i(16)].pack("U*")}.join(", ") + ' */' if @debug + " L(#{chars.length})|#{chars.map {|c| "0x"+c }.join(', ')}#{ct}," + end.join + "\n" + end.join + "};\n" + end + + def self.load(*args) + new(*args) + end +end + +class CaseMappingDummy + def flags(from, type, to) + "" + end + + def titlecase_output() '' end + def debug!() end +end + +if $0 == __FILE__ + require 'optparse' + dest = nil + mapping_directory = nil + mapping_data = nil + debug = false + fold_1 = false + ARGV.options do |opt| + opt.banner << " [INPUT]" + opt.on("--output-file=FILE", "-o", "output to the FILE instead of STDOUT") {|output| + dest = (output unless output == '-') + } + opt.on('--mapping-data-directory=DIRECTORY', '-m', 'data DIRECTORY of mapping files') { |directory| + mapping_directory = directory + } + opt.on('--debug', '-d') { + debug = true + } + opt.parse! + abort(opt.to_s) if ARGV.size > 1 + end + if mapping_directory + if ARGV[0] + warn "Either specify directory or individual file, but not both." + exit + end + filename = File.join(mapping_directory, 'CaseFolding.txt') + mapping_data = CaseMapping.load(mapping_directory) + end + filename ||= ARGV[0] || 'CaseFolding.txt' + data = CaseFolding.load(filename) + if mapping_data and data.version != mapping_data.version + abort "Unicode data version mismatch\n" \ + " #{filename} = #{data.version}\n" \ + " #{mapping_data.filename} = #{mapping_data.version}" + end + mapping_data ||= CaseMappingDummy.new + + if debug + data.debug! + mapping_data.debug! + end + f = StringIO.new + begin + data.display(f, mapping_data) + rescue Errno::ENOENT => e + raise unless /gperf/ =~ e.message + warn e.message + abort unless dest + File.utime(nil, nil, dest) # assume existing file is OK + exit + else + s = f.string + end + if dest + File.binwrite(dest, s) + else + STDOUT.print(s) + end +end diff --git a/tool/enc-emoji-citrus-gen.rb b/tool/enc-emoji-citrus-gen.rb index da9c8a6b62..0b37e48d3f 100644 --- a/tool/enc-emoji-citrus-gen.rb +++ b/tool/enc-emoji-citrus-gen.rb @@ -71,7 +71,7 @@ end def generate_to_ucs(params, pairs) pairs.sort_by! {|u, c| c } name = "EMOJI_#{params[:name]}%UCS" - open("#{name}.src", "w") do |io| + File.open("#{name}.src", "w") do |io| io.print header(params.merge(name: name.tr('%', '/'))) io.puts io.puts "BEGIN_MAP" @@ -83,7 +83,7 @@ end def generate_from_ucs(params, pairs) pairs.sort_by! {|u, c| u } name = "UCS%EMOJI_#{params[:name]}" - open("#{name}.src", "w") do |io| + File.open("#{name}.src", "w") do |io| io.print header(params.merge(name: name.tr('%', '/'))) io.puts io.puts "BEGIN_MAP" diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index 93f6e869f8..a89390ad8f 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -5,14 +5,33 @@ # # To use this, get UnicodeData.txt, Scripts.txt, PropList.txt, # PropertyAliases.txt, PropertyValueAliases.txt, DerivedCoreProperties.txt, -# DerivedAge.txt and Blocks.txt from unicode.org. +# DerivedAge.txt, Blocks.txt, emoji/emoji-data.txt, +# auxiliary/GraphemeBreakProperty.txt from unicode.org # (http://unicode.org/Public/UNIDATA/) And run following command. -# ruby1.9 tool/enc-unicode.rb data_dir > enc/unicode/name2ctype.kwd +# tool/enc-unicode.rb data_dir emoji_data_dir > enc/unicode/name2ctype.kwd # You can get source file for gperf. After this, simply make ruby. - -if ARGV[0] == "--header" - header = true - ARGV.shift +# Or directly run: +# tool/enc-unicode.rb --header data_dir emoji_data_dir > enc/unicode/<VERSION>/name2ctype.h +# +# There are Makefile rules that automate steps above: `make update-unicode` and +# `make enc/unicode/<VERSION>/name2ctype.h`. + +while arg = ARGV.shift + case arg + when "--" + break + when "--header" + header = true + when "--diff" + diff = ARGV.shift or abort "#{$0}: --diff=DIFF-COMMAND" + when /\A--diff=(.+)/m + diff = $1 + when /\A-/ + abort "#{$0}: unknown option #{arg}" + else + ARGV.unshift(arg) + break + end end unless ARGV.size == 2 abort "Usage: #{$0} data_directory emoji_data_directory" @@ -59,7 +78,7 @@ def parse_unicode_data(file) data = {'Any' => (0x0000..0x10ffff).to_a, 'Assigned' => [], 'ASCII' => (0..0x007F).to_a, 'NEWLINE' => [0x0a], 'Cn' => []} beg_cp = nil - IO.foreach(file) do |line| + File.foreach(file) do |line| fields = line.split(';') cp = fields[0].to_i(16) @@ -127,7 +146,8 @@ def define_posix_props(data) data['Space'] = data['White_Space'] data['Blank'] = data['Space_Separator'] + [0x0009] data['Cntrl'] = data['Cc'] - data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + + data['Connector_Punctuation'] + data['Join_Control'] data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - data['Surrogate'] - data['Unassigned'] data['Print'] = data['Graph'] + data['Space_Separator'] @@ -145,14 +165,24 @@ def parse_scripts(data, categories) names = {} files.each do |file| data_foreach(file[:fn]) do |line| + # Parse Unicode data files and store code points and properties. if /^# Total (?:code points|elements): / =~ line data[current] = cps categories[current] = file[:title] (names[file[:title]] ||= []) << current cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line - current = $3 + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w(?:[\w\s;]*\w)?)/ =~ line + # $1: The first hexadecimal code point or the start of a range. + # $2: The end code point of the range, if present. + # If there's no range (just a single code point), $2 is nil. + # $3: The property or other info. + # Example: + # line = "0915..0939 ; InCB; Consonant # Lo [37] DEVANAGARI LETTER KA..DEVANAGARI LETTER HA" + # $1 = "0915" + # $2 = "0939" + # $3 = "InCB; Consonant" $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + current = $3.gsub(/\W+/, '_') end end end @@ -205,7 +235,7 @@ def parse_age(data) ages << current last_constname = constname cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\d+\.\d+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\d+\.\d+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -224,7 +254,7 @@ def parse_GraphemeBreakProperty(data) make_const(constname, cps, "Grapheme_Cluster_Break=#{current}") ages << current cps = [] - elsif /^([0-9a-fA-F]+)(?:\.\.([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -236,7 +266,7 @@ def parse_block(data) cps = [] blocks = [] data_foreach('Blocks.txt') do |line| - if /^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+);\s*(.*)/ =~ line + if /^(\h+)\.\.(\h+);\s*(.*)/ =~ line cps = ($1.to_i(16)..$2.to_i(16)).to_a constname = constantize_blockname($3) data[constname] = cps @@ -253,23 +283,12 @@ def parse_block(data) blocks << constname end -# shim for Ruby 1.8 -unless {}.respond_to?(:key) - class Hash - alias key index - end -end - $const_cache = {} # make_const(property, pairs, name): Prints a 'static const' structure for a # given property, group of paired codepoints, and a human-friendly name for # the group def make_const(prop, data, name) - if name.empty? - puts "\n/* '#{prop}' */" - else - puts "\n/* '#{prop}': #{name} */" - end + puts "\n/* '#{prop}': #{name} */" # comment used to generate documentation if origprop = $const_cache.key(data) puts "#define CR_#{prop} CR_#{origprop}" else @@ -311,18 +330,19 @@ end def data_foreach(name, &block) fn = get_file(name) warn "Reading #{name}" - if /^emoji/ =~ name - sep = "" - pat = /^# #{Regexp.quote(File.basename(name))}.*^# Version: ([\d.]+)/m - type = :Emoji - else - sep = "\n" - pat = /^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/ - type = :Unicode - end File.open(fn, 'rb') do |f| - line = f.gets(sep) - unless version = line[pat, 1] + if /^emoji/ =~ name + line = f.gets("") + # Headers till Emoji 13 or 15 + version = line[/^# #{Regexp.quote(File.basename(name))}.*(?:^# Version:|Emoji Version) ([\d.]+)/m, 1] + type = :Emoji + else + # Headers since Emoji 14 or other Unicode data + line = f.gets("\n") + type = :Unicode + end + version ||= line[/^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/, 1] + unless version raise ArgumentError, <<-ERROR #{name}: no #{type} version #{line.gsub(/^/, '> ')} @@ -330,7 +350,7 @@ def data_foreach(name, &block) end if !(v = $versions[type]) $versions[type] = version - elsif v != version + elsif v != version and "#{v}.0" != version raise ArgumentError, <<-ERROR #{name}: #{type} version mismatch: #{version} to #{v} #{line.gsub(/^/, '> ')} @@ -420,8 +440,6 @@ define_posix_props(data) POSIX_NAMES.each do |name| if name == 'XPosixPunct' make_const(name, data[name], "[[:Punct:]]") - elsif name == 'Punct' - make_const(name, data[name], "") else make_const(name, data[name], "[[:#{name}:]]") end @@ -464,11 +482,7 @@ struct uniname2ctype_struct { }; #define uniname2ctype_offset(str) offsetof(struct uniname2ctype_pool_t, uniname2ctype_pool_##str) -static const struct uniname2ctype_struct *uniname2ctype_p( -#if !(/*ANSI*/+0) /* if ANSI, old style not to conflict with generated prototype */ - const char *, unsigned int -#endif -); +static const struct uniname2ctype_struct *uniname2ctype_p(register const char *str, register size_t len); %} struct uniname2ctype_struct; %% @@ -486,7 +500,11 @@ end output.ifdef :USE_UNICODE_PROPERTIES do props.each do |name| i += 1 - name = normalize_propname(name) + name = if name.start_with?('InCB') + name.downcase.gsub(/_/, '=') + else + normalize_propname(name) + end name_to_index[name] = i puts "%-40s %3d" % [name + ',', i] end @@ -547,6 +565,47 @@ output.restore if header require 'tempfile' + def diff_args(diff) + ok = IO.popen([diff, "-DDIFF_TEST", IO::NULL, "-"], "r+") do |f| + f.puts "Test for diffutils 3.8" + f.close_write + /^#if/ =~ f.read + end + if ok + proc {|macro, *inputs| + [diff, "-D#{macro}", *inputs] + } + else + IO.popen([diff, "--old-group-format=%<", "--new-group-format=%>", IO::NULL, IO::NULL], err: %i[child out], &:read) + unless $?.success? + abort "#{$0}: #{diff} -D does not work" + end + warn "Avoiding diffutils 3.8 bug#61193" + proc {|macro, *inputs| + [diff] + [ + "--old-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#endif /* ! @ */\n", + + "--new-group-format=" \ + "#ifdef @\n" \ + "%>" \ + "#endif /* @ */\n", + + "--changed-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#else /* @ */\n" \ + "%>" \ + "#endif /* @ */\n" + ].map {|opt| opt.gsub(/@/) {macro}} + inputs + } + end + end + + ifdef = diff_args(diff || "diff") + NAME2CTYPE = %w[gperf -7 -c -j1 -i1 -t -C -P -T -H uniname2ctype_hash -Q uniname2ctype_pool -N uniname2ctype_p] fds = [] @@ -556,9 +615,8 @@ if header IO.popen([*NAME2CTYPE, out: tmp], "w") {|f| output.show(f, *syms)} end while syms.pop fds.each(&:close) - ff = nil - IO.popen(%W[diff -DUSE_UNICODE_AGE_PROPERTIES #{fds[1].path} #{fds[0].path}], "r") {|age| - IO.popen(%W[diff -DUSE_UNICODE_PROPERTIES #{fds[2].path} -], "r", in: age) {|f| + IO.popen(ifdef["USE_UNICODE_AGE_PROPERTIES", fds[1].path, fds[0].path], "r") {|age| + IO.popen(ifdef["USE_UNICODE_PROPERTIES", fds[2].path, "-"], "r", in: age) {|f| ansi = false f.each {|line| if /ANSI-C code produced by gperf/ =~ line @@ -567,7 +625,7 @@ if header line.sub!(/\/\*ANSI\*\//, '1') if ansi line.gsub!(/\(int\)\((?:long|size_t)\)&\(\(struct uniname2ctype_pool_t \*\)0\)->uniname2ctype_pool_(str\d+),\s+/, 'uniname2ctype_offset(\1), ') - if ff = (!ff ? /^(uniname2ctype_hash) /=~line : /^\}/!~line) # no line can match both, exclusive flip-flop + if line.start_with?("uniname2ctype_hash\s") ... line.start_with?("}") line.sub!(/^( *(?:register\s+)?(.*\S)\s+hval\s*=\s*)(?=len;)/, '\1(\2)') end puts line diff --git a/tool/expand-config.rb b/tool/expand-config.rb index 81ffa6cb98..ac0ffbfd41 100755 --- a/tool/expand-config.rb +++ b/tool/expand-config.rb @@ -7,7 +7,17 @@ config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)$/, '\1true\2') rbconfig = Module.new {module_eval(config, conffile)}::RbConfig config = $expand ? rbconfig::CONFIG : rbconfig::MAKEFILE_CONFIG config["RUBY_RELEASE_DATE"] ||= - File.read(File.expand_path("../../version.h", __FILE__))[/^\s*#\s*define\s+RUBY_RELEASE_DATE\s+"(.*)"/, 1] + [ + ["revision.h"], + ["../../revision.h", __FILE__], + ["../../version.h", __FILE__], + ].find do |hdr, dir| + hdr = File.expand_path(hdr, dir) if dir + if date = File.read(hdr)[/^\s*#\s*define\s+RUBY_RELEASE_DATE(?:TIME)?\s+"([0-9-]*)/, 1] + break date + end +rescue +end while /\A(\w+)=(.*)/ =~ ARGV[0] config[$1] = $2 @@ -16,7 +26,7 @@ while /\A(\w+)=(.*)/ =~ ARGV[0] end if $output - output = open($output, "wb", $mode &&= $mode.oct) + output = File.open($output, "wb", $mode &&= $mode.oct) output.chmod($mode) if $mode else output = STDOUT diff --git a/tool/extlibs.rb b/tool/extlibs.rb index cd8e5239b3..cef6712833 100755 --- a/tool/extlibs.rb +++ b/tool/extlibs.rb @@ -5,24 +5,36 @@ require 'digest' require_relative 'downloader' -require_relative 'lib/colorize' +begin + require_relative 'lib/colorize' +rescue LoadError +end -class Vars < Hash - def pattern - /\$\((#{Regexp.union(keys)})\)/ +class ExtLibs + unless defined?(Colorize) + class Colorize + def pass(str) str; end + def fail(str) str; end + end end - def expand(str) - if empty? - str - else - str.gsub(pattern) {self[$1]} + class Vars < Hash + def pattern + /\$\((#{Regexp.union(keys)})\)/ + end + + def expand(str) + if empty? + str + else + str.gsub(pattern) {self[$1]} + end end end -end -class ExtLibs - def initialize + def initialize(mode = :all, cache_dir: nil) + @mode = mode + @cache_dir = cache_dir @colorize = Colorize.new end @@ -31,7 +43,7 @@ class ExtLibs end def do_download(url, cache_dir) - Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) + Downloader.download(url, nil, nil, nil, cache_dir: cache_dir) end def do_checksum(cache, chksums) @@ -160,7 +172,78 @@ class ExtLibs extracted end - def run(argv) + def process(list) + mode = @mode + cache_dir = @cache_dir + after_extract = (mode == :all or mode == :patch) + success = true + if $VERBOSE + $stdout.puts "downloading for #{list}" + $stdout.flush + end + vars = Vars.new + extracted = false + dest = File.dirname(list) + url = chksums = nil + File.foreach(list) do |line| + line.sub!(/\s*#.*/, '') + if /^(\w+)\s*=\s*(.*)/ =~ line + vars[$1] = vars.expand($2) + next + end + if chksums + chksums.concat(line.split) + elsif /^\t/ =~ line + if extracted and after_extract + patch, *args = line.split.map {|s| vars.expand(s)} + do_patch(dest, patch, args) + end + next + elsif /^!\s*(?:chdir:\s*([^|\s]+)\|\s*)?(.*)/ =~ line + if extracted and after_extract + command = vars.expand($2.strip) + chdir = $1 and chdir = vars.expand(chdir) + do_exec(command, chdir, dest) + end + next + elsif /->/ =~ line + if extracted and after_extract + link, file = $`.strip, $'.strip + do_link(vars.expand(link), vars.expand(file), dest) + end + next + else + url, *chksums = line.split(' ') + end + if chksums.last == '\\' + chksums.pop + next + end + unless url + chksums = nil + next + end + url = vars.expand(url) + begin + extracted = do_command(mode, dest, url, cache_dir, chksums) + rescue => e + warn defined?(e.full_message) ? e.full_message : e.message + success = false + end + url = chksums = nil + end + success + end + + def process_under(dir) + success = true + Dir.glob("#{dir}/**/extlibs") do |list| + success &= process(list) + end + success + end + + def self.run(argv) cache_dir = nil mode = :all until argv.empty? @@ -190,71 +273,10 @@ class ExtLibs argv.shift end - success = true - argv.each do |dir| - Dir.glob("#{dir}/**/extlibs") do |list| - if $VERBOSE - $stdout.puts "downloading for #{list}" - $stdout.flush - end - vars = Vars.new - extracted = false - dest = File.dirname(list) - url = chksums = nil - IO.foreach(list) do |line| - line.sub!(/\s*#.*/, '') - if /^(\w+)\s*=\s*(.*)/ =~ line - vars[$1] = vars.expand($2) - next - end - if chksums - chksums.concat(line.split) - elsif /^\t/ =~ line - if extracted and (mode == :all or mode == :patch) - patch, *args = line.split.map {|s| vars.expand(s)} - do_patch(dest, patch, args) - end - next - elsif /^!\s*(?:chdir:\s*([^|\s]+)\|\s*)?(.*)/ =~ line - if extracted and (mode == :all or mode == :patch) - command = vars.expand($2.strip) - chdir = $1 and chdir = vars.expand(chdir) - do_exec(command, chdir, dest) - end - next - elsif /->/ =~ line - if extracted and (mode == :all or mode == :patch) - link, file = $`.strip, $'.strip - do_link(vars.expand(link), vars.expand(file), dest) - end - next - else - url, *chksums = line.split(' ') - end - if chksums.last == '\\' - chksums.pop - next - end - unless url - chksums = nil - next - end - url = vars.expand(url) - begin - extracted = do_command(mode, dest, url, cache_dir, chksums) - rescue => e - warn e.full_message - success = false - end - url = chksums = nil - end - end + extlibs = new(mode, cache_dir: cache_dir) + argv.inject(true) do |success, dir| + success & extlibs.process_under(dir) end - success - end - - def self.run(argv) - self.new.run(argv) end end diff --git a/tool/fake.rb b/tool/fake.rb index 42174052e2..2c458985d8 100644 --- a/tool/fake.rb +++ b/tool/fake.rb @@ -9,6 +9,15 @@ class File end end +[[libpathenv, "."], [preloadenv, libruby_so]].each do |env, path| + env or next + e = ENV[env] or next + e = e.split(File::PATH_SEPARATOR) + path = File.realpath(path, builddir) rescue next + e.delete(path) or next + ENV[env] = (e.join(File::PATH_SEPARATOR) unless e.empty?) +end + static = !!(defined?($static) && $static) $:.unshift(builddir) posthook = proc do @@ -24,6 +33,7 @@ posthook = proc do untrace_var(:$ruby, posthook) end prehook = proc do |extmk| +=begin pat = %r[(?:\A(?:\w:|//[^/]+)|\G)/[^/]*] dir = builddir.scan(pat) pwd = Dir.pwd.scan(pat) @@ -35,6 +45,7 @@ prehook = proc do |extmk| builddir = File.join((pwd.empty? ? ["."] : [".."]*pwd.size) + dir) builddir = "." if builddir.empty? end +=end join = proc {|*args| File.join(*args).sub!(/\A(?:\.\/)*/, '')} $topdir ||= builddir $top_srcdir ||= (File.identical?(top_srcdir, dir = join[$topdir, srcdir]) ? @@ -43,13 +54,17 @@ prehook = proc do |extmk| $extout_prefix = '$(extout)$(target_prefix)/' config = RbConfig::CONFIG mkconfig = RbConfig::MAKEFILE_CONFIG + $builtruby ||= File.join(builddir, config['RUBY_INSTALL_NAME'] + config['EXEEXT']) RbConfig.fire_update!("builddir", builddir) RbConfig.fire_update!("buildlibdir", builddir) RbConfig.fire_update!("libdir", builddir) + RbConfig.fire_update!("prefix", $topdir) RbConfig.fire_update!("top_srcdir", $top_srcdir ||= top_srcdir) RbConfig.fire_update!("extout", $extout) RbConfig.fire_update!("rubyhdrdir", "$(top_srcdir)/include") RbConfig.fire_update!("rubyarchhdrdir", "$(extout)/include/$(arch)") + RbConfig.fire_update!("rubyarchdir", "$(extout)/$(arch)") + RbConfig.fire_update!("rubylibdir", "$(extout)/common") RbConfig.fire_update!("libdirname", "buildlibdir") trace_var(:$ruby, posthook) untrace_var(:$extmk, prehook) diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb index ebb70e88f9..127ea236f3 100755 --- a/tool/fetch-bundled_gems.rb +++ b/tool/fetch-bundled_gems.rb @@ -1,6 +1,16 @@ -#!ruby -an +#!ruby -alnF\s+|#.* BEGIN { require 'fileutils' + require_relative 'lib/colorize' + + color = Colorize.new + + if ARGV.first.start_with?("BUNDLED_GEMS=") + bundled_gems = ARGV.shift[13..-1] + sep = bundled_gems.include?(",") ? "," : " " + bundled_gems = bundled_gems.split(sep) + bundled_gems = nil if bundled_gems.empty? + end dir = ARGV.shift ARGF.eof? @@ -10,18 +20,35 @@ BEGIN { n, v, u, r = $F -next if n =~ /^#/ +next unless n +next if bundled_gems&.all? {|pat| !File.fnmatch?(pat, n)} + +unless File.exist?("#{n}/.git") + puts "retrieving #{color.notice(n)} ..." + system(*%W"git clone --depth=1 --no-tags #{u} #{n}") or abort +end -if File.directory?(n) - puts "updating #{n} ..." - system("git", "fetch", chdir: n) or abort +if r + puts "fetching #{color.notice(r)} ..." + system("git", "fetch", "origin", r, chdir: n) or abort + c = r else - puts "retrieving #{n} ..." - system(*%W"git clone #{u} #{n}") or abort + c = ["v#{v}", v].find do |c| + puts "fetching #{color.notice(c)} ..." + system("git", "fetch", "origin", "refs/tags/#{c}:refs/tags/#{c}", chdir: n) + end or abort end -c = r || "v#{v}" + checkout = %w"git -c advice.detachedHead=false checkout" -puts "checking out #{c} (v=#{v}, r=#{r}) ..." +info = %[, r=#{color.info(r)}] if r +puts "checking out #{color.notice(c)} (v=#{color.info(v)}#{info}) ..." unless system(*checkout, c, "--", chdir: n) - abort + abort if r or !system(*checkout, v, "--", chdir: n) +end + +if r + unless File.exist? "#{n}/#{n}.gemspec" + require_relative "lib/bundled_gem" + BundledGem.dummy_gemspec("#{n}/#{n}.gemspec") + end end diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb index 3d8c69357d..2de8379606 100755 --- a/tool/file2lastrev.rb +++ b/tool/file2lastrev.rb @@ -8,50 +8,47 @@ require 'optparse' # this file run with BASERUBY, which may be older than 1.9, so no # require_relative require File.expand_path('../lib/vcs', __FILE__) +require File.expand_path('../lib/output', __FILE__) Program = $0 -@output = nil -def self.output=(output) - if @output and @output != output +@format = nil +def self.format=(format) + if @format and @format != format raise "you can specify only one of --changed, --revision.h and --doxygen" end - @output = output + @format = format end @suppress_not_found = false @limit = 20 +@output = Output.new -format = '%Y-%m-%dT%H:%M:%S%z' +time_format = '%Y-%m-%dT%H:%M:%S%z' vcs = nil +create_only = false OptionParser.new {|opts| opts.banner << " paths..." vcs_options = VCS.define_options(opts) - new_vcs = proc do |path| - begin - vcs = VCS.detect(path, vcs_options, opts.new) - rescue VCS::NotFoundError => e - abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found - opts.remove - end - nil - end + opts.new {@output.def_options(opts)} + srcdir = nil opts.new opts.on("--srcdir=PATH", "use PATH as source directory") do |path| - abort "#{File.basename(Program)}: srcdir is already set" if vcs - new_vcs[path] + abort "#{File.basename(Program)}: srcdir is already set" if srcdir + srcdir = path + @output.vpath.add(srcdir) end opts.on("--changed", "changed rev") do - self.output = :changed + self.format = :changed end opts.on("--revision.h", "RUBY_REVISION macro") do - self.output = :revision_h + self.format = :revision_h end opts.on("--doxygen", "Doxygen format") do - self.output = :doxygen + self.format = :doxygen end opts.on("--modified[=FORMAT]", "modified time") do |fmt| - self.output = :modified - format = fmt if fmt + self.format = :modified + time_format = fmt if fmt end opts.on("--limit=NUM", "limit branch name length (#@limit)", Integer) do |n| @limit = n @@ -60,44 +57,27 @@ OptionParser.new {|opts| @suppress_not_found = true end opts.order! rescue abort "#{File.basename(Program)}: #{$!}\n#{opts}" - if vcs - vcs.set_options(vcs_options) # options after --srcdir - else - new_vcs["."] + begin + vcs = VCS.detect(srcdir || ".", vcs_options, opts.new) + rescue VCS::NotFoundError => e + abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found + opts.remove + (vcs = VCS::Null.new(nil)).set_options(vcs_options) + if @format == :revision_h + create_only = true # don't overwrite existing revision.h when .git doesn't exist + end end } -exit unless vcs -@output = - case @output +formatter = + case @format when :changed, nil Proc.new {|last, changed| - changed + changed || "" } when :revision_h Proc.new {|last, changed, modified, branch, title| - short = vcs.short_revision(last) - if /[^\x00-\x7f]/ =~ title and title.respond_to?(:force_encoding) - title = title.dup.force_encoding("US-ASCII") - end - [ - "#define RUBY_REVISION #{short.inspect}", - ("#define RUBY_FULL_REVISION #{last.inspect}" unless short == last), - if branch - e = '..' - limit = @limit - name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} - name = name.dump.sub(/\\#/, '#') - "#define RUBY_BRANCH_NAME #{name}" - end, - if title - title = title.dump.sub(/\\#/, '#') - "#define RUBY_LAST_COMMIT_TITLE #{title}" - end, - if modified - modified.utc.strftime('#define RUBY_RELEASE_DATETIME "%FT%TZ"') - end, - ].compact + vcs.revision_header(last, modified, modified, branch, title, limit: @limit).join("\n") } when :doxygen Proc.new {|last, changed| @@ -105,16 +85,18 @@ exit unless vcs } when :modified Proc.new {|last, changed, modified| - modified.strftime(format) + modified.strftime(time_format) } else - raise "unknown output format `#{@output}'" + raise "unknown output format `#{@format}'" end ok = true (ARGV.empty? ? [nil] : ARGV).each do |arg| begin - puts @output[*vcs.get_revisions(arg)] + data = formatter[*vcs.get_revisions(arg)] + data.sub!(/(?<!\A|\n)\z/, "\n") + @output.write(data, overwrite: true, create_only: create_only) rescue => e next if @suppress_not_found and VCS::NotFoundError === e warn "#{File.basename(Program)}: #{e.message}" diff --git a/tool/format-release b/tool/format-release index cdc2d55840..8bb6154243 100755 --- a/tool/format-release +++ b/tool/format-release @@ -1,8 +1,15 @@ #!/usr/bin/env ruby -# https://rubygems.org/gems/diffy -require "diffy" + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "diffy" +end + require "open-uri" require "yaml" +require_relative "./ruby-version" Diffy::Diff.default_options.merge!( include_diff_info: true, @@ -24,10 +31,9 @@ class Tarball def gz?; @url.end_with?('.gz'); end def zip?; @url.end_with?('.zip'); end - def bz2?; @url.end_with?('.bz2'); end def xz?; @url.end_with?('.xz'); end - def ext; @url[/(?:zip|tar\.(?:gz|bz2|xz))\z/]; end + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end def to_md <<eom @@ -45,29 +51,14 @@ eom # SHA1: 21f62c369661a2ab1b521fd2fa8191a4273e12a1 # SHA256: 97cea8aa63dfa250ba6902b658a7aa066daf817b22f82b7ee28f44aec7c2e394 # SHA512: 1e2042324821bb4e110af7067f52891606dcfc71e640c194ab1c117f0b941550e0b3ac36ad3511214ac80c536b9e5cfaf8789eec74cf56971a832ea8fc4e6d94 - def self.parse(wwwdir, version) + def self.parse(wwwdir, version, rubydir) unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" - info = YAML.load(URI(uri).read) + info = YAML.unsafe_load(URI(uri).read) if info.size != 1 raise "unexpected info.yml '#{uri}'" end @@ -82,10 +73,11 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" - stat = `git diff --shortstat #{prev_tag}..#{tag}` + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) + stat = `git -C #{rubydir} diff -l0 --shortstat #{prev_tag}..#{tag}` files_changed, insertions, deletions = stat.scan(/\d+/) end @@ -175,8 +167,10 @@ eom post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ eom - if ver.end_with?(".0") + if /\.0(?:-\w+)?\z/ =~ ver + # preview, rc, or first release entry <<= <<eom + tag: #{RubyVersion.tag(ver)} stats: files_changed: #{files_changed} insertions: #{insertions} @@ -188,35 +182,28 @@ eom url: gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip - bz2: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.bz2 xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz size: gz: #{ary.find{|x|x.gz? }.size} zip: #{ary.find{|x|x.zip?}.size} - bz2: #{ary.find{|x|x.bz2?}&.size} xz: #{ary.find{|x|x.xz? }.size} sha1: gz: #{ary.find{|x|x.gz? }.sha1} zip: #{ary.find{|x|x.zip?}.sha1} - bz2: #{ary.find{|x|x.bz2?}&.sha1} xz: #{ary.find{|x|x.xz? }.sha1} sha256: gz: #{ary.find{|x|x.gz? }.sha256} zip: #{ary.find{|x|x.zip?}.sha256} - bz2: #{ary.find{|x|x.bz2?}&.sha256} xz: #{ary.find{|x|x.xz? }.sha256} sha512: gz: #{ary.find{|x|x.gz? }.sha512} zip: #{ary.find{|x|x.zip?}.sha512} - bz2: #{ary.find{|x|x.bz2?}&.sha512} xz: #{ary.find{|x|x.xz? }.sha512} eom - if ver.start_with?("3.") - entry = entry.gsub(/ bz2: .*\n/, "") - end - if data.include?("\n- version: #{ver}\n") + # update existing entry + data.sub!(/\n- version: #{ver}\n(^ .*\n)*\n/, "\n#{entry}\n") elsif data.sub!(/\n# #{Regexp.escape(xy)} series\n/, "\\&\n#{entry}") else data.sub!(/^$/, "\n# #{xy} series\n\n#{entry}") @@ -231,11 +218,11 @@ eom diff.each_with_index do |line, index| case index when 0 - line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do "--- a/#{filename}\t#{$2}" end when 1 - line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+ [\-+]\d+)\Z/) do + line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do "+++ b/#{filename}\t#{$2}" end end @@ -245,13 +232,14 @@ eom end def main - if ARGV.size != 2 - STDERR.puts "usage: format-release <dir-of-w.r-l.o> <version>" - exit - end wwwdir = ARGV.shift version = ARGV.shift - Tarball.parse(wwwdir, version) + rubydir = ARGV.shift + unless rubydir + STDERR.puts "usage: format-release <dir-of-w.r-l.o> <version> <ruby-dir>" + exit + end + Tarball.parse(wwwdir, version, rubydir) end main diff --git a/tool/gem-unpack.rb b/tool/gem-unpack.rb deleted file mode 100644 index cb05719463..0000000000 --- a/tool/gem-unpack.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'fileutils' -require 'rubygems' -require 'rubygems/package' - -# This library is used by "make extract-gems" to -# unpack bundled gem files. - -def Gem.unpack(file, dir = nil) - pkg = Gem::Package.new(file) - spec = pkg.spec - target = spec.full_name - target = File.join(dir, target) if dir - pkg.extract_files target - spec_file = File.join(target, "#{spec.name}-#{spec.version}.gemspec") - open(spec_file, 'wb') do |f| - f.print spec.to_ruby - end - puts "Unpacked #{file}" -end diff --git a/tool/gen-github-release.rb b/tool/gen-github-release.rb new file mode 100755 index 0000000000..cdb66080d9 --- /dev/null +++ b/tool/gen-github-release.rb @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby + +if ARGV.size < 2 + puts "Usage: #{$0} <from version tag> <to version tag> [--no-dry-run]" + puts " : if --no-dry-run is specified, it will create a release on GitHub" + exit 1 +end + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "octokit" + gem "faraday-retry" + gem "nokogiri" +end + +require "open-uri" + +Octokit.configure do |c| + c.access_token = ENV['GITHUB_TOKEN'] + c.auto_paginate = true + c.per_page = 100 +end + +client = Octokit::Client.new + +note = "## What's Changed\n\n" + +notes = [] + +diff = client.compare("ruby/ruby", ARGV[0], ARGV[1]) +diff[:commits].each do |c| + if c[:commit][:message] =~ /\[(Backport|Feature|Bug) #(\d*)\]/ + url = "https://bugs.ruby-lang.org/issues/#{$2}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ - Ruby master - Ruby Issue Tracking System/, "") + elsif c[:commit][:message] =~ /\(#(\d*)\)/ + url = "https://github.com/ruby/ruby/pull/#{$1}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ · ruby\/ruby · GitHub/, "") + else + next + end + notes << "* [#{title}](#{url})" +rescue OpenURI::HTTPError + puts "Error: #{url}" +end + +notes.uniq! + +note << notes.join("\n") + +note << "\n\n" +note << "Note: This list is automatically generated by tool/gen-github-release.rb. Because of this, some commits may be missing.\n\n" +note << "## Full Changelog\n\n" +note << "https://github.com/ruby/ruby/compare/#{ARGV[0]}...#{ARGV[1]}\n\n" + +if ARGV[2] == "--no-dry-run" + name = ARGV[1].gsub(/^v/, "").gsub(/_/, ".") + prerelease = ARGV[1].match?(/rc|preview/) ? true : false + client.create_release("ruby/ruby", ARGV[1], name: name, body: note, make_latest: "false", prerelease: prerelease) + puts "Created a release: https://github.com/ruby/ruby/releases/tag/#{ARGV[1]}" +else + puts note +end diff --git a/tool/gen-mailmap.rb b/tool/gen-mailmap.rb index 27b7abf8de..0cdedf1e7b 100755 --- a/tool/gen-mailmap.rb +++ b/tool/gen-mailmap.rb @@ -3,7 +3,7 @@ require "open-uri" require "yaml" -EMAIL_YML_URL = "https://cdn.jsdelivr.net/gh/ruby/ruby-commit-hook/config/email.yml" +EMAIL_YML_URL = "https://cdn.jsdelivr.net/gh/ruby/git.ruby-lang.org/config/email.yml" email_yml = URI(EMAIL_YML_URL).read.sub(/\A(?:#.*\n)+/, "").gsub(/^# +(.+)$/) { $1 + ": []" } @@ -13,7 +13,7 @@ YAML.load(DATA.read).each do |name, mails| email[name] |= mails end -open(File.join(__dir__, "../.mailmap"), "w") do |f| +File.open(File.join(__dir__, "../.mailmap"), "w") do |f| email.each do |name, mails| canonical = "#{ name }@ruby-lang.org" mails.delete(canonical) diff --git a/tool/generic_erb.rb b/tool/generic_erb.rb index 6af995fc13..b6cb51474e 100644 --- a/tool/generic_erb.rb +++ b/tool/generic_erb.rb @@ -5,57 +5,31 @@ require 'erb' require 'optparse' -require_relative 'lib/vpath' -require_relative 'lib/colorize' +require_relative 'lib/output' -vpath = VPath.new -timestamp = nil -output = nil -ifchange = nil +out = Output.new source = false -color = nil templates = [] ARGV.options do |o| - o.on('-t', '--timestamp[=PATH]') {|v| timestamp = v || true} - o.on('-i', '--input=PATH') {|v| template << v} - o.on('-o', '--output=PATH') {|v| output = v} - o.on('-c', '--[no-]if-change') {|v| ifchange = v} + o.on('-i', '--input=PATH') {|v| templates << v} o.on('-x', '--source') {source = true} - o.on('--color') {color = true} - vpath.def_options(o) + out.def_options(o) o.order!(ARGV) templates << (ARGV.shift or abort o.to_s) if templates.empty? end -color = Colorize.new(color) -unchanged = color.pass("unchanged") -updated = color.fail("updated") + +# Used in prelude.c.tmpl and unicode_norm_gen.tmpl +output = out.path +vpath = out.vpath + +# A hack to prevent "unused variable" warnings +output, vpath = output, vpath result = templates.map do |template| - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(File.read(template), trim_mode: '%-') - else - erb = ERB.new(File.read(template), nil, '%-') - end + erb = ERB.new(File.read(template), trim_mode: '%-') erb.filename = template source ? erb.src : proc{erb.result(binding)}.call end result = result.size == 1 ? result[0] : result.join("") -if output - if ifchange and (vpath.open(output, "rb") {|f| f.read} rescue nil) == result - puts "#{output} #{unchanged}" - else - open(output, "wb") {|f| f.print result} - puts "#{output} #{updated}" - end - if timestamp - if timestamp == true - dir, base = File.split(output) - timestamp = File.join(dir, ".time." + base) - end - File.open(timestamp, 'a') {} - File.utime(nil, nil, timestamp) - end -else - print result -end +out.write(result) diff --git a/tool/gperf.sed b/tool/gperf.sed index 6b3e1980be..9550a522b9 100644 --- a/tool/gperf.sed +++ b/tool/gperf.sed @@ -1,21 +1,3 @@ -/ANSI-C code/{ - h - s/.*/ANSI:offset:/ - x -} -/\/\*!ANSI{\*\//{ - G - s/\/\*!ANSI{\*\/\(.*\)\/\*}!ANSI\*\/\(.*\)\nANSI:.*/\/\*\1\*\/\2/ -} -s/(int)([a-z_]*)&((struct \([a-zA-Z_0-9][a-zA-Z_0-9]*\)_t *\*)0)->\1_str\([1-9][0-9]*\),/gperf_offsetof(\1, \2),/g -/^#line/{ - G - x - s/:offset:/:/ - x - s/\(.*\)\(\n\).*:offset:.*/#define gperf_offsetof(s, n) (short)offsetof(struct s##_t, s##_str##n)\2\1/ - s/\n[^#].*// -} /^[a-zA-Z_0-9]*hash/,/^}/{ s/ hval = / hval = (unsigned int)/ s/ return / return (unsigned int)/ diff --git a/tool/id2token.rb b/tool/id2token.rb index d12ea9c08e..cf73095842 100755 --- a/tool/id2token.rb +++ b/tool/id2token.rb @@ -5,18 +5,15 @@ BEGIN { require 'optparse' - require_relative 'lib/vpath' - vpath = VPath.new - header = nil opt = OptionParser.new do |o| - vpath.def_options(o) - header = o.order!(ARGV).shift + o.order!(ARGV) end or abort opt.opt_s TOKENS = {} - h = vpath.read(header) rescue abort("#{header} not found in #{vpath.inspect}") - h.scan(/^#define\s+RUBY_TOKEN_(\w+)\s+(\d+)/) do |token, id| + defs = File.join(File.dirname(File.dirname(__FILE__)), "defs/id.def") + ids = eval(File.read(defs), nil, defs) + ids[:token_op].each do |_id, _op, token, id| TOKENS[token] = id end diff --git a/tool/ifchange b/tool/ifchange index 5af41e0156..9e4a89533c 100755 --- a/tool/ifchange +++ b/tool/ifchange @@ -18,7 +18,7 @@ HELP set -e timestamp= keepsuffix= -empty= +srcavail=f color=auto until [ $# -eq 0 ]; do case "$1" in @@ -39,7 +39,7 @@ until [ $# -eq 0 ]; do keepsuffix=`expr \( "$1" : '[^=]*=\(.*\)' \)` ;; --empty) - empty=yes + srcavail=s ;; --color) color=always @@ -97,7 +97,7 @@ fi targetdir= case "$target" in */*) targetdir=`dirname "$target"`;; esac -if [ -f "$target" -a ! -${empty:+f}${empty:-s} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then +if [ -f "$target" -a ! -${srcavail} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then echo "$target ${msg_unchanged}unchanged${msg_reset}" rm -f "$temp" else diff --git a/tool/leaked-globals b/tool/leaked-globals index d95f3794e8..6118cd56e8 100755 --- a/tool/leaked-globals +++ b/tool/leaked-globals @@ -1,23 +1,33 @@ #!/usr/bin/ruby require_relative 'lib/colorize' +require 'shellwords' until ARGV.empty? case ARGV[0] - when /\ASYMBOL_PREFIX=(.*)/ + when /\A SYMBOL_PREFIX=(.*)/x SYMBOL_PREFIX = $1 - when /\ANM=(.*)/ # may be multiple words - NM = $1 - when /\APLATFORM=(.+)?/ + when /\A NM=(.*)/x # may be multiple words + NM = $1.shellsplit + when /\A PLATFORM=(.+)?/x platform = $1 + when /\A SOEXT=(.+)?/x + soext = $1 + when /\A SYMBOLS_IN_EMPTYLIB=(.*)/x + SYMBOLS_IN_EMPTYLIB = $1.split(" ") + when /\A EXTSTATIC=(.+)?/x + EXTSTATIC = true else break end ARGV.shift end +SYMBOLS_IN_EMPTYLIB ||= nil +EXTSTATIC ||= false config = ARGV.shift count = 0 col = Colorize.new + config_code = File.read(config) REPLACE = config_code.scan(/\bAC_(?:REPLACE|CHECK)_FUNCS?\((\w+)/).flatten # REPLACE << 'memcmp' if /\bAC_FUNC_MEMCMP\b/ =~ config_code @@ -29,27 +39,68 @@ if platform and !platform.empty? else REPLACE.concat( h .gsub(%r[/\*.*?\*/]m, " ") # delete block comments - .gsub(%r[//.*], ' ') # delete oneline comments + .gsub(%r[//.*], " ") # delete oneline comments .gsub(/^\s*#.*(?:\\\n.*)*/, "") # delete preprocessor directives + .gsub(/(?:\A|;)\K\s*typedef\s.*?;/m, "") .scan(/\b((?!rb_|DEPRECATED|_)\w+)\s*\(.*\);/) .flatten) end end missing = File.dirname(config) + "/missing/" ARGV.reject! do |n| - unless (src = Dir.glob(missing + File.basename(n, ".*") + ".[cS]")).empty? - puts "Ignore #{n} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" + base = File.basename(n, ".*") + next true if REPLACE.include?(base) + unless (src = Dir.glob(missing + base + ".[cS]")).empty? + puts "Ignore #{col.skip(n)} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" true end end + +# darwin's ld64 seems to require exception handling personality functions to be +# extern, so we allow the Rust one. +REPLACE.push("rust_eh_personality") if RUBY_PLATFORM.include?("darwin") + print "Checking leaked global symbols..." STDOUT.flush -IO.foreach("|#{NM} -Pgp #{ARGV.join(' ')}") do |line| +if soext + soext = /\.#{soext}(?:$|\.)/ + if EXTSTATIC + ARGV.delete_if {|n| soext =~ n} + elsif ARGV.size == 1 + so = soext =~ ARGV.first + end +end + +Pipe = Struct.new(:command) do + def open(&block) IO.popen(command, &block) end + def each(&block) open {|f| f.each(&block)} end +end + +Pipe.new(NM + ARGV).each do |line| + line.chomp! + next so = nil if line.empty? + if so.nil? and line.chomp!(":") + so = soext =~ line || false + next + end n, t, = line.split next unless /[A-TV-Z]/ =~ t next unless n.sub!(/^#{SYMBOL_PREFIX}/o, "") next if n.include?(".") - next if /\A(?:Init_|InitVM_|RUBY_|ruby_|rb_|[Oo]nig|dln_|mjit_|coroutine_)/ =~ n + next if !so and n.start_with?("___asan_") + next if !so and n.start_with?("__odr_asan_") + next if !so and n.start_with?("__retguard_") + next if !so and n.start_with?("__dtrace") + case n + when /\A(?:Init_|InitVM_|pm_|[Oo]nig|dln_|coroutine_)/ + next + when /\Aruby_static_id_/ + next unless so + when /\A(?:RUBY_|ruby_|rb_)/ + next unless so and /_(threadptr|ec)_/ =~ n + when *SYMBOLS_IN_EMPTYLIB + next + end next if REPLACE.include?(n) puts col.fail("leaked") if count.zero? count += 1 diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb new file mode 100644 index 0000000000..daa1a1f235 --- /dev/null +++ b/tool/lib/_tmpdir.rb @@ -0,0 +1,100 @@ +template = "rubytest." + +# This path is only for tests. +# Assume the directory by these environment variables are safe. +base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| + next unless tmp and tmp.size <= 50 and File.directory?(tmp) + # On macOS, the default TMPDIR is very long, in spite of UNIX socket + # path length being limited. + # + # Also Rubygems creates its own temporary directory per tests, and + # some tests copy the full path of gemhome there. In that case, the + # path contains both temporary names twice, and can exceed path name + # limit very easily. + tmp +end +begin + tmpdir = File.join(base, template + Random.new_seed.to_s(36)[-6..-1]) + Dir.mkdir(tmpdir, 0o700) +rescue Errno::EEXIST + retry +end +# warn "tmpdir(#{tmpdir.size}) = #{tmpdir}" + +pid = $$ +END { + if pid == $$ + begin + Dir.rmdir(tmpdir) + rescue Errno::ENOENT + rescue Errno::ENOTEMPTY + require_relative "colorize" + colorize = Colorize.new + ls = Struct.new(:colorize) do + def mode_inspect(m, s) + [ + (m & 0o4 == 0 ? ?- : ?r), + (m & 0o2 == 0 ? ?- : ?w), + (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)), + ] + end + def decorate_path(path, st) + case + when st.directory? + color = "bold;blue" + type = "/" + when st.symlink? + color = "bold;cyan" + # type = "@" + when st.executable? + color = "bold;green" + type = "*" + when path.end_with?(".gem") + color = "green" + end + colorize.decorate(path, color) + (type || "") + end + def list_tree(parent, indent = "", &block) + children = Dir.children(parent).map do |child| + [child, path = File.join(parent, child), File.lstat(path)] + end + nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size + size_width = children.map {|child, path, st| st.size}.max.to_s.size + + children.each do |child, path, st| + m = st.mode + m = [ + (st.file? ? ?- : st.ftype[0]), + mode_inspect(m >> 6, (?s unless m & 04000 == 0)), + mode_inspect(m >> 3, (?s unless m & 02000 == 0)), + mode_inspect(m, (?t unless m & 01000 == 0)), + ].join("") + warn sprintf("%s* %s %*d %*d %s % s%s", + indent, m, nlink_width, st.nlink, size_width, st.size, + st.mtime.to_s, decorate_path(child, st), + (" -> " + decorate_path(File.readlink(path), File.stat(path)) if + st.symlink?)) + if st.directory? + list_tree(File.join(parent, child), indent + " ", &block) + end + yield path, st if block + end + end + end.new(colorize) + warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":" + Dir.chdir(tmpdir) do + ls.list_tree(".") do |path, st| + if st.directory? + Dir.rmdir(path) + else + File.unlink(path) + end + end + end + require "fileutils" + FileUtils.rm_rf(tmpdir) + end + end +} + +ENV["TMPDIR"] = ENV["SPEC_TEMP_DIR"] = ENV["GEM_TEST_TMPDIR"] = tmpdir diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb new file mode 100644 index 0000000000..9ad5ea220b --- /dev/null +++ b/tool/lib/bundle_env.rb @@ -0,0 +1,4 @@ +ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_PATH__SYSTEM"] = "true" +ENV["BUNDLE_WITHOUT"] = "lint doc" diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb new file mode 100644 index 0000000000..d2ed61a508 --- /dev/null +++ b/tool/lib/bundled_gem.rb @@ -0,0 +1,126 @@ +require 'fileutils' +require 'rubygems' +require 'rubygems/package' + +# This library is used by "make extract-gems" to +# unpack bundled gem files. + +module BundledGem + DEFAULT_GEMS_DEPENDENCIES = [ + "net-protocol", # net-ftp + "time", # net-ftp + "singleton", # prime + "ipaddr", # rinda + "forwardable", # prime, rinda + "strscan", # rexml + "psych" # rdoc + ] + + module_function + + def unpack(file, *rest) + pkg = Gem::Package.new(file) + prepare_test(pkg.spec, *rest) do |dir| + pkg.extract_files(dir) + FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)}) + end + puts "Unpacked #{file}" + rescue Gem::Package::FormatError, Errno::ENOENT + puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." + if file =~ /^gems\/(\w+)-/ + file = Dir.glob("gems/#{$1}-*.gem").first + end + retry + end + + def build(gemspec, version, outdir = ".", validation: true) + outdir = File.expand_path(outdir) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.load(gemfile) + abort "Failed to load #{gemspec}" unless spec + output = File.join(outdir, spec.file_name) + FileUtils.rm_rf(output) + package = Gem::Package.new(output) + package.spec = spec + package.build(validation == false) + end + end + + def copy(path, *rest) + path, n = File.split(path) + spec = Dir.chdir(path) {Gem::Specification.load(n)} or raise "Cannot load #{path}" + prepare_test(spec, *rest) do |dir| + FileUtils.rm_rf(dir) + files = spec.files.reject {|f| f.start_with?(".git")} + dirs = files.map {|f| File.dirname(f) if f.include?("/")}.uniq + FileUtils.mkdir_p(dirs.map {|d| d ? "#{dir}/#{d}" : dir}.sort_by {|d| d.count("/")}) + files.each do |f| + File.copy_stream(File.join(path, f), File.join(dir, f)) + end + end + puts "Copied #{path}" + end + + def prepare_test(spec, dir = ".") + target = spec.full_name + Gem.ensure_gem_subdirectories(dir) + gem_dir = File.join(dir, "gems", target) + yield gem_dir + spec_dir = spec.extensions.empty? ? "specifications" : File.join("gems", target) + if spec.extensions.empty? + spec.dependencies.reject! {|dep| DEFAULT_GEMS_DEPENDENCIES.include?(dep.name)} + end + File.binwrite(File.join(dir, spec_dir, "#{target}.gemspec"), spec.to_ruby) + unless spec.extensions.empty? + spec.dependencies.clear + File.binwrite(File.join(dir, spec_dir, ".bundled.#{target}.gemspec"), spec.to_ruby) + end + if spec.bindir and spec.executables + bindir = File.join(dir, "bin") + Dir.mkdir(bindir) rescue nil + spec.executables.each do |exe| + File.open(File.join(bindir, exe), "wb", 0o777) {|f| + f.print "#!ruby\n", + %[load File.realpath("../gems/#{target}/#{spec.bindir}/#{exe}", __dir__)\n] + } + end + end + FileUtils.rm_rf(Dir.glob("#{gem_dir}/.git*")) + end + + def dummy_gemspec(gemspec) + return if File.exist?(gemspec) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.new do |s| + s.name = gemfile.chomp(".gemspec") + s.version = + File.read("lib/#{s.name}.rb")[/VERSION = "(.+?)"/, 1] || + begin File.read("lib/#{s.name}/version.rb")[/VERSION = "(.+?)"/, 1]; rescue; nil; end || + raise("cannot find the version of #{ s.name } gem") + s.authors = ["DUMMY"] + s.email = ["dummy@ruby-lang.org"] + s.files = Dir.glob("{lib,ext}/**/*").select {|f| File.file?(f)} + s.licenses = ["Ruby"] + s.description = "DO NOT USE; dummy gemspec only for test" + s.summary = "(dummy gemspec)" + end + File.write(gemfile, spec.to_ruby) + end + end + + def checkout(gemdir, repo, rev, git: $git) + return unless rev or !git or git.empty? + unless File.exist?("#{gemdir}/.git") + puts "Cloning #{repo}" + command = "#{git} clone #{repo} #{gemdir}" + system(command) or raise "failed: #{command}" + end + puts "Update #{File.basename(gemdir)} to #{rev}" + command = "#{git} fetch origin #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + command = "#{git} checkout --detach #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + end +end diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb index 11b878d318..0904312119 100644 --- a/tool/lib/colorize.rb +++ b/tool/lib/colorize.rb @@ -6,8 +6,8 @@ class Colorize # Colorize.new(color: color, colors_file: colors_file) def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color)) @colors = @reset = nil - @color = (opts[:color] if opts) - if color or (color == nil && STDOUT.tty?) + @color = opts && opts[:color] || color + if color or (color == nil && coloring?) if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil) @beg = "\e[" colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} @@ -27,21 +27,48 @@ class Colorize end DEFAULTS = { - "pass"=>"32", "fail"=>"31;1", "skip"=>"33;1", + # color names "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", "bold"=>"1", "underline"=>"4", "reverse"=>"7", + "bright_black"=>"90", "bright_red"=>"91", "bright_green"=>"92", "bright_yellow"=>"93", + "bright_blue"=>"94", "bright_magenta"=>"95", "bright_cyan"=>"96", "bright_white"=>"97", + + # abstract decorations + "pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold", + "note"=>"bright_yellow", "notice"=>"bright_yellow", "info"=>"bright_magenta", } + def coloring? + STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?) + end + # colorize.decorate(str, name = color_name) def decorate(str, name = @color) - if @colors and color = (@colors[name] || DEFAULTS[name]) + if coloring? and color = resolve_color(name) "#{@beg}#{color}m#{str}#{@reset}" else str end end + def resolve_color(color = @color, seen = {}, colors = nil) + return unless @colors + color.to_s.gsub(/\b[a-z][\w ]+/) do |n| + n.gsub!(/\W+/, "_") + n.downcase! + c = seen[n] and next c + if colors + c = colors[n] + elsif (c = (tbl = @colors)[n] || (tbl = DEFAULTS)[n]) + colors = tbl + else + next n + end + seen[n] = resolve_color(c, seen, colors) + end + end + DEFAULTS.each_key do |name| define_method(name) {|str| decorate(str, name) diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 9530187db4..e29a0e3c25 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -1,8 +1,49 @@ # frozen_string_literal: true module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + module Unit module Assertions + def assert_raises(*exp, &b) + raise NoMethodError, "use assert_raise", caller + end + def _assertions= n # :nodoc: @_assertions = n end @@ -16,9 +57,16 @@ module Test def message msg = nil, ending = nil, &default proc { - msg = msg.call.chomp(".") if Proc === msg - custom_message = "#{msg}.\n" unless msg.nil? or msg.to_s.empty? - "#{custom_message}#{default.call}#{ending || "."}" + ending ||= (ending_pattern = /(?<!\.)\z/; ".") + ending_pattern ||= /(?<!#{Regexp.quote(ending)})\z/ + msg = msg.call if Proc === msg + ary = [msg, (default.call if default)].compact.reject(&:empty?) + ary.map! {|str| str.to_s.sub(ending_pattern, ending) } + begin + ary.join("\n") + rescue Encoding::CompatibilityError + ary.map(&:b).join("\n") + end } end end @@ -26,6 +74,21 @@ module Test module CoreAssertions require_relative 'envutil' require 'pp' + begin + require '-test-/sanitizers' + rescue LoadError + # in test-unit-ruby-core gem + def sanitizers + nil + end + else + def sanitizers + Test::Sanitizers + end + end + module_function :sanitizers + + nil.pretty_inspect def mu_pp(obj) #:nodoc: obj.pretty_inspect.chomp @@ -43,11 +106,14 @@ module Test end def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, - success: nil, **opt) + success: nil, failed: nil, gems: false, **opt) args = Array(args).dup - args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') + unless gems.nil? + args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems") + end stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) - desc = FailDesc[status, message, stderr] + desc = failed[status, message, stderr] if failed + desc ||= FailDesc[status, message, stderr] if block_given? raise "test_stdout ignored, use block only or without block" if test_stdout != [] raise "test_stderr ignored, use block only or without block" if test_stderr != [] @@ -99,8 +165,13 @@ module Test end def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) - # TODO: consider choosing some appropriate limit for MJIT and stop skipping this once it does not randomly fail - pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::JIT) && RubyVM::JIT.enabled? + # TODO: consider choosing some appropriate limit for RJIT and stop skipping this once it does not randomly fail + pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + # For previous versions which implemented MJIT + pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + # ASAN has the same problem - its shadow memory greatly increases memory usage + # (plus asan has better ways to detect memory leaks than this assertion) + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled? require_relative 'memory_status' raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) @@ -165,27 +236,15 @@ module Test msg = args.pop end begin - line = __LINE__; yield - rescue Test::Unit::PendedError + yield + rescue Test::Unit::PendedError, *(Test::Unit::AssertionFailedError if args.empty?) raise - rescue Exception => e - bt = e.backtrace - as = e.instance_of?(Test::Unit::AssertionFailedError) - if as - ans = /\A#{Regexp.quote(__FILE__)}:#{line}:in /o - bt.reject! {|ln| ans =~ ln} - end - if ((args.empty? && !as) || - args.any? {|a| a.instance_of?(Module) ? e.is_a?(a) : e.class == a }) - msg = message(msg) { - "Exception raised:\n<#{mu_pp(e)}>\n" + - "Backtrace:\n" + - e.backtrace.map{|frame| " #{frame}"}.join("\n") - } - raise Test::Unit::AssertionFailedError, msg.call, bt - else - raise - end + rescue *(args.empty? ? Exception : args) => e + msg = message(msg) { + "Exception raised:\n<#{mu_pp(e)}>\n""Backtrace:\n" << + Test.filter_backtrace(e.backtrace).map{|frame| " #{frame}"}.join("\n") + } + raise Test::Unit::AssertionFailedError, msg.call, e.backtrace end end @@ -244,11 +303,40 @@ module Test def separated_runner(token, out = nil) include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + out = out ? IO.new(out, 'w') : STDOUT + + # avoid method redefinitions + out_write = out.method(:write) + integer_to_s = Integer.instance_method(:to_s) + array_pack = Array.instance_method(:pack) + marshal_dump = Marshal.method(:dump) + assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set) + assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get) + Test::Unit::Assertions.module_eval do + @_assertions = 0 + + undef _assertions= + define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)}) + + undef _assertions + define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)}) + end + # assume Method#call and UnboundMethod#bind_call need to work as the original + at_exit { - out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" + assertions = assertions_ivar_get.call(:@_assertions) + out_write.call <<~OUT + <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}> + #{array_pack.bind_call([marshal_dump.call($!)], 'm0')} + </error id="#{token}"> + OUT } - Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) if defined?(Test::Unit::Runner) + if defined?(Test::Unit::Runner) + Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) + elsif defined?(Test::Unit::AutoRunner) + Test::Unit::AutoRunner.need_auto_run = false + end end def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **opt) @@ -258,7 +346,7 @@ module Test line ||= loc.lineno end capture_stdout = true - unless /mswin|mingw/ =~ RUBY_PLATFORM + unless /mswin|mingw/ =~ RbConfig::CONFIG['host_os'] capture_stdout = false opt[:out] = Test::Unit::Runner.output if defined?(Test::Unit::Runner) res_p, res_c = IO.pipe @@ -268,14 +356,24 @@ module Test src = <<eom # -*- coding: #{line += __LINE__; src.encoding}; -*- BEGIN { - require "test/unit";include Test::Unit::Assertions;include Test::Unit::CoreAssertions;require #{__FILE__.dump} + require "test/unit";include Test::Unit::Assertions;require #{__FILE__.dump};include Test::Unit::CoreAssertions separated_runner #{token_dump}, #{res_c&.fileno || 'nil'} } #{line -= __LINE__; src} eom args = args.dup args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) + args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + # power_assert 3 requires ruby 3.1 or later + args << "-W:no-experimental" if RUBY_VERSION < "3.1." stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + + if sanitizers&.lsan_enabled? + # LSAN may output messages like the following line into stderr. We should ignore it. + # ==276855==Running thread 276851 was not suspended. False leaks are possible. + # See https://github.com/google/sanitizers/issues/1479 + stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "") + end ensure if res_c res_c.close @@ -286,15 +384,16 @@ eom end raise if $! abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + marshal_error = nil assert(!abort, FailDesc[status, nil, stderr]) - self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i - begin - res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m")) + res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do + self._assertions += $1.to_i + res = Marshal.load($2.unpack1("m")) or next rescue => marshal_error ignore_stderr = nil res = nil - end - if res and !(SystemExit === res) + else + next if SystemExit === res if bt = res.backtrace bt.each do |l| l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} @@ -306,7 +405,7 @@ eom raise res end - # really is it succeed? + # really did it succeed? unless ignore_stderr # the body of assert_separately must not output anything to detect error assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) @@ -317,9 +416,17 @@ eom # Run Ractor-related test without influencing the main test suite def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) - return unless defined?(Ractor) + omit unless defined?(Ractor) + + # https://bugs.ruby-lang.org/issues/21262 + shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value) + shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join) + + if require + require = [require] unless require.is_a?(Array) + require = require.map {|r| "require #{r.inspect}"}.join("\n") + end - require = "require #{require.inspect}" if require if require_relative dir = File.dirname(caller_locations[0,1][0].absolute_path) full_path = File.expand_path(require_relative, dir) @@ -327,6 +434,8 @@ eom end assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{shim_value} + #{shim_join} #{require} previous_verbose = $VERBOSE $VERBOSE = nil @@ -437,19 +546,15 @@ eom case expected when String assert = :assert_equal - when Regexp - assert = :assert_match else - raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" + assert_respond_to(expected, :===) + assert = :assert_match end - ex = m = nil - EnvUtil.with_default_internal(expected.encoding) do - ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do - yield - end - m = ex.message + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield end + m = ex.message msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} if assert == :assert_equal @@ -462,6 +567,43 @@ eom ex end + # :call-seq: + # assert_raise_kind_of(*args, &block) + # + #Tests if the given block raises one of the given exceptions or + #sub exceptions of the given exceptions. If the last argument + #is a String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise SystemCallErr do + # Dir.chdir(__FILE__) #Raises Errno::ENOTDIR, so assertion succeeds + # end + def assert_raise_kind_of(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + raise e unless exp.include? Test::Unit::PendedError + rescue *exp => e + pass + rescue Exception => e + flunk(message(msg) {"#{mu_pp(exp)} family exception expected, not #{mu_pp(e)}"}) + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} family expected but nothing was raised"}) + end + end + e + end + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: # :call-seq: @@ -534,11 +676,11 @@ eom refute_respond_to(obj, meth, msg) end - # pattern_list is an array which contains regexp and :*. + # pattern_list is an array which contains regexp, string and :*. # :* means any sequence. # # pattern_list is anchored. - # Use [:*, regexp, :*] for non-anchored match. + # Use [:*, regexp/string, :*] for non-anchored match. def assert_pattern_list(pattern_list, actual, message=nil) rest = actual anchored = true @@ -547,11 +689,13 @@ eom anchored = false else if anchored - match = /\A#{pattern}/.match(rest) + match = rest.rindex(pattern, 0) else - match = pattern.match(rest) + match = rest.index(pattern) end - unless match + if match + post_match = $~ ? $~.post_match : rest[match+pattern.size..-1] + else msg = message(msg) { expect_msg = "Expected #{mu_pp pattern}\n" if /\n[^\n]/ =~ rest @@ -568,7 +712,7 @@ eom } assert false, msg end - rest = match.post_match + rest = post_match anchored = true end } @@ -579,7 +723,7 @@ eom def assert_warning(pat, msg = nil) result = nil - stderr = EnvUtil.with_default_internal(pat.encoding) { + stderr = EnvUtil.with_default_internal(of: pat) { EnvUtil.verbose_warning { result = yield } @@ -593,17 +737,15 @@ eom assert_warning(*args) {$VERBOSE = false; yield} end - def assert_deprecated_warning(mesg = /deprecated/) + def assert_deprecated_warning(mesg = /deprecated/, &block) assert_warning(mesg) do - Warning[:deprecated] = true - yield + EnvUtil.deprecation_warning(&block) end end - def assert_deprecated_warn(mesg = /deprecated/) + def assert_deprecated_warn(mesg = /deprecated/, &block) assert_warn(mesg) do - Warning[:deprecated] = true - yield + EnvUtil.deprecation_warning(&block) end end @@ -640,7 +782,7 @@ eom def for(key) @count += 1 - yield + yield key rescue Exception => e @failures[key] = [@count, e] end @@ -694,7 +836,7 @@ eom msg = "exceptions on #{errs.length} threads:\n" + errs.map {|t, err| "#{t.inspect}:\n" + - RUBY_VERSION >= "2.5.0" ? err.full_message(highlight: false, order: :top) : err.message + (err.respond_to?(:full_message) ? err.full_message(highlight: false, order: :top) : err.message) }.join("\n---\n") if message msg = "#{message}\n#{msg}" @@ -729,21 +871,64 @@ eom end alias all_assertions_foreach assert_all_assertions_foreach - def message(msg = nil, *args, &default) # :nodoc: - if Proc === msg - super(nil, *args) do - ary = [msg.call, (default.call if default)].compact.reject(&:empty?) - if 1 < ary.length - ary[0...-1] = ary[0...-1].map {|str| str.sub(/(?<!\.)\z/, '.') } - end + %w[ + CLOCK_THREAD_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID + CLOCK_MONOTONIC + ].find do |c| + if Process.const_defined?(c) + [c.to_sym, Process.const_get(c)].find do |clk| begin - ary.join("\n") - rescue Encoding::CompatibilityError - ary.map(&:b).join("\n") + Process.clock_gettime(clk) + rescue + # Constants may be defined but not implemented, e.g., mingw. + else + unless Process.clock_getres(clk) < 1.0e-03 + next # needs msec precision + end + PERFORMANCE_CLOCK = clk end end - else - super + end + end + + # Expect +seq+ to respond to +first+ and +each+ methods, e.g., + # Array, Range, Enumerator::ArithmeticSequence and other + # Enumerable-s, and each elements should be size factors. + # + # :yield: each elements of +seq+. + def assert_linear_performance(seq, rehearsal: nil, pre: ->(n) {n}) + pend "No PERFORMANCE_CLOCK found" unless defined?(PERFORMANCE_CLOCK) + + # Timeout testing generally doesn't work when RJIT compilation happens. + rjit_enabled = defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + measure = proc do |arg, message| + st = Process.clock_gettime(PERFORMANCE_CLOCK) + yield(*arg) + t = (Process.clock_gettime(PERFORMANCE_CLOCK) - st) + assert_operator 0, :<=, t, message unless rjit_enabled + t + end + + first = seq.first + *arg = pre.call(first) + times = (0..(rehearsal || (2 * first))).map do + measure[arg, "rehearsal"].nonzero? + end + times.compact! + tmin, tmax = times.minmax + + # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) + tbase = 10 * tmax * [(tmax / tmin) ** 2 / 4, 1].max + info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" + + seq.each do |i| + next if i == first + t = tbase * i.fdiv(first) + *arg = pre.call(i) + message = "[#{i}]: in #{t}s #{info}" + Timeout.timeout(t, Timeout::Error, message) do + measure[arg, message] + end end end @@ -767,6 +952,82 @@ eom token = "\e[7;1m#{$$.to_s}:#{Time.now.strftime('%s.%L')}:#{rand(0x10000).to_s(16)}:\e[m" return token.dump, Regexp.quote(token) end + + # Platform predicates + + def self.mswin? + defined?(@mswin) ? @mswin : @mswin = RUBY_PLATFORM.include?('mswin') + end + private def mswin? + CoreAssertions.mswin? + end + + def self.mingw? + defined?(@mingw) ? @mingw : @mingw = RUBY_PLATFORM.include?('mingw') + end + private def mingw? + CoreAssertions.mingw? + end + + module_function def windows? + mswin? or mingw? + end + + def self.version_compare(expected, actual) + expected.zip(actual).each {|e, a| z = (e <=> a); return z if z.nonzero?} + 0 + end + + def self.version_match?(expected, actual) + if !actual + false + elsif expected.empty? + true + elsif expected.size == 1 and Range === (range = expected.first) + b, e = range.begin, range.end + return false if b and (c = version_compare(Array(b), actual)) > 0 + return false if e and (c = version_compare(Array(e), actual)) < 0 + return false if e and range.exclude_end? and c == 0 + true + else + version_compare(expected, actual).zero? + end + end + + def self.linux?(*ver) + unless defined?(@linux) + @linux = RUBY_PLATFORM.include?('linux') && `uname -r`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @linux + end + private def linux?(*ver) + CoreAssertions.linux?(*ver) + end + + def self.glibc?(*ver) + unless defined?(@glibc) + libc = `/usr/bin/ldd /bin/sh`[/^\s*libc.*=> *\K\S*/] + if libc and /version (\d+)\.(\d+)\.$/ =~ IO.popen([libc], &:read)[] + @glibc = [$1.to_i, $2.to_i] + else + @glibc = false + end + end + version_match? ver, @glibc + end + private def glibc?(*ver) + CoreAssertions.glibc?(*ver) + end + + def self.macos?(*ver) + unless defined?(@macos) + @macos = RUBY_PLATFORM.include?('darwin') && `sw_vers -productVersion`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @macos + end + private def macos?(*ver) + CoreAssertions.macos?(*ver) + end end end end diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb new file mode 100644 index 0000000000..56b420a546 --- /dev/null +++ b/tool/lib/dump.gdb @@ -0,0 +1,17 @@ +set height 0 +set width 0 +set confirm off + +echo \n>>> Threads\n\n +info threads + +echo \n>>> Machine level backtrace\n\n +thread apply all info stack full + +echo \n>>> Dump Ruby level backtrace (if possible)\n\n +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +echo ">>> Finish\n" +detach +quit diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb new file mode 100644 index 0000000000..ed9cb89010 --- /dev/null +++ b/tool/lib/dump.lldb @@ -0,0 +1,13 @@ +script print("\n>>> Threads\n\n") +thread list + +script print("\n>>> Machine level backtrace\n\n") +thread backtrace all + +script print("\n>>> Dump Ruby level backtrace (if possible)\n\n") +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +script print(">>> Finish\n") +detach +quit diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index 0391b90c1c..6089605056 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -15,23 +15,22 @@ end module EnvUtil def rubybin if ruby = ENV["RUBY"] - return ruby - end - ruby = "ruby" - exeext = RbConfig::CONFIG["EXEEXT"] - rubyexe = (ruby + exeext if exeext and !exeext.empty?) - 3.times do - if File.exist? ruby and File.executable? ruby and !File.directory? ruby - return File.expand_path(ruby) - end - if rubyexe and File.exist? rubyexe and File.executable? rubyexe - return File.expand_path(rubyexe) - end - ruby = File.join("..", ruby) - end - if defined?(RbConfig.ruby) + ruby + elsif defined?(RbConfig.ruby) RbConfig.ruby else + ruby = "ruby" + exeext = RbConfig::CONFIG["EXEEXT"] + rubyexe = (ruby + exeext if exeext and !exeext.empty?) + 3.times do + if File.exist? ruby and File.executable? ruby and !File.directory? ruby + return File.expand_path(ruby) + end + if rubyexe and File.exist? rubyexe and File.executable? rubyexe + return File.expand_path(rubyexe) + end + ruby = File.join("..", ruby) + end "ruby" end end @@ -53,7 +52,14 @@ module EnvUtil @original_internal_encoding = Encoding.default_internal @original_external_encoding = Encoding.default_external @original_verbose = $VERBOSE - @original_warning = defined?(Warning.[]) ? %i[deprecated experimental].to_h {|i| [i, Warning[i]]} : nil + @original_warning = + if defined?(Warning.categories) + Warning.categories.to_h {|i| [i, Warning[i]]} + elsif defined?(Warning.[]) # 2.7+ + %i[deprecated experimental performance].to_h do |i| + [i, begin Warning[i]; rescue ArgumentError; end] + end.compact + end end end @@ -73,6 +79,72 @@ module EnvUtil end module_function :timeout + class Debugger + @list = [] + + attr_accessor :name + + def self.register(name, &block) + @list << new(name, &block) + end + + def initialize(name, &block) + @name = name + instance_eval(&block) + end + + def usable?; false; end + + def start(pid, *args) end + + def dump(pid, timeout: 60, reprieve: timeout&.div(4)) + dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err) + rescue Errno::ENOENT + return + else + return unless dpid + [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig| + begin + return EnvUtil.timeout(t) {Process.wait(dpid)} + rescue Timeout::Error + Process.kill(sig, dpid) + end + end + true + end + + # sudo -n: --non-interactive + PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM) + + def spawn(*args, **opts) + super(*PRECOMMAND, *args, **opts) + end + + register("gdb") do + class << self + def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end + def start(pid, *args, **opts) + spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts) + end + def command_file(file) "--command=#{file}"; end + end + end + + register("lldb") do + class << self + def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end + def start(pid, *args, **opts) + spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts) + end + def command_file(file) ["--source", file]; end + end + end + + def self.search + @debugger ||= @list.find(&:usable?) + end + end + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) reprieve = apply_timeout_scale(reprieve) if reprieve @@ -88,17 +160,12 @@ module EnvUtil pgroup = pid end - lldb = true if /darwin/ =~ RUBY_PLATFORM - + dumped = false while signal = signals.shift - if lldb and [:ABRT, :KILL].include?(signal) - lldb = false - # sudo -n: --non-interactive - # lldb -p: attach - # -o: run command - system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit]) - true + if !dumped and [:ABRT, :KILL].include?(signal) + Debugger.search&.dump(pid) + dumped = true end begin @@ -152,8 +219,18 @@ module EnvUtil if RUBYLIB and lib = child_env["RUBYLIB"] child_env["RUBYLIB"] = [lib, RUBYLIB].join(File::PATH_SEPARATOR) end - child_env['ASAN_OPTIONS'] = ENV['ASAN_OPTIONS'] if ENV['ASAN_OPTIONS'] + + # remain env + %w(ASAN_OPTIONS RUBY_ON_BUG).each{|name| + child_env[name] = ENV[name] if !child_env.key?(name) and ENV.key?(name) + } + args = [args] if args.kind_of?(String) + # use the same parser as current ruby + if (args.none? { |arg| arg.start_with?("--parser=") } and + /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read)) + args = ["--parser=#{current_parser}"] + args + end pid = spawn(child_env, *precommand, rubybin, *args, opt) in_c.close out_c&.close @@ -201,6 +278,12 @@ module EnvUtil end module_function :invoke_ruby + def current_parser + features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}] + features&.split&.include?("+PRISM") ? "prism" : "parse.y" + end + module_function :current_parser + def verbose_warning class << (stderr = "".dup) alias write concat @@ -217,6 +300,21 @@ module EnvUtil end module_function :verbose_warning + if defined?(Warning.[]=) + def deprecation_warning + previous_deprecated = Warning[:deprecated] + Warning[:deprecated] = true + yield + ensure + Warning[:deprecated] = previous_deprecated + end + else + def deprecation_warning + yield + end + end + module_function :deprecation_warning + def default_warning $VERBOSE = false yield @@ -241,7 +339,30 @@ module EnvUtil end module_function :under_gc_stress - def with_default_external(enc) + def under_gc_compact_stress(val = :empty, &block) + raise "compaction doesn't work well on s390x. Omit the test in the caller." if RUBY_PLATFORM =~ /s390x/ # https://github.com/ruby/ruby/pull/5077 + + if GC.respond_to?(:auto_compact) + auto_compact = GC.auto_compact + GC.auto_compact = val + end + + under_gc_stress(&block) + ensure + GC.auto_compact = auto_compact if GC.respond_to?(:auto_compact) + end + module_function :under_gc_compact_stress + + def without_gc + prev_disabled = GC.disable + yield + ensure + GC.enable unless prev_disabled + end + module_function :without_gc + + def with_default_external(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_external = enc } yield ensure @@ -249,7 +370,8 @@ module EnvUtil end module_function :with_default_external - def with_default_internal(enc) + def with_default_internal(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_internal = enc } yield ensure @@ -292,16 +414,24 @@ module EnvUtil cmd = @ruby_install_name if "ruby-runner#{RbConfig::CONFIG["EXEEXT"]}" == cmd path = DIAGNOSTIC_REPORTS_PATH timeformat = DIAGNOSTIC_REPORTS_TIMEFORMAT - pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.crash" + pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.{crash,ips}" first = true 30.times do first ? (first = false) : sleep(0.1) Dir.glob(pat) do |name| log = File.read(name) rescue next - if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log - File.unlink(name) - File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil - return log + case name + when /\.crash\z/ + if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log + File.unlink(name) + File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil + return log + end + when /\.ips\z/ + if /^ *"pid" *: *#{pid},/ =~ log + File.unlink(name) + return log + end end end end diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb new file mode 100644 index 0000000000..1893e07657 --- /dev/null +++ b/tool/lib/gem_env.rb @@ -0,0 +1 @@ +ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__) diff --git a/tool/lib/iseq_loader_checker.rb b/tool/lib/iseq_loader_checker.rb index 3f07b3a999..73784f8450 100644 --- a/tool/lib/iseq_loader_checker.rb +++ b/tool/lib/iseq_loader_checker.rb @@ -76,6 +76,15 @@ class RubyVM::InstructionSequence # return value i2_bin if CHECK_TO_BINARY end if CHECK_TO_A || CHECK_TO_BINARY + + if opt == "prism" + # If RUBY_ISEQ_DUMP_DEBUG is "prism", we'll set up + # InstructionSequence.load_iseq to intercept loading filepaths to compile + # using prism. + def self.load_iseq(filepath) + RubyVM::InstructionSequence.compile_file_prism(filepath) + end + end end #require_relative 'x'; exit(1) diff --git a/tool/lib/launchable.rb b/tool/lib/launchable.rb new file mode 100644 index 0000000000..38f4fe92b3 --- /dev/null +++ b/tool/lib/launchable.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +require 'json' +require 'uri' + +module Launchable + ## + # JsonStreamWriter writes a JSON file using a stream. + # By utilizing a stream, we can minimize memory usage, especially for large files. + class JsonStreamWriter + def initialize(path) + @file = File.open(path, "w") + @file.write("{") + @indent_level = 0 + @is_first_key_val = true + @is_first_obj = true + write_new_line + end + + def write_object obj + if @is_first_obj + @is_first_obj = false + else + write_comma + write_new_line + end + @indent_level += 1 + @file.write(to_json_str(obj)) + @indent_level -= 1 + @is_first_key_val = true + # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified. + # { + # "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds", + # "status": "TEST_PASSED", + # "duration": 2.7e-05, + # "createdAt": "2024-02-09 12:21:07 +0000", + # "stderr": null, + # "stdout": null + # }: null <- here + # }, + # To prevent this, IO#flush is called here. + @file.flush + end + + def write_array(key) + @indent_level += 1 + @file.write(to_json_str(key)) + write_colon + @file.write(" ", "[") + write_new_line + end + + def close + return if @file.closed? + close_array + @indent_level -= 1 + write_new_line + @file.write("}", "\n") + @file.flush + @file.close + end + + private + def to_json_str(obj) + json = JSON.pretty_generate(obj) + json.gsub(/^/, ' ' * (2 * @indent_level)) + end + + def write_indent + @file.write(" " * 2 * @indent_level) + end + + def write_new_line + @file.write("\n") + end + + def write_comma + @file.write(',') + end + + def write_colon + @file.write(":") + end + + def close_array + write_new_line + write_indent + @file.write("]") + @indent_level -= 1 + end + end +end diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb index 052bea5e53..69df9a64b8 100644 --- a/tool/lib/leakchecker.rb +++ b/tool/lib/leakchecker.rb @@ -112,7 +112,7 @@ class LeakChecker } unless fd_leaked.empty? unless @@try_lsof == false - @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + @@try_lsof |= system(*%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) end end h.each {|fd, list| @@ -136,14 +136,14 @@ class LeakChecker attr_accessor :count end - def new(data) + def new(...) LeakChecker::TempfileCounter.count += 1 - super(data) + super end } LeakChecker.const_set(:TempfileCounter, m) - class << Tempfile::Remover + class << Tempfile prepend LeakChecker::TempfileCounter end end @@ -155,8 +155,8 @@ class LeakChecker if prev_count == count [prev_count, []] else - tempfiles = ObjectSpace.each_object(Tempfile).find_all {|t| - t.instance_variable_defined?(:@tmpfile) and t.path + tempfiles = ObjectSpace.each_object(Tempfile).reject {|t| + t.instance_variables.empty? || t.closed? } [count, tempfiles] end @@ -182,7 +182,8 @@ class LeakChecker def find_threads Thread.list.find_all {|t| - t != Thread.current && t.alive? + t != Thread.current && t.alive? && + !(t.thread_variable?(:"\0__detached_thread__") && t.thread_variable_get(:"\0__detached_thread__")) } end @@ -209,15 +210,36 @@ class LeakChecker return leaked end - def find_env - ENV.to_h + e = ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] + begin + ENV["_Ruby_Env_Ignorecase_"] = ENV["_RUBY_ENV_IGNORECASE_"] = nil + ENV["_RUBY_ENV_IGNORECASE_"] = "ENV_CASE_TEST" + ENV_IGNORECASE = ENV["_Ruby_Env_Ignorecase_"] == "ENV_CASE_TEST" + ensure + ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] = e + end + + if ENV_IGNORECASE + def find_env + ENV.to_h {|k, v| [k.upcase, v]} + end + else + def find_env + ENV.to_h + end end def check_env(test_name) old_env = @env_info - new_env = ENV.to_h + new_env = find_env return false if old_env == new_env + if defined?(Bundler::EnvironmentPreserver) + bundler_prefix = Bundler::EnvironmentPreserver::BUNDLER_PREFIX + end (old_env.keys | new_env.keys).sort.each {|k| + # Don't report changed environment variables caused by Bundler's backups + next if bundler_prefix and k.start_with?(bundler_prefix) + if old_env.has_key?(k) if new_env.has_key?(k) if old_env[k] != new_env[k] diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb index 5e9e80a68a..429e5f6a1d 100644 --- a/tool/lib/memory_status.rb +++ b/tool/lib/memory_status.rb @@ -12,7 +12,7 @@ module Memory PROC_FILE = procfile VM_PAT = pat def self.read_status - IO.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| + File.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| yield($1.downcase.intern, $2.to_i * 1024) if VM_PAT =~ l end end @@ -20,48 +20,68 @@ module Memory data.scan(pat) {|k, v| keys << k.downcase.intern} when /mswin|mingw/ =~ RUBY_PLATFORM - require 'fiddle/import' - require 'fiddle/types' - - module Win32 - extend Fiddle::Importer - dlload "kernel32.dll", "psapi.dll" - include Fiddle::Win32Types - typealias "SIZE_T", "size_t" - - PROCESS_MEMORY_COUNTERS = struct [ - "DWORD cb", - "DWORD PageFaultCount", - "SIZE_T PeakWorkingSetSize", - "SIZE_T WorkingSetSize", - "SIZE_T QuotaPeakPagedPoolUsage", - "SIZE_T QuotaPagedPoolUsage", - "SIZE_T QuotaPeakNonPagedPoolUsage", - "SIZE_T QuotaNonPagedPoolUsage", - "SIZE_T PagefileUsage", - "SIZE_T PeakPagefileUsage", - ] - - typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" - - extern "HANDLE GetCurrentProcess()", :stdcall - extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall - - module_function - def memory_info - size = PROCESS_MEMORY_COUNTERS.size - data = PROCESS_MEMORY_COUNTERS.malloc - data.cb = size - data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + keys.push(:size, :rss, :peak) + + begin + require 'fiddle/import' + require 'fiddle/types' + rescue LoadError + # Fallback to PowerShell command to get memory information for current process + def self.read_status + cmd = [ + "powershell.exe", "-NoProfile", "-Command", + "Get-Process -Id #{$$} | " \ + "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }" + ] + + IO.popen(cmd, "r", err: [:child, :out]) do |out| + if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read + yield :size, $1.to_i + yield :rss, $2.to_i + yield :peak, $3.to_i + end + end + end + else + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end end - end - keys.push(:size, :rss, :peak) - def self.read_status - if info = Win32.memory_info - yield :size, info.PagefileUsage - yield :rss, info.WorkingSetSize - yield :peak, info.PeakWorkingSetSize + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end end end when (require_relative 'find_executable' diff --git a/tool/lib/output.rb b/tool/lib/output.rb new file mode 100644 index 0000000000..8cb426ae4a --- /dev/null +++ b/tool/lib/output.rb @@ -0,0 +1,70 @@ +require_relative 'vpath' +require_relative 'colorize' + +class Output + attr_reader :path, :vpath + + def initialize(path: nil, timestamp: nil, ifchange: nil, color: nil, + overwrite: false, create_only: false, vpath: VPath.new) + @path = path + @timestamp = timestamp + @ifchange = ifchange + @color = color + @overwrite = overwrite + @create_only = create_only + @vpath = vpath + end + + COLOR_WHEN = { + 'always' => true, 'auto' => nil, 'never' => false, + nil => true, false => false, + } + + def def_options(opt) + opt.separator(" Output common options:") + opt.on('-o', '--output=PATH') {|v| @path = v} + opt.on('-t', '--timestamp[=PATH]') {|v| @timestamp = v || true} + opt.on('-c', '--[no-]if-change') {|v| @ifchange = v} + opt.on('--[no-]color=[WHEN]', COLOR_WHEN.keys) {|v| @color = COLOR_WHEN[v]} + opt.on('--[no-]create-only') {|v| @create_only = v} + opt.on('--[no-]overwrite') {|v| @overwrite = v} + @vpath.def_options(opt) + end + + def write(data, overwrite: @overwrite, create_only: @create_only) + unless @path + $stdout.print data + return true + end + color = Colorize.new(@color) + unchanged = color.pass("unchanged") + updated = color.fail("updated") + outpath = nil + + if (@ifchange or overwrite or create_only) and (@vpath.open(@path, "rb") {|f| + outpath = f.path + if @ifchange or create_only + original = f.read + (@ifchange and original == data) or (create_only and !original.empty?) + end + } rescue false) + puts "#{outpath} #{unchanged}" + written = false + else + unless overwrite and outpath and (File.binwrite(outpath, data) rescue nil) + File.binwrite(outpath = @path, data) + end + puts "#{outpath} #{updated}" + written = true + end + if timestamp = @timestamp + if timestamp == true + dir, base = File.split(@path) + timestamp = File.join(dir, ".time." + base) + end + File.binwrite(timestamp, '') + File.utime(nil, nil, timestamp) + end + written + end +end diff --git a/tool/lib/path.rb b/tool/lib/path.rb new file mode 100644 index 0000000000..f16a164338 --- /dev/null +++ b/tool/lib/path.rb @@ -0,0 +1,101 @@ +module Path + module_function + + def clean(path) + path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') + nil while path.sub!(/[^\/]+\/\.\.\//, '') + path + end + + def relative(path, base) + path = clean(path) + base = clean(base) + path, base = [path, base].map{|s|s.split("/")} + until path.empty? or base.empty? or path[0] != base[0] + path.shift + base.shift + end + path, base = [path, base].map{|s|s.join("/")} + if base.empty? + path + elsif base.start_with?("../") or File.absolute_path?(base) + File.expand_path(path) + else + base.gsub!(/[^\/]+/, '..') + File.join(base, path) + end + end + + def clean_link(src, dest) + begin + link = File.readlink(dest) + rescue + else + return if link == src + File.unlink(dest) + end + yield src, dest + end + + # Extensions to FileUtils + + module Mswin + def ln_safe(src, dest, real_src, *opt) + cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] + cmd[1, 0] = opt + return if system("cmd", "/c", *cmd) + # TODO: use RUNAS or something + puts cmd.join(" ") + end + + def ln_dir_safe(src, dest, real_src) + ln_safe(src, dest, "/d") + end + end + + module HardlinkExcutable + def ln_exe(relative_src, dest, src) + ln(src, dest, force: true) + end + end + + def ln_safe(src, dest, real_src) + ln_sf(src, dest) + rescue Errno::ENOENT + # Windows disallows to create broken symboic links, probably because + # it is a kind of reparse points. + raise if File.exist?(real_src) + end + + alias ln_dir_safe ln_safe + alias ln_exe ln_safe + + def ln_relative(src, dest, executable = false) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + if executable + return (ln_exe(relative(src, parent), dest, src) if File.exist?(src)) + end + clean_link(relative(src, parent), dest) {|s, d| ln_safe(s, d, src)} + end + + def ln_dir_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative(src, parent), dest) {|s, d| ln_dir_safe(s, d, src)} + end + + case (CROSS_COMPILING || RUBY_PLATFORM) + when /linux|darwin|solaris/ + prepend HardlinkExcutable + extend HardlinkExcutable + when /mingw|mswin/ + unless File.respond_to?(:symlink) + prepend Mswin + extend Mswin + end + else + end +end diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb new file mode 100644 index 0000000000..7b889163b0 --- /dev/null +++ b/tool/lib/test/jobserver.rb @@ -0,0 +1,47 @@ +module Test + module JobServer + end +end + +class << Test::JobServer + def connect(makeflags = ENV["MAKEFLAGS"]) + return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags + begin + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end + rescue + r&.close + nil + else + return r, w + end + end + + def acquire_possible(r, w, max) + return unless tokens = r.read_nonblock(max - 1, exception: false) + if (jobs = tokens.size) > 0 + jobserver, w = w, nil + at_exit do + jobserver.print(tokens) + jobserver.close + end + end + return jobs + 1 + rescue Errno::EBADF + ensure + r&.close + w&.close + end + + def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"]) + if max > 1 and (r, w = connect(makeflags)) + acquire_possible(r, w, max) + end + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index bfa1964963..2663b7b76a 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -1,61 +1,138 @@ # frozen_string_literal: true +# Enable deprecation warnings for test-all, so deprecated methods/constants/functions are dealt with early. +Warning[:deprecated] = true + +if ENV['BACKTRACE_FOR_DEPRECATION_WARNINGS'] + Warning.extend Module.new { + def warn(message, category: nil, **kwargs) + if category == :deprecated and $stderr.respond_to?(:puts) + $stderr.puts nil, message, caller, nil + else + super + end + end + } +end + require_relative '../envutil' require_relative '../colorize' -require 'test/unit/testcase' +require_relative '../leakchecker' +require_relative '../test/unit/testcase' +require_relative '../test/jobserver' require 'optparse' -require "leakchecker" # See Test::Unit module Test - class << self + ## + # Test::Unit is an implementation of the xUnit testing framework for Ruby. + module Unit ## - # Filter object for backtraces. + # Assertion base class - attr_accessor :backtrace_filter - end + class AssertionFailedError < Exception; end + + ## + # Assertion raised when skipping a test + + class PendedError < AssertionFailedError; end + + class << self + ## + # Extract the location where the last assertion method was + # called. Returns "<empty>" if _e_ does not have backtrace, or + # an empty string if no assertion method location was found. - class BacktraceFilter # :nodoc: - def filter bt - return ["No backtrace"] unless bt + def location e + last_before_assertion = nil - new_bt = [] + return '<empty>' unless e&.backtrace # SystemStackError can return nil. - unless $DEBUG then - bt.each do |line| - break if line =~ /lib\/test/ - new_bt << line + e.backtrace.reverse_each do |s| + break if s =~ /:in \W(?:.*\#)?(?:assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s end + return "" unless last_before_assertion + /:in / =~ last_before_assertion ? $` : last_before_assertion + end + end - new_bt = bt.reject { |line| line =~ /lib\/test/ } if new_bt.empty? - new_bt = bt.dup if new_bt.empty? - else - new_bt = bt.dup + module Order + class NoSort + def initialize(seed) + end + + def sort_by_name(list) + list + end + + alias sort_by_string sort_by_name + + def group(list) + list + end end - new_bt - end - end + class Alpha < NoSort + def sort_by_name(list) + list.sort_by(&:name) + end - self.backtrace_filter = BacktraceFilter.new + def sort_by_string(list) + list.sort + end - def self.filter_backtrace bt # :nodoc: - backtrace_filter.filter bt - end + end - ## - # Test::Unit is an implementation of the xUnit testing framework for Ruby. - module Unit - ## - # Assertion base class + # shuffle test suites based on CRC32 of their names + Shuffle = Struct.new(:seed, :salt) do + def initialize(seed) + self.class::CRC_TBL ||= (0..255).map {|i| + (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } + }.freeze - class AssertionFailedError < Exception; end + salt = [seed].pack("V").unpack1("H*") + super(seed, "\n#{salt}".freeze).freeze + end - ## - # Assertion raised when skipping a test + def sort_by_name(list) + list.sort_by {|e| randomize_key(e.name)} + end - class PendedError < AssertionFailedError; end + def sort_by_string(list) + list.sort_by {|e| randomize_key(e)} + end + + def group(list) + list + end + + private + + def crc32(str, crc32 = 0xffffffff) + crc_tbl = self.class::CRC_TBL + str.each_byte do |data| + crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) + end + crc32 + end + + def randomize_key(name) + crc32(salt, crc32(name)) ^ 0xffffffff + end + end + + Types = { + random: Shuffle, + alpha: Alpha, + sorted: Alpha, + nosort: NoSort, + } + Types.default_proc = proc {|_, order| + raise "Unknown test_order: #{order.inspect}" + } + end module RunCount # :nodoc: all @@run_count = 0 @@ -103,17 +180,18 @@ module Test order = options[:test_order] if seed = options[:seed] order ||= :random - srand(seed) - else - seed = options[:seed] = srand % 100_000 - srand(seed) + elsif (order ||= :random) == :random + seed = options[:seed] = rand(0x10000) orig_args.unshift "--seed=#{seed}" end Test::Unit::TestCase.test_order = order if order + order = Test::Unit::TestCase.test_order + @order = Test::Unit::Order::Types[order].new(seed) @help = "\n" + orig_args.map { |s| " " + (s =~ /[\s|&<>$()]/ ? s.inspect : s) }.join("\n") + @options = options end @@ -139,7 +217,8 @@ module Test (options[:filter] ||= []) << a end - opts.on '--test-order=random|alpha|sorted|nosort', [:random, :alpha, :sorted, :nosort] do |a| + orders = Test::Unit::Order::Types.keys + opts.on "--test-order=#{orders.join('|')}", orders do |a| options[:test_order] = a end end @@ -154,6 +233,9 @@ module Test filter = nil elsif negative.empty? and positive.size == 1 and pos_pat !~ positive[0] filter = positive[0] + unless /\A[A-Z]\w*(?:::[A-Z]\w*)*#/ =~ filter + filter = /##{Regexp.quote(filter)}\z/ + end else filter = Regexp.union(*positive.map! {|s| Regexp.new(s[pos_pat, 1] || "\\A#{Regexp.quote(s)}\\z")}) end @@ -168,6 +250,8 @@ module Test end module Parallel # :nodoc: all + attr_accessor :prefix + def process_args(args = []) return @options if @options options = super @@ -179,22 +263,10 @@ module Test def non_options(files, options) @jobserver = nil - makeflags = ENV.delete("MAKEFLAGS") - if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(\d+),(\d+)/ =~ makeflags - begin - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) - rescue - r.close if r - nil - else - r.close_on_exec = true - w.close_on_exec = true - @jobserver = [r, w] - options[:parallel] ||= 1 - end + if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS")) + options[:parallel] ||= 256 # number of tokens to acquire first end + @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) super end @@ -210,13 +282,17 @@ module Test opts.separator "parallel test options:" - options[:retry] = true + options[:retry] = false opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| options[:testing] = true & t # For testing options[:parallel] = a.to_i end + opts.on '--worker-timeout=N', Integer, "Timeout workers not responding in N seconds" do |a| + options[:worker_timeout] = a + end + opts.on '--separate', "Restart job process after one testcase has done" do options[:parallel] ||= 1 options[:separate] = true @@ -230,7 +306,8 @@ module Test options[:retry] = false end - opts.on '--ruby VAL', "Path to ruby which is used at -j option" do |a| + opts.on '--ruby VAL', "Path to ruby which is used at -j option", + "Also used as EnvUtil.rubybin by some assertion methods" do |a| options[:ruby] = a.split(/ /).reject(&:empty?) end @@ -243,7 +320,7 @@ module Test def self.launch(ruby,args=[]) scale = EnvUtil.timeout_scale io = IO.popen([*ruby, "-W1", - "#{File.dirname(__FILE__)}/unit/parallel.rb", + "#{__dir__}/unit/parallel.rb", *("--timeout-scale=#{scale}" if scale), *args], "rb+") new(io, io.pid, :waiting) @@ -251,6 +328,8 @@ module Test attr_reader :quit_called attr_accessor :start_time + attr_accessor :response_at + attr_accessor :current @@worker_number = 0 @@ -264,6 +343,7 @@ module Test @loadpath = [] @hooks = {} @quit_called = false + @response_at = nil end def name @@ -274,8 +354,12 @@ module Test @io.puts(*args) end - def run(task,type) - @file = File.basename(task, ".rb") + def run(task, type, base = nil) + if base + @file = task.delete_prefix(base).chomp(".rb") + else + @file = File.basename(task, ".rb") + end @real_file = task begin puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" @@ -283,6 +367,7 @@ module Test puts "run #{task} #{type}" @status = :prepare @start_time = Time.now + @response_at = @start_time rescue Errno::EPIPE died rescue IOError @@ -299,6 +384,7 @@ module Test def read res = (@status == :quit) ? @io.read : @io.gets + @response_at = Time.now res && res.chomp end @@ -308,16 +394,19 @@ module Test rescue IOError end - def quit + def quit(reason = :normal) return if @io.closed? @quit_called = true - @io.puts "quit" + @io.puts "quit #{reason}" rescue Errno::EPIPE => e warn "#{@pid}:#{@status.to_s.ljust(7)}:#{@file}: #{e.message}" end def kill - Process.kill(:KILL, @pid) + EnvUtil::Debugger.search&.dump(@pid) + signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV + Process.kill(signal, @pid) + warn "worker #{to_s} does not respond; #{signal} is sent" rescue Errno::ESRCH end @@ -371,8 +460,8 @@ module Test real_file = worker.real_file and warn "running file: #{real_file}" @need_quit = true warn "" - warn "Some worker was crashed. It seems ruby interpreter's bug" - warn "or, a bug of test/unit/parallel.rb. try again without -j" + warn "A test worker crashed. It might be an interpreter bug or" + warn "a bug in test/unit/parallel.rb. Try again without the -j" warn "option." warn "" if File.exist?('core') @@ -429,30 +518,45 @@ module Test @ios.delete worker.io end - def quit_workers + def quit_workers(&cond) return if @workers.empty? + closed = [] if cond @workers.reject! do |worker| + next unless cond&.call(worker) begin - Timeout.timeout(1) do - worker.quit + Timeout.timeout(5) do + worker.quit(cond ? :timeout : :normal) end rescue Errno::EPIPE rescue Timeout::Error end - worker.close + closed&.push worker + begin + Timeout.timeout(1) do + worker.close + end + rescue Timeout::Error + worker.kill + retry + end + @ios.delete worker.io end - return if @workers.empty? + return if (closed ||= @workers).empty? + pids = closed.map(&:pid) begin - Timeout.timeout(0.2 * @workers.size) do + Timeout.timeout(1 * closed.size) do Process.waitall end rescue Timeout::Error - @workers.each do |worker| - worker.kill + if pids + Process.kill(:KILL, *pids) rescue nil + pids = nil + retry end - @worker.clear end + @workers.clear unless cond + closed end FakeClass = Struct.new(:name) @@ -482,15 +586,17 @@ module Test worker.quit worker = launch_worker end - worker.run(task, type) + worker.run(task, type, (@prefix unless @options[:job_status] == :replace)) @test_count += 1 jobs_status(worker) + when /^start (.+?)$/ + worker.current = Marshal.load($1.unpack1("m")) when /^done (.+?)$/ begin - r = Marshal.load($1.unpack("m")[0]) + r = Marshal.load($1.unpack1("m")) rescue - print "unknown object: #{$1.unpack("m")[0].dump}" + print "unknown object: #{$1.unpack1("m").dump}" return true end result << r[0..1] unless r[0..1] == [nil,nil] @@ -501,7 +607,7 @@ module Test return true when /^record (.+?)$/ begin - r = Marshal.load($1.unpack("m")[0]) + r = Marshal.load($1.unpack1("m")) suite = r.first key = [worker.name, suite] @@ -511,18 +617,18 @@ module Test @records[key] = [worker.start_time, Time.now] end rescue => e - print "unknown record: #{e.message} #{$1.unpack("m")[0].dump}" + print "unknown record: #{e.message} #{$1.unpack1("m").dump}" return true end record(fake_class(r[0]), *r[1..-1]) when /^p (.+?)$/ del_jobs_status - print $1.unpack("m")[0] + print $1.unpack1("m") jobs_status(worker) if @options[:job_status] == :replace when /^after (.+?)$/ - @warnings << Marshal.load($1.unpack("m")[0]) + @warnings << Marshal.load($1.unpack1("m")) when /^bye (.+?)$/ - after_worker_down worker, Marshal.load($1.unpack("m")[0]) + after_worker_down worker, Marshal.load($1.unpack1("m")) when /^bye$/, nil if shutting_down || worker.quit_called after_worker_quit worker @@ -545,16 +651,7 @@ module Test # Require needed thing for parallel running require 'timeout' - @tasks = @files.dup # Array of filenames. - - case Test::Unit::TestCase.test_order - when :random - @tasks.shuffle! - else - # JIT first - ts = @tasks.group_by{|e| /test_jit/ =~ e ? 0 : 1} - @tasks = ts[0] + ts[1] if ts.size == 2 - end + @tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames. @need_quit = false @dead_workers = [] # Array of dead workers. @@ -567,20 +664,34 @@ module Test @ios = [] # Array of worker IOs @job_tokens = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver begin - [@tasks.size, @options[:parallel]].min.times {launch_worker} + while true + newjobs = [@tasks.size, @options[:parallel]].min - @workers.size + if newjobs > 0 + if @jobserver + t = @jobserver[0].read_nonblock(newjobs, exception: false) + @job_tokens << t if String === t + newjobs = @job_tokens.size + 1 - @workers.size + end + newjobs.times {launch_worker} + end - while _io = IO.select(@ios)[0] - break if _io.any? do |io| + timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0), 0].max + @worker_timeout + + if !(_io = IO.select(@ios, nil, nil, timeout)) + timeout = Time.now - @worker_timeout + quit_workers {|w| w.response_at&.<(timeout) }&.map {|w| + rep << {file: w.real_file, result: nil, testcase: w.current[0], error: w.current} + } + elsif _io.first.any? {|io| @need_quit or (deal(io, type, result, rep).nil? and !@workers.any? {|x| [:running, :prepare].include? x.status}) + } + break end - if @jobserver and @job_tokens and !@tasks.empty? and !@workers.any? {|x| x.status == :ready} - t = @jobserver[0].read_nonblock([@tasks.size, @options[:parallel]].min, exception: false) - if String === t - @job_tokens << t - t.size.times {launch_worker} - end + if @tasks.empty? + break if @workers.empty? + next # wait for all workers to finish end end rescue Interrupt => ex @@ -588,7 +699,7 @@ module Test return result ensure if file = @options[:timetable_data] - open(file, 'w'){|f| + File.open(file, 'w'){|f| @records.each{|(worker, suite), (st, ed)| f.puts '[' + [worker.dump, suite.dump, st.to_f * 1_000, ed.to_f * 1_000].join(", ") + '],' } @@ -608,15 +719,50 @@ module Test unless @interrupt || !@options[:retry] || @need_quit parallel = @options[:parallel] @options[:parallel] = false - suites, rep = rep.partition {|r| r[:testcase] && r[:file] && r[:report].any? {|e| !e[2].is_a?(Test::Unit::PendedError)}} + suites, rep = rep.partition {|r| + r[:testcase] && r[:file] && + (!r.key?(:report) || r[:report].any? {|e| !e[2].is_a?(Test::Unit::PendedError)}) + } suites.map {|r| File.realpath(r[:file])}.uniq.each {|file| require file} - suites.map! {|r| eval("::"+r[:testcase])} del_status_line or puts + error, suites = suites.partition {|r| r[:error]} unless suites.empty? - puts "\n""Retrying..." + puts "\n" + @failed_output.puts "Failed tests:" + suites.each {|r| + r[:report].each {|c, m, e| + @failed_output.puts "#{c}##{m}: #{e&.class}: #{e&.message&.slice(/\A.*/)}" + } + } + @failed_output.puts "\n" + puts "Retrying..." @verbose = options[:verbose] + suites.map! {|r| ::Object.const_get(r[:testcase])} _run_suites(suites, type) end + unless error.empty? + puts "\n""Retrying hung up testcases..." + error = error.map do |r| + begin + ::Object.const_get(r[:testcase]) + rescue NameError + # testcase doesn't specify the correct case, so show `r` for information + require 'pp' + + $stderr.puts "Retrying is failed because the file and testcase is not consistent:" + PP.pp r, $stderr + @errors += 1 + nil + end + end.compact + verbose = @verbose + job_status = options[:job_status] + options[:verbose] = @verbose = true + options[:job_status] = :normal + result.concat _run_suites(error, type) + options[:verbose] = @verbose = verbose + options[:job_status] = job_status + end @options[:parallel] = parallel end unless @options[:retry] @@ -624,20 +770,28 @@ module Test end unless rep.empty? rep.each do |r| - r[:report].each do |f| + if r[:error] + puke(*r[:error], Timeout::Error.new) + next + end + r[:report]&.each do |f| puke(*f) if f end end if @options[:retry] - @errors += rep.map{|x| x[:result][0] }.inject(:+) - @failures += rep.map{|x| x[:result][1] }.inject(:+) - @skips += rep.map{|x| x[:result][2] }.inject(:+) + rep.each do |x| + (e, f, s = x[:result]) or next + @errors += e + @failures += f + @skips += s + end end end unless @warnings.empty? warn "" @warnings.uniq! {|w| w[1].message} @warnings.each do |w| + @errors += 1 warn "#{w[0]}: #{w[1].message} (#{w[1].class})" end warn "" @@ -707,7 +861,7 @@ module Test end end - def record(suite, method, assertions, time, error) + def record(suite, method, assertions, time, error, source_location = nil) if @options.values_at(:longest, :most_asserted).any? @tops ||= {} rec = [suite.name, method, assertions, time, error] @@ -786,7 +940,7 @@ module Test end def jobs_status(worker) - return if !@options[:job_status] or @options[:verbose] + return if !@options[:job_status] or @verbose if @options[:job_status] == :replace status_line = @workers.map(&:to_s).join(" ") else @@ -805,7 +959,7 @@ module Test end def _prepare_run(suites, type) - options[:job_status] ||= :replace if @tty && !@verbose + options[:job_status] ||= @tty ? :replace : :normal unless @verbose case options[:color] when :always color = true @@ -821,11 +975,14 @@ module Test @output = Output.new(self) unless @options[:testing] filter = options[:filter] type = "#{type}_methods" - total = if filter - suites.inject(0) {|n, suite| n + suite.send(type).grep(filter).size} - else - suites.inject(0) {|n, suite| n + suite.send(type).size} - end + total = suites.sum {|suite| + methods = suite.send(type) + if filter + methods.count {|method| filter === "#{suite}##{method}"} + else + methods.size + end + } @test_count = 0 @total_tests = total.to_s(10) end @@ -863,7 +1020,7 @@ module Test end first, msg = msg.split(/$/, 2) first = sprintf("%3d) %s", @report_count += 1, first) - $stdout.print(sep, @colorize.decorate(first, color), msg, "\n") + @failed_output.print(sep, @colorize.decorate(first, color), msg, "\n") sep = nil end report.clear @@ -919,7 +1076,7 @@ module Test runner.add_status(" = #$1") when /\A\.+\z/ runner.succeed - when /\A\.*[EFS][EFS.]*\z/ + when /\A\.*[EFST][EFST.]*\z/ runner.failed(s) else $stdout.print(s) @@ -1024,6 +1181,28 @@ module Test end end + module OutputOption # :nodoc: all + def setup_options(parser, options) + super + parser.separator "output options:" + + options[:failed_output] = $stdout + parser.on '--stderr-on-failure', 'Use stderr to print failure messages' do + options[:failed_output] = $stderr + end + parser.on '--stdout-on-failure', 'Use stdout to print failure messages', '(default)' do + options[:failed_output] = $stdout + end + end + + def process_args(args = []) + return @options if @options + options = super + @failed_output = options[:failed_output] + options + end + end + module GCOption # :nodoc: all def setup_options(parser, options) super @@ -1085,8 +1264,13 @@ module Test puts "#{f}: #{$!}" end } + @load_failed = errors.size.nonzero? result end + + def run(*) + super or @load_failed + end end module RepeatOption # :nodoc: all @@ -1097,10 +1281,15 @@ module Test parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| options[:repeat_count] = n end + options[:keep_repeating] = false + parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n| + options[:keep_repeating] = true + end end def _run_anything(type) @repeat_count = @options[:repeat_count] + @keep_repeating = @options[:keep_repeating] super end end @@ -1188,6 +1377,95 @@ module Test end end + module LaunchableOption + module Nothing + private + def setup_options(opts, options) + super + opts.define_tail 'Launchable options:' + # This is expected to be called by Test::Unit::Worker. + opts.on_tail '--launchable-test-reports=PATH', String, 'Do nothing' + end + end + + def record(suite, method, assertions, time, error, source_location = nil) + if writer = @options[:launchable_test_reports] + if loc = (source_location || suite.instance_method(method).source_location) + path, lineno = loc + # Launchable JSON schema is defined at + # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code. + e = case error + when nil + status = 'TEST_PASSED' + nil + when Test::Unit::PendedError + status = 'TEST_SKIPPED' + "Skipped:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Test::Unit::AssertionFailedError + status = 'TEST_FAILED' + "Failure:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Timeout::Error + status = 'TEST_FAILED' + "Timeout:\n#{suite.name}##{method}\n" + else + status = 'TEST_FAILED' + bt = Test::filter_backtrace(error.backtrace).join "\n " + "Error:\n#{suite.name}##{method}:\n#{error.class}: #{error.message.b}\n #{bt}\n" + end + repo_path = File.expand_path("#{__dir__}/../../../") + relative_path = path.delete_prefix("#{repo_path}/") + # The test path is a URL-encoded representation. + # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18 + test_path = {file: relative_path, class: suite.name, testcase: method}.map{|key, val| + "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}" + }.join('#') + end + end + super + ensure + if writer && test_path && status + # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified. + # In such cases, we proceed to execute the operation here. + writer.write_object( + { + testPath: test_path, + status: status, + duration: time, + createdAt: Time.now.to_s, + stderr: e, + stdout: nil, + data: { + lineNumber: lineno + } + } + ) + end + end + + private + def setup_options(opts, options) + super + opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path| + require_relative '../launchable' + options[:launchable_test_reports] = writer = Launchable::JsonStreamWriter.new(path) + writer.write_array('testCases') + main_pid = Process.pid + at_exit { + # This block is executed when the fork block in a test is completed. + # Therefore, we need to verify whether all tests have been completed. + stack = caller + if stack.size == 0 && main_pid == Process.pid && $!.is_a?(SystemExit) + writer.close + end + } + end + + def encode_test_path_component component + component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26') + end + end + end + class Runner # :nodoc: all attr_accessor :report, :failures, :errors, :skips # :nodoc: @@ -1302,6 +1580,8 @@ module Test suites = Test::Unit::TestCase.send "#{type}_suites" return if suites.empty? + suites = @order.sort_by_name(suites) + puts puts "# Running #{type}s:" puts @@ -1331,7 +1611,7 @@ module Test [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, t, @test_count.fdiv(t), @assertion_count.fdiv(t)] end while @repeat_count && @@current_repeat_count < @repeat_count && - report.empty? && failures.zero? && errors.zero? + (@keep_repeating || report.empty? && failures.zero? && errors.zero?) output.sync = old_sync if sync @@ -1356,6 +1636,12 @@ module Test filter = options[:filter] all_test_methods = suite.send "#{type}_methods" + if filter + all_test_methods.select! {|method| + filter === "#{suite}##{method}" + } + end + all_test_methods = @order.sort_by_name(all_test_methods) leakchecker = LeakChecker.new if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] @@ -1363,15 +1649,13 @@ module Test trace = true end - assertions = all_test_methods.filter_map { |method| - if filter - next unless filter === method || filter === "#{suite}##{method}" - end + assertions = all_test_methods.map { |method| inst = suite.new method + _start_method(inst) inst._assertions = 0 - print "#{suite}##{method} = " if @verbose + print "#{suite}##{method.inspect.sub(/\A:/, '')} = " if @verbose start_time = Time.now if @verbose result = @@ -1386,15 +1670,20 @@ module Test puts if @verbose $stdout.flush - unless defined?(RubyVM::JIT) && RubyVM::JIT.enabled? # compiler process is wrongly considered as leak - leakchecker.check("#{inst.class}\##{inst.__name__}") - end + leakchecker.check("#{inst.class}\##{inst.__name__}") + + _end_method(inst) inst._assertions } return assertions.size, assertions.inject(0) { |sum, n| sum + n } end + def _start_method(inst) + end + def _end_method(inst) + end + ## # Record the result of a single test. Makes it very easy to gather # information. Eg: @@ -1412,19 +1701,11 @@ module Test # failure or error in teardown, it will be sent again with the # error or failure. - def record suite, method, assertions, time, error + def record suite, method, assertions, time, error, source_location = nil end def location e # :nodoc: - last_before_assertion = "" - - return '<empty>' unless e.backtrace # SystemStackError can return nil. - - e.backtrace.reverse_each do |s| - break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ - last_before_assertion = s - end - last_before_assertion.sub(/:in .*$/, '') + Test::Unit.location e end ## @@ -1470,7 +1751,7 @@ module Test break unless report.empty? end - return failures + errors if self.test_count > 0 # or return nil... + return (failures + errors).nonzero? # or return nil... rescue Interrupt abort 'Interrupted' end @@ -1496,12 +1777,14 @@ module Test prepend Test::Unit::Statistics prepend Test::Unit::Skipping prepend Test::Unit::GlobOption + prepend Test::Unit::OutputOption prepend Test::Unit::RepeatOption prepend Test::Unit::LoadPathOption prepend Test::Unit::GCOption prepend Test::Unit::ExcludesOption prepend Test::Unit::TimeoutOption prepend Test::Unit::RunCount + prepend Test::Unit::LaunchableOption::Nothing ## # Begins the full test run. Delegates to +runner+'s #_run method. @@ -1537,6 +1820,9 @@ module Test when Test::Unit::AssertionFailedError then @failures += 1 "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Timeout::Error + @errors += 1 + "Timeout:\n#{klass}##{meth}\n" else @errors += 1 bt = Test::filter_backtrace(e.backtrace).join "\n " @@ -1555,6 +1841,7 @@ module Test class AutoRunner # :nodoc: all class Runner < Test::Unit::Runner include Test::Unit::RequireFiles + include Test::Unit::LaunchableOption end attr_accessor :to_run, :options @@ -1563,6 +1850,7 @@ module Test @force_standalone = force_standalone @runner = Runner.new do |files, options| base = options[:base_directory] ||= default_dir + @runner.prefix = base ? (base + "/") : nil files << default_dir if files.empty? and default_dir @to_run = files yield self if block_given? diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index 065a7a0a46..19581fc3ab 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -193,6 +193,22 @@ module Test end ## + # Fails unless +obj+ is true + + def assert_true obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be true" } + assert obj == true, msg + end + + ## + # Fails unless +obj+ is false + + def assert_false obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be false" } + assert obj == false, msg + end + + ## # For testing with binary operators. # # assert_operator 5, :<=, 4 @@ -345,20 +361,6 @@ module Test end ## - # Returns details for exception +e+ - - def exception_details e, msg - [ - "#{msg}", - "Class: <#{e.class}>", - "Message: <#{e.message.inspect}>", - "---Backtrace---", - "#{Test::filter_backtrace(e.backtrace).join("\n")}", - "---------------", - ].join "\n" - end - - ## # Fails with +msg+ def flunk msg = nil @@ -520,18 +522,16 @@ module Test # Skips the current test. Gets listed at the end of the run but # doesn't cause a failure exit code. - def pend msg = nil, bt = caller + def pend msg = nil, bt = caller, &_ msg ||= "Skipped, no message given" @skip = true raise Test::Unit::PendedError, msg, bt end alias omit pend - # TODO: Removed this and enabled to raise NoMethodError with skip - alias skip pend - # def skip(msg = nil, bt = caller) - # raise NoMethodError, "use omit or pend", caller - # end + def skip(msg = nil, bt = caller) + raise NoMethodError, "use omit or pend", caller + end ## # Was this testcase skipped? Meant for #teardown. @@ -563,10 +563,6 @@ module Test assert yield, *msgs end - def assert_raises(*exp, &b) - raise NoMethodError, "use assert_raise", caller - end - # :call-seq: # assert_nothing_thrown( failure_message = nil, &block ) # @@ -772,7 +768,14 @@ EOT e = assert_raise(SyntaxError, mesg) do syntax_check(src, fname, line) end - assert_match(error, e.message, mesg) + + # Prism adds ANSI escape sequences to syntax error messages to + # colorize and format them. We strip them out here to make them easier + # to match against in tests. + message = e.message + message.gsub!(/\e\[.*?m/, "") + + assert_match(error, message, mesg) e end end @@ -795,33 +798,37 @@ EOT MIN_MEASURABLE = 1.0 / MIN_HZ def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) - require 'benchmark' - wait = EnvUtil.apply_timeout_scale(wait) if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" end - tms = Benchmark.measure(msg || '') do - if stop - th = Thread.start {sleep wait; stop.call} - yield - th.join - else - begin - Timeout.timeout(wait) {yield} - rescue Timeout::Error - end + + t0, r0 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error end end - max = pct * tms.real + t1, r1 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + total = t1.utime - t0.utime + t1.stime - t0.stime + t1.cutime - t0.cutime + t1.cstime - t0.cstime + real = r1 - r0 + + max = pct * real min_measurable = MIN_MEASURABLE min_measurable *= 1.30 # add a little (30%) to account for misc. overheads if max < min_measurable max = min_measurable end - assert_operator tms.total, :<=, max, msg + assert_operator total, :<=, max, msg end def assert_is_minus_zero(f) diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb index 47ff0b360a..188a0d1a19 100644 --- a/tool/lib/test/unit/parallel.rb +++ b/tool/lib/test/unit/parallel.rb @@ -1,12 +1,6 @@ # frozen_string_literal: true -$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../.." -require 'test/unit' -require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') -require "tracepointchecker" -require "zombie_hunter" -require "iseq_loader_checker" -require "gc_checker" +require_relative "../../../test/init" module Test module Unit @@ -31,6 +25,10 @@ module Test end end + def _start_method(inst) + _report "start", Marshal.dump([inst.class.name, inst.__name__]) + end + def _run_suite(suite, type) # :nodoc: @partial_report = [] orig_testout = Test::Unit::Runner.output @@ -107,7 +105,7 @@ module Test case buf.chomp when /^loadpath (.+?)$/ @old_loadpath = $:.dup - $:.push(*Marshal.load($1.unpack("m")[0].force_encoding("ASCII-8BIT"))).uniq! + $:.push(*Marshal.load($1.unpack1("m").force_encoding("ASCII-8BIT"))).uniq! when /^run (.+?) (.+?)$/ _report "okay" @@ -129,7 +127,18 @@ module Test else _report "ready" end - when /^quit$/ + when /^quit (.+?)$/, "quit" + if $1 == "timeout" + err = ["", "!!! worker #{$$} killed due to timeout:"] + Thread.list.each do |th| + err << "#{ th.inspect }:" + th.backtrace.each do |s| + err << " #{ s }" + end + end + err << "" + STDERR.puts err.join("\n") + end _report "bye" exit end @@ -182,7 +191,7 @@ module Test else error = ProxyError.new(error) end - _report "record", Marshal.dump([suite.name, method, assertions, time, error]) + _report "record", Marshal.dump([suite.name, method, assertions, time, error, suite.instance_method(method).source_location]) super end end @@ -204,5 +213,9 @@ if $0 == __FILE__ end end require 'rubygems' + begin + require 'rake' + rescue LoadError + end Test::Unit::Worker.new.run(ARGV) end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb index dab25defff..51ffff37eb 100644 --- a/tool/lib/test/unit/testcase.rb +++ b/tool/lib/test/unit/testcase.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require 'test/unit/assertions' +require_relative 'assertions' require_relative '../../core_assertions' module Test @@ -137,6 +137,9 @@ module Test attr_reader :__name__ # :nodoc: + # Method name of this test. + alias method_name __name__ + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, Interrupt, SystemExit] # :nodoc: @@ -144,8 +147,7 @@ module Test # Runs the tests reporting the status to +runner+ def run runner - @options = runner.options - + @__runner_options__ = runner.options trap "INFO" do runner.report.each_with_index do |msg, i| warn "\n%3d) %s" % [i + 1, msg] @@ -159,10 +161,9 @@ module Test start_time = Time.now result = "" - srand(runner.options[:seed]) begin - @passed = nil + @__passed__ = nil self.before_setup self.setup self.after_setup @@ -170,11 +171,11 @@ module Test result = "." unless io? time = Time.now - start_time runner.record self.class, self.__name__, self._assertions, time, nil - @passed = true + @__passed__ = true rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - @passed = Test::Unit::PendedError === e + @__passed__ = Test::Unit::PendedError === e time = Time.now - start_time runner.record self.class, self.__name__, self._assertions, time, e result = runner.puke self.class, self.__name__, e @@ -185,7 +186,7 @@ module Test rescue *PASSTHROUGH_EXCEPTIONS raise rescue Exception => e - @passed = false + @__passed__ = false runner.record self.class, self.__name__, self._assertions, time, e result = runner.puke self.class, self.__name__, e end @@ -207,12 +208,12 @@ module Test def initialize name # :nodoc: @__name__ = name @__io__ = nil - @passed = nil - @@current = self # FIX: make thread local + @__passed__ = nil + @@__current__ = self # FIX: make thread local end def self.current # :nodoc: - @@current # FIX: make thread local + @@__current__ # FIX: make thread local end ## @@ -237,20 +238,6 @@ module Test reset - ## - # Make diffs for this TestCase use #pretty_inspect so that diff - # in assert_equal can be more details. NOTE: this is much slower - # than the regular inspect but much more usable for complex - # objects. - - def self.make_my_diffs_pretty! - require 'pp' - - define_method :mu_pp do |o| - o.pretty_inspect - end - end - def self.inherited klass # :nodoc: @@test_suites[klass] = true super @@ -267,53 +254,18 @@ module Test end def self.test_suites # :nodoc: - suites = @@test_suites.keys - - case self.test_order - when :random - # shuffle test suites based on CRC32 of their names - salt = "\n" + rand(1 << 32).to_s - crc_tbl = (0..255).map do |i| - (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } - end - suites = suites.sort_by do |suite| - crc32 = 0xffffffff - "#{suite.name}#{salt}".each_byte do |data| - crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) - end - crc32 ^ 0xffffffff - end - when :nosort - suites - else - suites.sort_by { |ts| ts.name.to_s } - end + @@test_suites.keys end def self.test_methods # :nodoc: - methods = public_instance_methods(true).grep(/^test/).map { |m| m.to_s } - - case self.test_order - when :parallel - max = methods.size - ParallelEach.new methods.sort.sort_by { rand max } - when :random then - max = methods.size - methods.sort.sort_by { rand max } - when :alpha, :sorted then - methods.sort - when :nosort - methods - else - raise "Unknown test_order: #{self.test_order.inspect}" - end + public_instance_methods(true).grep(/^test/) end ## # Returns true if the test passed. def passed? - @passed + @__passed__ end ## @@ -337,7 +289,7 @@ module Test return unless name.to_s.start_with?("test_") @test_methods ||= {} if @test_methods[name] - warn "test/unit warning: method #{ self }##{ name } is redefined" + raise AssertionFailedError, "test/unit: method #{ self }##{ name } is redefined" end @test_methods[name] = true end diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb index c41276e3b4..26c9763c13 100644 --- a/tool/lib/vcs.rb +++ b/tool/lib/vcs.rb @@ -1,6 +1,8 @@ # vcs require 'fileutils' require 'optparse' +require 'pp' +require 'tempfile' # This library is used by several other tools/ scripts to detect the current # VCS in use (e.g. SVN, Git) or to interact with that VCS. @@ -9,6 +11,22 @@ ENV.delete('PWD') class VCS DEBUG_OUT = STDERR.dup + + def self.dump(obj, pre = nil) + out = DEBUG_OUT + @pp ||= PP.new(out) + @pp.guard_inspect_key do + if pre + @pp.group(pre.size, pre) { + obj.pretty_print(@pp) + } + else + obj.pretty_print(@pp) + end + @pp.flush + out << "\n" + end + end end unless File.respond_to? :realpath @@ -19,37 +37,23 @@ unless File.respond_to? :realpath end def IO.pread(*args) - VCS::DEBUG_OUT.puts(args.inspect) if $DEBUG + VCS.dump(args, "args: ") if $DEBUG popen(*args) {|f|f.read} end module DebugPOpen refine IO.singleton_class do def popen(*args) - VCS::DEBUG_OUT.puts args.inspect if $DEBUG + VCS.dump(args, "args: ") if $DEBUG super end end end using DebugPOpen module DebugSystem - def system(*args) - VCS::DEBUG_OUT.puts args.inspect if $DEBUG - exception = false - opts = Hash.try_convert(args[-1]) - if RUBY_VERSION >= "2.6" - unless opts - opts = {} - args << opts - end - exception = opts.fetch(:exception) {opts[:exception] = true} - elsif opts - exception = opts.delete(:exception) {true} - args.pop if opts.empty? - end - ret = super(*args) - raise "Command failed with status (#$?): #{args[0]}" if exception and !ret - ret + def system(*args, exception: true, **opts) + VCS.dump(args, "args: ") if $DEBUG + super(*args, exception: exception, **opts) end end @@ -69,6 +73,9 @@ class VCS begin @@dirs.each do |dir, klass, pred| if pred ? pred[curr, dir] : File.directory?(File.join(curr, dir)) + if klass.const_defined?(:COMMAND) + IO.pread([{'LANG' => 'C', 'LC_ALL' => 'C'}, klass::COMMAND, "--version"]) rescue next + end vcs = klass.new(curr) vcs.define_options(parser) if parser vcs.set_options(options) @@ -92,9 +99,23 @@ class VCS parser.separator(" VCS common options:") parser.define("--[no-]dryrun") {|v| opts[:dryrun] = v} parser.define("--[no-]debug") {|v| opts[:debug] = v} + parser.define("-z", "--zone=OFFSET", /\A[-+]\d\d:\d\d\z/) {|v| opts[:zone] = v} opts end + def release_date(time) + t = time.getlocal(@zone) + [ + t.strftime('#define RUBY_RELEASE_YEAR %Y'), + t.strftime('#define RUBY_RELEASE_MONTH %-m'), + t.strftime('#define RUBY_RELEASE_DAY %-d'), + ] + end + + def self.short_revision(rev) + rev + end + attr_reader :srcdir def initialize(path) @@ -112,14 +133,14 @@ class VCS def set_options(opts) @debug = opts.fetch(:debug) {$DEBUG} @dryrun = opts.fetch(:dryrun) {@debug} + @zone = opts.fetch(:zone) {'+09:00'} end attr_reader :dryrun, :debug alias dryrun? dryrun alias debug? debug - NullDevice = defined?(IO::NULL) ? IO::NULL : - %w[/dev/null NUL NIL: NL:].find {|dev| File.exist?(dev)} + NullDevice = IO::NULL # returns # * the last revision of the current branch @@ -132,7 +153,7 @@ class VCS end last, changed, modified, *rest = ( begin - if NullDevice + if NullDevice and !debug? save_stderr = STDERR.dup STDERR.reopen NullDevice, 'w' end @@ -159,6 +180,7 @@ class VCS rescue ArgumentError modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 end + modified = modified.getlocal(@zone) end return last, changed, modified, *rest end @@ -190,6 +212,7 @@ class VCS def after_export(dir) FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + FileUtils.rm_rf(Dir.glob("#{dir}/.mailmap")) end def revision_handler(rev) @@ -204,170 +227,60 @@ class VCS revision_handler(rev).short_revision(rev) end - class SVN < self - register(".svn") - COMMAND = ENV['SVN'] || 'svn' - - def self.revision_name(rev) - "r#{rev}" - end - - def self.short_revision(rev) - rev - end - - def _get_revisions(path, srcdir = nil) - if srcdir and self.class.local_path?(path) - path = File.join(srcdir, path) - end - if srcdir - info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") - info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s - end - info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") - _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) - modified = info_xml[/<date>([^<>]*)/, 1] - branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] - [Integer(last), Integer(changed), modified, branch] - end - - def self.search_root(path) - return unless local_path?(path) - parent = File.realpath(path) - begin - parent = File.dirname(wkdir = parent) - return wkdir if File.directory?(wkdir + "/.svn") - end until parent == wkdir - end - - def get_info - @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") - end - - def url - @url ||= begin - url = get_info[/<root>(.*)<\/root>/, 1] - @url = URI.parse(url+"/") if url - end - end - - def wcroot - @wcroot ||= begin - info = get_info - @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] - @wcroot ||= self.class.search_root(@srcdir) - end - end - - def branch(name) - return trunk if name == "trunk" - url + "branches/#{name}" - end - - def tag(name) - url + "tags/#{name}" - end - - def trunk - url + "trunk" - end - alias master trunk - - def branch_list(pat) - IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| - f.each do |line| - line.chomp! - line.chomp!('/') - yield(line) if File.fnmatch?(pat, line) - end - end - end - - def grep(pat, tag, *files, &block) - cmd = %W"#{COMMAND} cat" - files.map! {|n| File.join(tag, n)} if tag - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files]) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end + # make-snapshot generates only release_date whereas file2lastrev generates both release_date and release_datetime + def revision_header(last, release_date, release_datetime = nil, branch = nil, title = nil, limit: 20) + short = short_revision(last) + if /[^\x00-\x7f]/ =~ title and title.respond_to?(:force_encoding) + title = title.dup.force_encoding("US-ASCII") + end + code = [ + "#define RUBY_REVISION #{short.inspect}", + ] + unless short == last + code << "#define RUBY_FULL_REVISION #{last.inspect}" + end + if branch + e = '..' + name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} + name = name.dump.sub(/\\#/, '#') + code << "#define RUBY_BRANCH_NAME #{name}" + end + if title + title = title.dump.sub(/\\#/, '#') + code << "#define RUBY_LAST_COMMIT_TITLE #{title}" + end + if release_datetime + t = release_datetime.utc + code << t.strftime('#define RUBY_RELEASE_DATETIME "%FT%TZ"') + end + code += self.release_date(release_date) + code + end - def export(revision, url, dir, keep_temp = false) - if @srcdir and (rootdir = wcroot) - srcdir = File.realpath(@srcdir) - rootdir << "/" - if srcdir.start_with?(rootdir) - subdir = srcdir[rootdir.size..-1] - subdir = nil if subdir.empty? - FileUtils.mkdir_p(svndir = dir+"/.svn") - FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) - system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false - FileUtils.rm_rf(svndir) unless keep_temp - if subdir - tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") - File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") - FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) - begin - Dir.rmdir("#{dir}/#{subdir}") - end until (subdir = File.dirname(subdir)) == '.' - FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) - Dir.rmdir(tmpdir) - end - return self + class GIT < self + register(".git") do |path, dir| + SAFE_DIRECTORIES ||= + begin + command = ENV["GIT"] || 'git' + dirs = IO.popen(%W"#{command} config --global --get-all safe.directory", &:read).split("\n") + rescue + command = nil + dirs = [] + ensure + VCS.dump(dirs, "safe.directory: ") if $DEBUG + COMMAND = command end - end - IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| - pipe.each {|line| /^A/ =~ line or yield line} - end - self if $?.success? - end - def after_export(dir) - super - FileUtils.rm_rf(dir+"/.svn") + COMMAND and File.exist?(File.join(path, dir)) end - def branch_beginning(url) - # `--limit` of svn-log is useless in this case, because it is - # applied before `--search`. - rev = IO.pread(%W[ #{COMMAND} log --xml - --search=matz --search-and=has\ started - -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] - rev.to_i if rev - end - - def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path) - range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') - IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -r#{range} #{url}") do |r| - IO.copy_stream(r, path) - end - end - - def commit - args = %W"#{COMMAND} commit" - if dryrun? - VCS::DEBUG_OUT.puts(args.inspect) - return true - end - system(*args) - end - end - - class GIT < self - register(".git") {|path, dir| File.exist?(File.join(path, dir))} - COMMAND = ENV["GIT"] || 'git' - def cmd_args(cmds, srcdir = nil) (opts = cmds.last).kind_of?(Hash) or cmds << (opts = {}) opts[:external_encoding] ||= "UTF-8" if srcdir opts[:chdir] ||= srcdir end - VCS::DEBUG_OUT.puts cmds.inspect if debug? + VCS.dump(cmds, "cmds: ") if debug? and !$DEBUG cmds end @@ -377,7 +290,7 @@ class VCS def cmd_read_at(srcdir, cmds) result = without_gitconfig { IO.pread(*cmd_args(cmds, srcdir)) } - VCS::DEBUG_OUT.puts result.inspect if debug? + VCS.dump(result, "result: ") if debug? result end @@ -398,7 +311,14 @@ class VCS def _get_revisions(path, srcdir = nil) ref = Branch === path ? path.to_str : 'HEAD' gitcmd = [COMMAND] - last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref]]).rstrip + last = nil + IO.pipe do |r, w| + last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref, err: w]]).rstrip + w.close + unless r.eof? + raise VCS::NotFoundError, "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" + end + end log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) changed = log[/\Acommit (\h+)/, 1] modified = log[/^CommitDate:\s+(.*)/, 1] @@ -450,26 +370,36 @@ class VCS rev[0, 10] end - def revision_handler(rev) - case rev - when Integer - SVN - else - super - end - end - def without_gitconfig - home = ENV.delete('HOME') + envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| + h[v] = ENV.delete(v) + end + ENV['GIT_CONFIG_SYSTEM'] = NullDevice + ENV['GIT_CONFIG_GLOBAL'] = global_config yield ensure - ENV['HOME'] = home if home + ENV.update(envs) + end + + def global_config + return NullDevice if SAFE_DIRECTORIES.empty? + unless @gitconfig + @gitconfig = Tempfile.new(%w"vcs_ .gitconfig") + @gitconfig.close + ENV['GIT_CONFIG_GLOBAL'] = @gitconfig.path + SAFE_DIRECTORIES.each do |dir| + system(*%W[#{COMMAND} config --global --add safe.directory #{dir}]) + end + VCS.dump(`#{COMMAND} config --global --get-all safe.directory`, "safe.directory: ") if debug? + end + @gitconfig.path end def initialize(*) super @srcdir = File.realpath(@srcdir) - VCS::DEBUG_OUT.puts @srcdir.inspect if debug? + @gitconfig = nil + VCS.dump(@srcdir, "srcdir: ") if debug? self end @@ -514,32 +444,26 @@ class VCS def export(revision, url, dir, keep_temp = false) system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return - (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir)) + GIT.new(File.expand_path(dir)) end def branch_beginning(url) - cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H + year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i + cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25 --author=matz --committer=matz --grep=started\\.$ - #{url.to_str} -- version.h include/ruby/version.h]) + #{url} -- version.h include/ruby/version.h])[/.*/] end - def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil) - svn = nil + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true) from, to = [from, to].map do |rev| rev or next - if Integer === rev - svn = true - rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -n1 --format=format:%H" << - "--grep=^ *git-svn-id: .*@#{rev} ") - end rev unless rev.empty? end - unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from)) + unless from&.match?(/./) or (from = branch_beginning(url))&.match?(/./) warn "no starting commit found", uplevel: 1 from = nil end - if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", chdir: @srcdir, exception: false) system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", chdir: @srcdir, exception: false) @@ -552,22 +476,18 @@ class VCS else arg = ["--since=25 Dec 00:00:00", to] end - writer = - if svn - format_changelog_as_svn(path, arg) - else - if base_url == true - remote, = upstream - if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) - remote.chomp! - # hack to redirect git.r-l.o to github - remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') - remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') - end - base_url = remote - end - format_changelog(path, arg, base_url) + if base_url == true + env = CHANGELOG_ENV + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') end + base_url = remote + end + writer = changelog_formatter(path, arg, base_url) if !path or path == '-' writer[$stdout] else @@ -576,36 +496,84 @@ class VCS end LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' - - def format_changelog(path, arg, base_url = nil) - env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} - cmd = %W"#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges" + CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + + def changelog_formatter(path, arg, base_url = nil) + env = CHANGELOG_ENV + cmd = %W[#{COMMAND} log + --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges + --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] + ] date = "--date=iso-local" - unless system(env, *cmd, date, chdir: @srcdir, out: NullDevice, exception: false) + unless system(env, *cmd, date, "-1", chdir: @srcdir, out: NullDevice, exception: false) date = "--date=iso" end cmd << date cmd.concat(arg) proc do |w| - w.print "-*- coding: utf-8 -*-\n\n" - w.print "base-url = #{base_url}\n\n" if base_url + w.print "-*- coding: utf-8 -*-\n" + w.print "\n""base-url = #{base_url}\n" if base_url + + begin + ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true) + .grep_v(/^ *(?:#|$)/) + .to_h {|v| [v, true]} + ignore_revs = nil if ignore_revs.empty? + rescue Errno::ENOENT + end + cmd_pipe(env, cmd, chdir: @srcdir) do |r| - while s = r.gets("\ncommit ") + r.gets(sep = "commit ") + sep = "\n" + sep + while s = r.gets(sep, chomp: true) h, s = s.split(/^$/, 2) + if ignore_revs&.key?(h[/\A\h{40}/]) + next + end + + next if /^Author: *dependabot\[bot\]/ =~ h + h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') fix = $1 + next if /\A *skip\Z/ =~ fix s = s.lines fix.each_line do |x| + next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') + b = ($2&.to_i || (s.size - 1 + $3.to_i)) + sp = $1 + if x.sub!(/^,(?:(\d+)|\$(?:-\d+)?)/, '') + range = b..($1&.to_i || (s.size - 1 + $2.to_i)) + else + range = b..b + end case x - when %r[^ +(\d+)s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\2(.*)\2]o - n = $1.to_i - wrong = $3 - correct = $4 - begin - s[n][wrong] = correct - rescue IndexError - message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #$1\n"] + when %r[^s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\1(.*)\1([gr]+)?]o + wrong = $2 + correct = $3 + if opt = $4 and opt.include?("r") # regexp + wrong = Regexp.new(wrong) + correct.gsub!(/(?<!\\)(?:\\\\)*\K(?:\\n)+/) {"\n" * ($&.size / 2)} + sub = opt.include?("g") ? :gsub! : :sub! + else + sub = false + end + range.each do |n| + if sub + ss = s[n].sub(/^#{sp}/, "") # un-indent for /^/ + if ss.__send__(sub, wrong, correct) + s[n, 1] = ss.lines.map {|l| "#{sp}#{l}"} + next + end + else + begin + s[n][wrong] = correct + rescue IndexError + else + next + end + end + message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] from = [1, n-2].max to = [s.size-1, n+2].min s.each_with_index do |e, i| @@ -615,46 +583,30 @@ class VCS end raise message.join('') end - when %r[^( +)(\d+)i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\3]o - s[$2.to_i, 0] = "#{$1}#{$4}\n" - when %r[^ +(\d+)(?:,(\d+))?d] - n = $1.to_i - e = $2 - s[n..(e ? e.to_i : n)] = [] + when %r[^i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\1]o + insert = "#{sp}#{$2}\n" + range.reverse_each do |n| + s[n, 0] = insert + end + when %r[^d] + s[range] = [] end end s = s.join('') end + s.gsub!(%r[(?!<\w)([-\w]+/[-\w]+)(?:@(\h{8,40})|#(\d{5,}))\b]) do + path = defined?($2) ? "commit/#{$2}" : "pull/#{$3}" + "[#$&](https://github.com/#{$1}/#{path})" + end if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s issue = "#{$1}pull/" - s.gsub!(/\b[Ff]ix(?:e[sd])? \K#(?=\d+)/) {issue} + s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} end s.gsub!(/ +\n/, "\n") s.sub!(/^Notes:/, ' \&') - w.print h, s - end - end - end - end - - def format_changelog_as_svn(path, arg) - cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B" - cmd.concat(arg) - proc do |w| - sep = "-"*72 + "\n" - w.print sep - cmd_pipe(cmd) do |r| - while s = r.gets("\0") - s.chomp!("\0") - author, time, s = s.split("\n", 3) - s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '') - rev = $1 - time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)") - lines = s.count("\n") + 1 - lines = "#{lines} line#{lines == 1 ? '' : 's'}" - w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep + w.print sep, h, s end end end @@ -674,13 +626,13 @@ class VCS def commit(opts = {}) args = [COMMAND, "push"] - args << "-n" if dryrun + args << "-n" if dryrun? remote, branch = upstream args << remote branches = %W[refs/notes/commits:refs/notes/commits HEAD:#{branch}] if dryrun? branches.each do |b| - VCS::DEBUG_OUT.puts((args + [b]).inspect) + VCS.dump(args + [b], "commit: ") end return true end @@ -691,43 +643,14 @@ class VCS end end - class GITSVN < GIT - def self.revision_name(rev) - SVN.revision_name(rev) + class Null < self + def get_revisions(path, srcdir = nil) + @modified ||= Time.now - 10 + return nil, nil, @modified end - def last_changed_revision - rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] - com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp - return rev, com - end - - def commit(opts = {}) - rev, com = last_changed_revision - head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp - - commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") - commits.each_with_index do |l, i| - r, a, c = l.split(' ') - dcommit = [COMMAND, "svn", "dcommit"] - dcommit.insert(-2, "-n") if dryrun - dcommit << "--add-author-from" unless a == c - dcommit << r - system(*dcommit) or return false - system(COMMAND, "checkout", head) or return false - system(COMMAND, "rebase") or return false - end - - if rev - old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] - old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] - 3.times do - sleep 2 - system(*%W"#{COMMAND} pull --no-edit --rebase") - break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) - end - end - true + def revision_header(last, release_date, release_datetime = nil, branch = nil, title = nil, limit: 20) + self.release_date(release_date) end end end diff --git a/tool/lib/vpath.rb b/tool/lib/vpath.rb index 48ab148405..fa819f3242 100644 --- a/tool/lib/vpath.rb +++ b/tool/lib/vpath.rb @@ -53,10 +53,11 @@ class VPath end def def_options(opt) + opt.separator(" VPath common options:") opt.on("-I", "--srcdir=DIR", "add a directory to search path") {|dir| @additional << dir } - opt.on("-L", "--vpath=PATH LIST", "add directories to search path") {|dirs| + opt.on("-L", "--vpath=PATH-LIST", "add directories to search path") {|dirs| @additional << [dirs] } opt.on("--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, "separator for vpath") {|sep, vsep| @@ -80,6 +81,10 @@ class VPath @list end + def add(path) + @additional << path + end + def strip(path) prefix = list.map {|dir| Regexp.quote(dir)} path.sub(/\A#{prefix.join('|')}(?:\/|\z)/, '') diff --git a/tool/lib/webrick.rb b/tool/lib/webrick.rb deleted file mode 100644 index b854b68db4..0000000000 --- a/tool/lib/webrick.rb +++ /dev/null @@ -1,232 +0,0 @@ -# frozen_string_literal: false -## -# = WEB server toolkit. -# -# WEBrick is an HTTP server toolkit that can be configured as an HTTPS server, -# a proxy server, and a virtual-host server. WEBrick features complete -# logging of both server operations and HTTP access. WEBrick supports both -# basic and digest authentication in addition to algorithms not in RFC 2617. -# -# A WEBrick server can be composed of multiple WEBrick servers or servlets to -# provide differing behavior on a per-host or per-path basis. WEBrick -# includes servlets for handling CGI scripts, ERB pages, Ruby blocks and -# directory listings. -# -# WEBrick also includes tools for daemonizing a process and starting a process -# at a higher privilege level and dropping permissions. -# -# == Security -# -# *Warning:* WEBrick is not recommended for production. It only implements -# basic security checks. -# -# == Starting an HTTP server -# -# To create a new WEBrick::HTTPServer that will listen to connections on port -# 8000 and serve documents from the current user's public_html folder: -# -# require 'webrick' -# -# root = File.expand_path '~/public_html' -# server = WEBrick::HTTPServer.new :Port => 8000, :DocumentRoot => root -# -# To run the server you will need to provide a suitable shutdown hook as -# starting the server blocks the current thread: -# -# trap 'INT' do server.shutdown end -# -# server.start -# -# == Custom Behavior -# -# The easiest way to have a server perform custom operations is through -# WEBrick::HTTPServer#mount_proc. The block given will be called with a -# WEBrick::HTTPRequest with request info and a WEBrick::HTTPResponse which -# must be filled in appropriately: -# -# server.mount_proc '/' do |req, res| -# res.body = 'Hello, world!' -# end -# -# Remember that +server.mount_proc+ must precede +server.start+. -# -# == Servlets -# -# Advanced custom behavior can be obtained through mounting a subclass of -# WEBrick::HTTPServlet::AbstractServlet. Servlets provide more modularity -# when writing an HTTP server than mount_proc allows. Here is a simple -# servlet: -# -# class Simple < WEBrick::HTTPServlet::AbstractServlet -# def do_GET request, response -# status, content_type, body = do_stuff_with request -# -# response.status = 200 -# response['Content-Type'] = 'text/plain' -# response.body = 'Hello, World!' -# end -# end -# -# To initialize the servlet you mount it on the server: -# -# server.mount '/simple', Simple -# -# See WEBrick::HTTPServlet::AbstractServlet for more details. -# -# == Virtual Hosts -# -# A server can act as a virtual host for multiple host names. After creating -# the listening host, additional hosts that do not listen can be created and -# attached as virtual hosts: -# -# server = WEBrick::HTTPServer.new # ... -# -# vhost = WEBrick::HTTPServer.new :ServerName => 'vhost.example', -# :DoNotListen => true, # ... -# vhost.mount '/', ... -# -# server.virtual_host vhost -# -# If no +:DocumentRoot+ is provided and no servlets or procs are mounted on the -# main server it will return 404 for all URLs. -# -# == HTTPS -# -# To create an HTTPS server you only need to enable SSL and provide an SSL -# certificate name: -# -# require 'webrick' -# require 'webrick/https' -# -# cert_name = [ -# %w[CN localhost], -# ] -# -# server = WEBrick::HTTPServer.new(:Port => 8000, -# :SSLEnable => true, -# :SSLCertName => cert_name) -# -# This will start the server with a self-generated self-signed certificate. -# The certificate will be changed every time the server is restarted. -# -# To create a server with a pre-determined key and certificate you can provide -# them: -# -# require 'webrick' -# require 'webrick/https' -# require 'openssl' -# -# cert = OpenSSL::X509::Certificate.new File.read '/path/to/cert.pem' -# pkey = OpenSSL::PKey::RSA.new File.read '/path/to/pkey.pem' -# -# server = WEBrick::HTTPServer.new(:Port => 8000, -# :SSLEnable => true, -# :SSLCertificate => cert, -# :SSLPrivateKey => pkey) -# -# == Proxy Server -# -# WEBrick can act as a proxy server: -# -# require 'webrick' -# require 'webrick/httpproxy' -# -# proxy = WEBrick::HTTPProxyServer.new :Port => 8000 -# -# trap 'INT' do proxy.shutdown end -# -# See WEBrick::HTTPProxy for further details including modifying proxied -# responses. -# -# == Basic and Digest authentication -# -# WEBrick provides both Basic and Digest authentication for regular and proxy -# servers. See WEBrick::HTTPAuth, WEBrick::HTTPAuth::BasicAuth and -# WEBrick::HTTPAuth::DigestAuth. -# -# == WEBrick as a daemonized Web Server -# -# WEBrick can be run as a daemonized server for small loads. -# -# === Daemonizing -# -# To start a WEBrick server as a daemon simple run WEBrick::Daemon.start -# before starting the server. -# -# === Dropping Permissions -# -# WEBrick can be started as one user to gain permission to bind to port 80 or -# 443 for serving HTTP or HTTPS traffic then can drop these permissions for -# regular operation. To listen on all interfaces for HTTP traffic: -# -# sockets = WEBrick::Utils.create_listeners nil, 80 -# -# Then drop privileges: -# -# WEBrick::Utils.su 'www' -# -# Then create a server that does not listen by default: -# -# server = WEBrick::HTTPServer.new :DoNotListen => true, # ... -# -# Then overwrite the listening sockets with the port 80 sockets: -# -# server.listeners.replace sockets -# -# === Logging -# -# WEBrick can separately log server operations and end-user access. For -# server operations: -# -# log_file = File.open '/var/log/webrick.log', 'a+' -# log = WEBrick::Log.new log_file -# -# For user access logging: -# -# access_log = [ -# [log_file, WEBrick::AccessLog::COMBINED_LOG_FORMAT], -# ] -# -# server = WEBrick::HTTPServer.new :Logger => log, :AccessLog => access_log -# -# See WEBrick::AccessLog for further log formats. -# -# === Log Rotation -# -# To rotate logs in WEBrick on a HUP signal (like syslogd can send), open the -# log file in 'a+' mode (as above) and trap 'HUP' to reopen the log file: -# -# trap 'HUP' do log_file.reopen '/path/to/webrick.log', 'a+' -# -# == Copyright -# -# Author: IPR -- Internet Programming with Ruby -- writers -# -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -#-- -# $IPR: webrick.rb,v 1.12 2002/10/01 17:16:31 gotoyuzo Exp $ - -module WEBrick -end - -require 'webrick/compat.rb' - -require 'webrick/version.rb' -require 'webrick/config.rb' -require 'webrick/log.rb' -require 'webrick/server.rb' -require_relative 'webrick/utils.rb' -require 'webrick/accesslog' - -require 'webrick/htmlutils.rb' -require 'webrick/httputils.rb' -require 'webrick/cookie.rb' -require 'webrick/httpversion.rb' -require 'webrick/httpstatus.rb' -require 'webrick/httprequest.rb' -require 'webrick/httpresponse.rb' -require 'webrick/httpserver.rb' -require 'webrick/httpservlet.rb' -require 'webrick/httpauth.rb' diff --git a/tool/lib/webrick/.document b/tool/lib/webrick/.document deleted file mode 100644 index c62f89083b..0000000000 --- a/tool/lib/webrick/.document +++ /dev/null @@ -1,6 +0,0 @@ -# Add files to this as they become documented - -*.rb - -httpauth -httpservlet diff --git a/tool/lib/webrick/accesslog.rb b/tool/lib/webrick/accesslog.rb deleted file mode 100644 index e4849637f3..0000000000 --- a/tool/lib/webrick/accesslog.rb +++ /dev/null @@ -1,157 +0,0 @@ -# frozen_string_literal: false -#-- -# accesslog.rb -- Access log handling utilities -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 keita yamaguchi -# Copyright (c) 2002 Internet Programming with Ruby writers -# -# $IPR: accesslog.rb,v 1.1 2002/10/01 17:16:32 gotoyuzo Exp $ - -module WEBrick - - ## - # AccessLog provides logging to various files in various formats. - # - # Multiple logs may be written to at the same time: - # - # access_log = [ - # [$stderr, WEBrick::AccessLog::COMMON_LOG_FORMAT], - # [$stderr, WEBrick::AccessLog::REFERER_LOG_FORMAT], - # ] - # - # server = WEBrick::HTTPServer.new :AccessLog => access_log - # - # Custom log formats may be defined. WEBrick::AccessLog provides a subset - # of the formatting from Apache's mod_log_config - # http://httpd.apache.org/docs/mod/mod_log_config.html#formats. See - # AccessLog::setup_params for a list of supported options - - module AccessLog - - ## - # Raised if a parameter such as %e, %i, %o or %n is used without fetching - # a specific field. - - class AccessLogError < StandardError; end - - ## - # The Common Log Format's time format - - CLF_TIME_FORMAT = "[%d/%b/%Y:%H:%M:%S %Z]" - - ## - # Common Log Format - - COMMON_LOG_FORMAT = "%h %l %u %t \"%r\" %s %b" - - ## - # Short alias for Common Log Format - - CLF = COMMON_LOG_FORMAT - - ## - # Referer Log Format - - REFERER_LOG_FORMAT = "%{Referer}i -> %U" - - ## - # User-Agent Log Format - - AGENT_LOG_FORMAT = "%{User-Agent}i" - - ## - # Combined Log Format - - COMBINED_LOG_FORMAT = "#{CLF} \"%{Referer}i\" \"%{User-agent}i\"" - - module_function - - # This format specification is a subset of mod_log_config of Apache: - # - # %a:: Remote IP address - # %b:: Total response size - # %e{variable}:: Given variable in ENV - # %f:: Response filename - # %h:: Remote host name - # %{header}i:: Given request header - # %l:: Remote logname, always "-" - # %m:: Request method - # %{attr}n:: Given request attribute from <tt>req.attributes</tt> - # %{header}o:: Given response header - # %p:: Server's request port - # %{format}p:: The canonical port of the server serving the request or the - # actual port or the client's actual port. Valid formats are - # canonical, local or remote. - # %q:: Request query string - # %r:: First line of the request - # %s:: Request status - # %t:: Time the request was received - # %T:: Time taken to process the request - # %u:: Remote user from auth - # %U:: Unparsed URI - # %%:: Literal % - - def setup_params(config, req, res) - params = Hash.new("") - params["a"] = req.peeraddr[3] - params["b"] = res.sent_size - params["e"] = ENV - params["f"] = res.filename || "" - params["h"] = req.peeraddr[2] - params["i"] = req - params["l"] = "-" - params["m"] = req.request_method - params["n"] = req.attributes - params["o"] = res - params["p"] = req.port - params["q"] = req.query_string - params["r"] = req.request_line.sub(/\x0d?\x0a\z/o, '') - params["s"] = res.status # won't support "%>s" - params["t"] = req.request_time - params["T"] = Time.now - req.request_time - params["u"] = req.user || "-" - params["U"] = req.unparsed_uri - params["v"] = config[:ServerName] - params - end - - ## - # Formats +params+ according to +format_string+ which is described in - # setup_params. - - def format(format_string, params) - format_string.gsub(/\%(?:\{(.*?)\})?>?([a-zA-Z%])/){ - param, spec = $1, $2 - case spec[0] - when ?e, ?i, ?n, ?o - raise AccessLogError, - "parameter is required for \"#{spec}\"" unless param - (param = params[spec][param]) ? escape(param) : "-" - when ?t - params[spec].strftime(param || CLF_TIME_FORMAT) - when ?p - case param - when 'remote' - escape(params["i"].peeraddr[1].to_s) - else - escape(params["p"].to_s) - end - when ?% - "%" - else - escape(params[spec].to_s) - end - } - end - - ## - # Escapes control characters in +data+ - - def escape(data) - data = data.gsub(/[[:cntrl:]\\]+/) {$&.dump[1...-1]} - data.untaint if RUBY_VERSION < '2.7' - data - end - end -end diff --git a/tool/lib/webrick/cgi.rb b/tool/lib/webrick/cgi.rb deleted file mode 100644 index bb0ae2fc84..0000000000 --- a/tool/lib/webrick/cgi.rb +++ /dev/null @@ -1,313 +0,0 @@ -# frozen_string_literal: false -# -# cgi.rb -- Yet another CGI library -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $Id$ - -require_relative "httprequest" -require_relative "httpresponse" -require_relative "config" -require "stringio" - -module WEBrick - - # A CGI library using WEBrick requests and responses. - # - # Example: - # - # class MyCGI < WEBrick::CGI - # def do_GET req, res - # res.body = 'it worked!' - # res.status = 200 - # end - # end - # - # MyCGI.new.start - - class CGI - - # The CGI error exception class - - CGIError = Class.new(StandardError) - - ## - # The CGI configuration. This is based on WEBrick::Config::HTTP - - attr_reader :config - - ## - # The CGI logger - - attr_reader :logger - - ## - # Creates a new CGI interface. - # - # The first argument in +args+ is a configuration hash which would update - # WEBrick::Config::HTTP. - # - # Any remaining arguments are stored in the <code>@options</code> instance - # variable for use by a subclass. - - def initialize(*args) - if defined?(MOD_RUBY) - unless ENV.has_key?("GATEWAY_INTERFACE") - Apache.request.setup_cgi_env - end - end - if %r{HTTP/(\d+\.\d+)} =~ ENV["SERVER_PROTOCOL"] - httpv = $1 - end - @config = WEBrick::Config::HTTP.dup.update( - :ServerSoftware => ENV["SERVER_SOFTWARE"] || "null", - :HTTPVersion => HTTPVersion.new(httpv || "1.0"), - :RunOnCGI => true, # to detect if it runs on CGI. - :NPH => false # set true to run as NPH script. - ) - if config = args.shift - @config.update(config) - end - @config[:Logger] ||= WEBrick::BasicLog.new($stderr) - @logger = @config[:Logger] - @options = args - end - - ## - # Reads +key+ from the configuration - - def [](key) - @config[key] - end - - ## - # Starts the CGI process with the given environment +env+ and standard - # input and output +stdin+ and +stdout+. - - def start(env=ENV, stdin=$stdin, stdout=$stdout) - sock = WEBrick::CGI::Socket.new(@config, env, stdin, stdout) - req = HTTPRequest.new(@config) - res = HTTPResponse.new(@config) - unless @config[:NPH] or defined?(MOD_RUBY) - def res.setup_header - unless @header["status"] - phrase = HTTPStatus::reason_phrase(@status) - @header["status"] = "#{@status} #{phrase}" - end - super - end - def res.status_line - "" - end - end - - begin - req.parse(sock) - req.script_name = (env["SCRIPT_NAME"] || File.expand_path($0)).dup - req.path_info = (env["PATH_INFO"] || "").dup - req.query_string = env["QUERY_STRING"] - req.user = env["REMOTE_USER"] - res.request_method = req.request_method - res.request_uri = req.request_uri - res.request_http_version = req.http_version - res.keep_alive = req.keep_alive? - self.service(req, res) - rescue HTTPStatus::Error => ex - res.set_error(ex) - rescue HTTPStatus::Status => ex - res.status = ex.code - rescue Exception => ex - @logger.error(ex) - res.set_error(ex, true) - ensure - req.fixup - if defined?(MOD_RUBY) - res.setup_header - Apache.request.status_line = "#{res.status} #{res.reason_phrase}" - Apache.request.status = res.status - table = Apache.request.headers_out - res.header.each{|key, val| - case key - when /^content-encoding$/i - Apache::request.content_encoding = val - when /^content-type$/i - Apache::request.content_type = val - else - table[key] = val.to_s - end - } - res.cookies.each{|cookie| - table.add("Set-Cookie", cookie.to_s) - } - Apache.request.send_http_header - res.send_body(sock) - else - res.send_response(sock) - end - end - end - - ## - # Services the request +req+ which will fill in the response +res+. See - # WEBrick::HTTPServlet::AbstractServlet#service for details. - - def service(req, res) - method_name = "do_" + req.request_method.gsub(/-/, "_") - if respond_to?(method_name) - __send__(method_name, req, res) - else - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - end - end - - ## - # Provides HTTP socket emulation from the CGI environment - - class Socket # :nodoc: - include Enumerable - - private - - def initialize(config, env, stdin, stdout) - @config = config - @env = env - @header_part = StringIO.new - @body_part = stdin - @out_port = stdout - @out_port.binmode - - @server_addr = @env["SERVER_ADDR"] || "0.0.0.0" - @server_name = @env["SERVER_NAME"] - @server_port = @env["SERVER_PORT"] - @remote_addr = @env["REMOTE_ADDR"] - @remote_host = @env["REMOTE_HOST"] || @remote_addr - @remote_port = @env["REMOTE_PORT"] || 0 - - begin - @header_part << request_line << CRLF - setup_header - @header_part << CRLF - @header_part.rewind - rescue Exception - raise CGIError, "invalid CGI environment" - end - end - - def request_line - meth = @env["REQUEST_METHOD"] || "GET" - unless url = @env["REQUEST_URI"] - url = (@env["SCRIPT_NAME"] || File.expand_path($0)).dup - url << @env["PATH_INFO"].to_s - url = WEBrick::HTTPUtils.escape_path(url) - if query_string = @env["QUERY_STRING"] - unless query_string.empty? - url << "?" << query_string - end - end - end - # we cannot get real HTTP version of client ;) - httpv = @config[:HTTPVersion] - return "#{meth} #{url} HTTP/#{httpv}" - end - - def setup_header - @env.each{|key, value| - case key - when "CONTENT_TYPE", "CONTENT_LENGTH" - add_header(key.gsub(/_/, "-"), value) - when /^HTTP_(.*)/ - add_header($1.gsub(/_/, "-"), value) - end - } - end - - def add_header(hdrname, value) - unless value.empty? - @header_part << hdrname << ": " << value << CRLF - end - end - - def input - @header_part.eof? ? @body_part : @header_part - end - - public - - def peeraddr - [nil, @remote_port, @remote_host, @remote_addr] - end - - def addr - [nil, @server_port, @server_name, @server_addr] - end - - def gets(eol=LF, size=nil) - input.gets(eol, size) - end - - def read(size=nil) - input.read(size) - end - - def each - input.each{|line| yield(line) } - end - - def eof? - input.eof? - end - - def <<(data) - @out_port << data - end - - def write(data) - @out_port.write(data) - end - - def cert - return nil unless defined?(OpenSSL) - if pem = @env["SSL_SERVER_CERT"] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - - def peer_cert - return nil unless defined?(OpenSSL) - if pem = @env["SSL_CLIENT_CERT"] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - - def peer_cert_chain - return nil unless defined?(OpenSSL) - if @env["SSL_CLIENT_CERT_CHAIN_0"] - keys = @env.keys - certs = keys.sort.collect{|k| - if /^SSL_CLIENT_CERT_CHAIN_\d+$/ =~ k - if pem = @env[k] - OpenSSL::X509::Certificate.new(pem) unless pem.empty? - end - end - } - certs.compact - end - end - - def cipher - return nil unless defined?(OpenSSL) - if cipher = @env["SSL_CIPHER"] - ret = [ cipher ] - ret << @env["SSL_PROTOCOL"] - ret << @env["SSL_CIPHER_USEKEYSIZE"] - ret << @env["SSL_CIPHER_ALGKEYSIZE"] - ret - end - end - end - end -end diff --git a/tool/lib/webrick/compat.rb b/tool/lib/webrick/compat.rb deleted file mode 100644 index c497a1933c..0000000000 --- a/tool/lib/webrick/compat.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: false -# -# compat.rb -- cross platform compatibility -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: compat.rb,v 1.6 2002/10/01 17:16:32 gotoyuzo Exp $ - -## -# System call error module used by webrick for cross platform compatibility. -# -# EPROTO:: protocol error -# ECONNRESET:: remote host reset the connection request -# ECONNABORTED:: Client sent TCP reset (RST) before server has accepted the -# connection requested by client. -# -module Errno - ## - # Protocol error. - - class EPROTO < SystemCallError; end - - ## - # Remote host reset the connection request. - - class ECONNRESET < SystemCallError; end - - ## - # Client sent TCP reset (RST) before server has accepted the connection - # requested by client. - - class ECONNABORTED < SystemCallError; end -end diff --git a/tool/lib/webrick/config.rb b/tool/lib/webrick/config.rb deleted file mode 100644 index 9f2ab44f49..0000000000 --- a/tool/lib/webrick/config.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: false -# -# config.rb -- Default configurations. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: config.rb,v 1.52 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'version' -require_relative 'httpversion' -require_relative 'httputils' -require_relative 'utils' -require_relative 'log' - -module WEBrick - module Config - LIBDIR = File::dirname(__FILE__) # :nodoc: - - # for GenericServer - General = Hash.new { |hash, key| - case key - when :ServerName - hash[key] = Utils.getservername - else - nil - end - }.update( - :BindAddress => nil, # "0.0.0.0" or "::" or nil - :Port => nil, # users MUST specify this!! - :MaxClients => 100, # maximum number of the concurrent connections - :ServerType => nil, # default: WEBrick::SimpleServer - :Logger => nil, # default: WEBrick::Log.new - :ServerSoftware => "WEBrick/#{WEBrick::VERSION} " + - "(Ruby/#{RUBY_VERSION}/#{RUBY_RELEASE_DATE})", - :TempDir => ENV['TMPDIR']||ENV['TMP']||ENV['TEMP']||'/tmp', - :DoNotListen => false, - :StartCallback => nil, - :StopCallback => nil, - :AcceptCallback => nil, - :DoNotReverseLookup => true, - :ShutdownSocketWithoutClose => false, - ) - - # for HTTPServer, HTTPRequest, HTTPResponse ... - HTTP = General.dup.update( - :Port => 80, - :RequestTimeout => 30, - :HTTPVersion => HTTPVersion.new("1.1"), - :AccessLog => nil, - :MimeTypes => HTTPUtils::DefaultMimeTypes, - :DirectoryIndex => ["index.html","index.htm","index.cgi","index.rhtml"], - :DocumentRoot => nil, - :DocumentRootOptions => { :FancyIndexing => true }, - :RequestCallback => nil, - :ServerAlias => nil, - :InputBufferSize => 65536, # input buffer size in reading request body - :OutputBufferSize => 65536, # output buffer size in sending File or IO - - # for HTTPProxyServer - :ProxyAuthProc => nil, - :ProxyContentHandler => nil, - :ProxyVia => true, - :ProxyTimeout => true, - :ProxyURI => nil, - - :CGIInterpreter => nil, - :CGIPathEnv => nil, - - # workaround: if Request-URIs contain 8bit chars, - # they should be escaped before calling of URI::parse(). - :Escape8bitURI => false - ) - - ## - # Default configuration for WEBrick::HTTPServlet::FileHandler - # - # :AcceptableLanguages:: - # Array of languages allowed for accept-language. There is no default - # :DirectoryCallback:: - # Allows preprocessing of directory requests. There is no default - # callback. - # :FancyIndexing:: - # If true, show an index for directories. The default is true. - # :FileCallback:: - # Allows preprocessing of file requests. There is no default callback. - # :HandlerCallback:: - # Allows preprocessing of requests. There is no default callback. - # :HandlerTable:: - # Maps file suffixes to file handlers. DefaultFileHandler is used by - # default but any servlet can be used. - # :NondisclosureName:: - # Do not show files matching this array of globs. .ht* and *~ are - # excluded by default. - # :UserDir:: - # Directory inside ~user to serve content from for /~user requests. - # Only works if mounted on /. Disabled by default. - - FileHandler = { - :NondisclosureName => [".ht*", "*~"], - :FancyIndexing => false, - :HandlerTable => {}, - :HandlerCallback => nil, - :DirectoryCallback => nil, - :FileCallback => nil, - :UserDir => nil, # e.g. "public_html" - :AcceptableLanguages => [] # ["en", "ja", ... ] - } - - ## - # Default configuration for WEBrick::HTTPAuth::BasicAuth - # - # :AutoReloadUserDB:: Reload the user database provided by :UserDB - # automatically? - - BasicAuth = { - :AutoReloadUserDB => true, - } - - ## - # Default configuration for WEBrick::HTTPAuth::DigestAuth. - # - # :Algorithm:: MD5, MD5-sess (default), SHA1, SHA1-sess - # :Domain:: An Array of URIs that define the protected space - # :Qop:: 'auth' for authentication, 'auth-int' for integrity protection or - # both - # :UseOpaque:: Should the server send opaque values to the client? This - # helps prevent replay attacks. - # :CheckNc:: Should the server check the nonce count? This helps the - # server detect replay attacks. - # :UseAuthenticationInfoHeader:: Should the server send an - # AuthenticationInfo header? - # :AutoReloadUserDB:: Reload the user database provided by :UserDB - # automatically? - # :NonceExpirePeriod:: How long should we store used nonces? Default is - # 30 minutes. - # :NonceExpireDelta:: How long is a nonce valid? Default is 1 minute - # :InternetExplorerHack:: Hack which allows Internet Explorer to work. - # :OperaHack:: Hack which allows Opera to work. - - DigestAuth = { - :Algorithm => 'MD5-sess', # or 'MD5' - :Domain => nil, # an array includes domain names. - :Qop => [ 'auth' ], # 'auth' or 'auth-int' or both. - :UseOpaque => true, - :UseNextNonce => false, - :CheckNc => false, - :UseAuthenticationInfoHeader => true, - :AutoReloadUserDB => true, - :NonceExpirePeriod => 30*60, - :NonceExpireDelta => 60, - :InternetExplorerHack => true, - :OperaHack => true, - } - end -end diff --git a/tool/lib/webrick/cookie.rb b/tool/lib/webrick/cookie.rb deleted file mode 100644 index 5fd3bfb228..0000000000 --- a/tool/lib/webrick/cookie.rb +++ /dev/null @@ -1,172 +0,0 @@ -# frozen_string_literal: false -# -# cookie.rb -- Cookie class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cookie.rb,v 1.16 2002/09/21 12:23:35 gotoyuzo Exp $ - -require 'time' -require_relative 'httputils' - -module WEBrick - - ## - # Processes HTTP cookies - - class Cookie - - ## - # The cookie name - - attr_reader :name - - ## - # The cookie value - - attr_accessor :value - - ## - # The cookie version - - attr_accessor :version - - ## - # The cookie domain - attr_accessor :domain - - ## - # The cookie path - - attr_accessor :path - - ## - # Is this a secure cookie? - - attr_accessor :secure - - ## - # The cookie comment - - attr_accessor :comment - - ## - # The maximum age of the cookie - - attr_accessor :max_age - - #attr_accessor :comment_url, :discard, :port - - ## - # Creates a new cookie with the given +name+ and +value+ - - def initialize(name, value) - @name = name - @value = value - @version = 0 # Netscape Cookie - - @domain = @path = @secure = @comment = @max_age = - @expires = @comment_url = @discard = @port = nil - end - - ## - # Sets the cookie expiration to the time +t+. The expiration time may be - # a false value to disable expiration or a Time or HTTP format time string - # to set the expiration date. - - def expires=(t) - @expires = t && (t.is_a?(Time) ? t.httpdate : t.to_s) - end - - ## - # Retrieves the expiration time as a Time - - def expires - @expires && Time.parse(@expires) - end - - ## - # The cookie string suitable for use in an HTTP header - - def to_s - ret = "" - ret << @name << "=" << @value - ret << "; " << "Version=" << @version.to_s if @version > 0 - ret << "; " << "Domain=" << @domain if @domain - ret << "; " << "Expires=" << @expires if @expires - ret << "; " << "Max-Age=" << @max_age.to_s if @max_age - ret << "; " << "Comment=" << @comment if @comment - ret << "; " << "Path=" << @path if @path - ret << "; " << "Secure" if @secure - ret - end - - ## - # Parses a Cookie field sent from the user-agent. Returns an array of - # cookies. - - def self.parse(str) - if str - ret = [] - cookie = nil - ver = 0 - str.split(/;\s+/).each{|x| - key, val = x.split(/=/,2) - val = val ? HTTPUtils::dequote(val) : "" - case key - when "$Version"; ver = val.to_i - when "$Path"; cookie.path = val - when "$Domain"; cookie.domain = val - when "$Port"; cookie.port = val - else - ret << cookie if cookie - cookie = self.new(key, val) - cookie.version = ver - end - } - ret << cookie if cookie - ret - end - end - - ## - # Parses the cookie in +str+ - - def self.parse_set_cookie(str) - cookie_elem = str.split(/;/) - first_elem = cookie_elem.shift - first_elem.strip! - key, value = first_elem.split(/=/, 2) - cookie = new(key, HTTPUtils.dequote(value)) - cookie_elem.each{|pair| - pair.strip! - key, value = pair.split(/=/, 2) - if value - value = HTTPUtils.dequote(value.strip) - end - case key.downcase - when "domain" then cookie.domain = value - when "path" then cookie.path = value - when "expires" then cookie.expires = value - when "max-age" then cookie.max_age = Integer(value) - when "comment" then cookie.comment = value - when "version" then cookie.version = Integer(value) - when "secure" then cookie.secure = true - end - } - return cookie - end - - ## - # Parses the cookies in +str+ - - def self.parse_set_cookies(str) - return str.split(/,(?=[^;,]*=)|,$/).collect{|c| - parse_set_cookie(c) - } - end - end -end diff --git a/tool/lib/webrick/htmlutils.rb b/tool/lib/webrick/htmlutils.rb deleted file mode 100644 index ed9f4ac0d3..0000000000 --- a/tool/lib/webrick/htmlutils.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: false -#-- -# htmlutils.rb -- HTMLUtils Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htmlutils.rb,v 1.7 2002/09/21 12:23:35 gotoyuzo Exp $ - -module WEBrick - module HTMLUtils - - ## - # Escapes &, ", > and < in +string+ - - def escape(string) - return "" unless string - str = string.b - str.gsub!(/&/n, '&') - str.gsub!(/\"/n, '"') - str.gsub!(/>/n, '>') - str.gsub!(/</n, '<') - str.force_encoding(string.encoding) - end - module_function :escape - - end -end diff --git a/tool/lib/webrick/httpauth.rb b/tool/lib/webrick/httpauth.rb deleted file mode 100644 index f8bf09a6f1..0000000000 --- a/tool/lib/webrick/httpauth.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: false -# -# httpauth.rb -- HTTP access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpauth.rb,v 1.14 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'httpauth/basicauth' -require_relative 'httpauth/digestauth' -require_relative 'httpauth/htpasswd' -require_relative 'httpauth/htdigest' -require_relative 'httpauth/htgroup' - -module WEBrick - - ## - # HTTPAuth provides both basic and digest authentication. - # - # To enable authentication for requests in WEBrick you will need a user - # database and an authenticator. To start, here's an Htpasswd database for - # use with a DigestAuth authenticator: - # - # config = { :Realm => 'DigestAuth example realm' } - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' - # htpasswd.auth_type = WEBrick::HTTPAuth::DigestAuth - # htpasswd.set_passwd config[:Realm], 'username', 'password' - # htpasswd.flush - # - # The +:Realm+ is used to provide different access to different groups - # across several resources on a server. Typically you'll need only one - # realm for a server. - # - # This database can be used to create an authenticator: - # - # config[:UserDB] = htpasswd - # - # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config - # - # To authenticate a request call #authenticate with a request and response - # object in a servlet: - # - # def do_GET req, res - # @authenticator.authenticate req, res - # end - # - # For digest authentication the authenticator must not be created every - # request, it must be passed in as an option via WEBrick::HTTPServer#mount. - - module HTTPAuth - module_function - - def _basic_auth(req, res, realm, req_field, res_field, err_type, - block) # :nodoc: - user = pass = nil - if /^Basic\s+(.*)/o =~ req[req_field] - userpass = $1 - user, pass = userpass.unpack("m*")[0].split(":", 2) - end - if block.call(user, pass) - req.user = user - return - end - res[res_field] = "Basic realm=\"#{realm}\"" - raise err_type - end - - ## - # Simple wrapper for providing basic authentication for a request. When - # called with a request +req+, response +res+, authentication +realm+ and - # +block+ the block will be called with a +username+ and +password+. If - # the block returns true the request is allowed to continue, otherwise an - # HTTPStatus::Unauthorized error is raised. - - def basic_auth(req, res, realm, &block) # :yield: username, password - _basic_auth(req, res, realm, "Authorization", "WWW-Authenticate", - HTTPStatus::Unauthorized, block) - end - - ## - # Simple wrapper for providing basic authentication for a proxied request. - # When called with a request +req+, response +res+, authentication +realm+ - # and +block+ the block will be called with a +username+ and +password+. - # If the block returns true the request is allowed to continue, otherwise - # an HTTPStatus::ProxyAuthenticationRequired error is raised. - - def proxy_basic_auth(req, res, realm, &block) # :yield: username, password - _basic_auth(req, res, realm, "Proxy-Authorization", "Proxy-Authenticate", - HTTPStatus::ProxyAuthenticationRequired, block) - end - end -end diff --git a/tool/lib/webrick/httpauth/authenticator.rb b/tool/lib/webrick/httpauth/authenticator.rb deleted file mode 100644 index 8f0eaa3aca..0000000000 --- a/tool/lib/webrick/httpauth/authenticator.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: false -#-- -# httpauth/authenticator.rb -- Authenticator mix-in module. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: authenticator.rb,v 1.3 2003/02/20 07:15:47 gotoyuzo Exp $ - -module WEBrick - module HTTPAuth - - ## - # Module providing generic support for both Digest and Basic - # authentication schemes. - - module Authenticator - - RequestField = "Authorization" # :nodoc: - ResponseField = "WWW-Authenticate" # :nodoc: - ResponseInfoField = "Authentication-Info" # :nodoc: - AuthException = HTTPStatus::Unauthorized # :nodoc: - - ## - # Method of authentication, must be overridden by the including class - - AuthScheme = nil - - ## - # The realm this authenticator covers - - attr_reader :realm - - ## - # The user database for this authenticator - - attr_reader :userdb - - ## - # The logger for this authenticator - - attr_reader :logger - - private - - # :stopdoc: - - ## - # Initializes the authenticator from +config+ - - def check_init(config) - [:UserDB, :Realm].each{|sym| - unless config[sym] - raise ArgumentError, "Argument #{sym.inspect} missing." - end - } - @realm = config[:Realm] - @userdb = config[:UserDB] - @logger = config[:Logger] || Log::new($stderr) - @reload_db = config[:AutoReloadUserDB] - @request_field = self::class::RequestField - @response_field = self::class::ResponseField - @resp_info_field = self::class::ResponseInfoField - @auth_exception = self::class::AuthException - @auth_scheme = self::class::AuthScheme - end - - ## - # Ensures +req+ has credentials that can be authenticated. - - def check_scheme(req) - unless credentials = req[@request_field] - error("no credentials in the request.") - return nil - end - unless match = /^#{@auth_scheme}\s+/i.match(credentials) - error("invalid scheme in %s.", credentials) - info("%s: %s", @request_field, credentials) if $DEBUG - return nil - end - return match.post_match - end - - def log(meth, fmt, *args) - msg = format("%s %s: ", @auth_scheme, @realm) - msg << fmt % args - @logger.__send__(meth, msg) - end - - def error(fmt, *args) - if @logger.error? - log(:error, fmt, *args) - end - end - - def info(fmt, *args) - if @logger.info? - log(:info, fmt, *args) - end - end - - # :startdoc: - end - - ## - # Module providing generic support for both Digest and Basic - # authentication schemes for proxies. - - module ProxyAuthenticator - RequestField = "Proxy-Authorization" # :nodoc: - ResponseField = "Proxy-Authenticate" # :nodoc: - InfoField = "Proxy-Authentication-Info" # :nodoc: - AuthException = HTTPStatus::ProxyAuthenticationRequired # :nodoc: - end - end -end diff --git a/tool/lib/webrick/httpauth/basicauth.rb b/tool/lib/webrick/httpauth/basicauth.rb deleted file mode 100644 index 7d0a9cfc8f..0000000000 --- a/tool/lib/webrick/httpauth/basicauth.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/basicauth.rb -- HTTP basic access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: basicauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ - -require_relative '../config' -require_relative '../httpstatus' -require_relative 'authenticator' - -module WEBrick - module HTTPAuth - - ## - # Basic Authentication for WEBrick - # - # Use this class to add basic authentication to a WEBrick servlet. - # - # Here is an example of how to set up a BasicAuth: - # - # config = { :Realm => 'BasicAuth example realm' } - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file', password_hash: :bcrypt - # htpasswd.set_passwd config[:Realm], 'username', 'password' - # htpasswd.flush - # - # config[:UserDB] = htpasswd - # - # basic_auth = WEBrick::HTTPAuth::BasicAuth.new config - - class BasicAuth - include Authenticator - - AuthScheme = "Basic" # :nodoc: - - ## - # Used by UserDB to create a basic password entry - - def self.make_passwd(realm, user, pass) - pass ||= "" - pass.crypt(Utils::random_string(2)) - end - - attr_reader :realm, :userdb, :logger - - ## - # Creates a new BasicAuth instance. - # - # See WEBrick::Config::BasicAuth for default configuration entries - # - # You must supply the following configuration entries: - # - # :Realm:: The name of the realm being protected. - # :UserDB:: A database of usernames and passwords. - # A WEBrick::HTTPAuth::Htpasswd instance should be used. - - def initialize(config, default=Config::BasicAuth) - check_init(config) - @config = default.dup.update(config) - end - - ## - # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if - # the authentication was not correct. - - def authenticate(req, res) - unless basic_credentials = check_scheme(req) - challenge(req, res) - end - userid, password = basic_credentials.unpack("m*")[0].split(":", 2) - password ||= "" - if userid.empty? - error("user id was not given.") - challenge(req, res) - end - unless encpass = @userdb.get_passwd(@realm, userid, @reload_db) - error("%s: the user is not allowed.", userid) - challenge(req, res) - end - - case encpass - when /\A\$2[aby]\$/ - password_matches = BCrypt::Password.new(encpass.sub(/\A\$2[aby]\$/, '$2a$')) == password - else - password_matches = password.crypt(encpass) == encpass - end - - unless password_matches - error("%s: password unmatch.", userid) - challenge(req, res) - end - info("%s: authentication succeeded.", userid) - req.user = userid - end - - ## - # Returns a challenge response which asks for authentication information - - def challenge(req, res) - res[@response_field] = "#{@auth_scheme} realm=\"#{@realm}\"" - raise @auth_exception - end - end - - ## - # Basic authentication for proxy servers. See BasicAuth for details. - - class ProxyBasicAuth < BasicAuth - include ProxyAuthenticator - end - end -end diff --git a/tool/lib/webrick/httpauth/digestauth.rb b/tool/lib/webrick/httpauth/digestauth.rb deleted file mode 100644 index 3cf12899d2..0000000000 --- a/tool/lib/webrick/httpauth/digestauth.rb +++ /dev/null @@ -1,395 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/digestauth.rb -- HTTP digest access authentication -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. -# Copyright (c) 2003 H.M. -# -# The original implementation is provided by H.M. -# URL: http://rwiki.jin.gr.jp/cgi-bin/rw-cgi.rb?cmd=view;name= -# %C7%A7%BE%DA%B5%A1%C7%BD%A4%F2%B2%FE%C2%A4%A4%B7%A4%C6%A4%DF%A4%EB -# -# $IPR: digestauth.rb,v 1.5 2003/02/20 07:15:47 gotoyuzo Exp $ - -require_relative '../config' -require_relative '../httpstatus' -require_relative 'authenticator' -require 'digest/md5' -require 'digest/sha1' - -module WEBrick - module HTTPAuth - - ## - # RFC 2617 Digest Access Authentication for WEBrick - # - # Use this class to add digest authentication to a WEBrick servlet. - # - # Here is an example of how to set up DigestAuth: - # - # config = { :Realm => 'DigestAuth example realm' } - # - # htdigest = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' - # htdigest.set_passwd config[:Realm], 'username', 'password' - # htdigest.flush - # - # config[:UserDB] = htdigest - # - # digest_auth = WEBrick::HTTPAuth::DigestAuth.new config - # - # When using this as with a servlet be sure not to create a new DigestAuth - # object in the servlet's #initialize. By default WEBrick creates a new - # servlet instance for every request and the DigestAuth object must be - # used across requests. - - class DigestAuth - include Authenticator - - AuthScheme = "Digest" # :nodoc: - - ## - # Struct containing the opaque portion of the digest authentication - - OpaqueInfo = Struct.new(:time, :nonce, :nc) # :nodoc: - - ## - # Digest authentication algorithm - - attr_reader :algorithm - - ## - # Quality of protection. RFC 2617 defines "auth" and "auth-int" - - attr_reader :qop - - ## - # Used by UserDB to create a digest password entry - - def self.make_passwd(realm, user, pass) - pass ||= "" - Digest::MD5::hexdigest([user, realm, pass].join(":")) - end - - ## - # Creates a new DigestAuth instance. Be sure to use the same DigestAuth - # instance for multiple requests as it saves state between requests in - # order to perform authentication. - # - # See WEBrick::Config::DigestAuth for default configuration entries - # - # You must supply the following configuration entries: - # - # :Realm:: The name of the realm being protected. - # :UserDB:: A database of usernames and passwords. - # A WEBrick::HTTPAuth::Htdigest instance should be used. - - def initialize(config, default=Config::DigestAuth) - check_init(config) - @config = default.dup.update(config) - @algorithm = @config[:Algorithm] - @domain = @config[:Domain] - @qop = @config[:Qop] - @use_opaque = @config[:UseOpaque] - @use_next_nonce = @config[:UseNextNonce] - @check_nc = @config[:CheckNc] - @use_auth_info_header = @config[:UseAuthenticationInfoHeader] - @nonce_expire_period = @config[:NonceExpirePeriod] - @nonce_expire_delta = @config[:NonceExpireDelta] - @internet_explorer_hack = @config[:InternetExplorerHack] - - case @algorithm - when 'MD5','MD5-sess' - @h = Digest::MD5 - when 'SHA1','SHA1-sess' # it is a bonus feature :-) - @h = Digest::SHA1 - else - msg = format('Algorithm "%s" is not supported.', @algorithm) - raise ArgumentError.new(msg) - end - - @instance_key = hexdigest(self.__id__, Time.now.to_i, Process.pid) - @opaques = {} - @last_nonce_expire = Time.now - @mutex = Thread::Mutex.new - end - - ## - # Authenticates a +req+ and returns a 401 Unauthorized using +res+ if - # the authentication was not correct. - - def authenticate(req, res) - unless result = @mutex.synchronize{ _authenticate(req, res) } - challenge(req, res) - end - if result == :nonce_is_stale - challenge(req, res, true) - end - return true - end - - ## - # Returns a challenge response which asks for authentication information - - def challenge(req, res, stale=false) - nonce = generate_next_nonce(req) - if @use_opaque - opaque = generate_opaque(req) - @opaques[opaque].nonce = nonce - end - - param = Hash.new - param["realm"] = HTTPUtils::quote(@realm) - param["domain"] = HTTPUtils::quote(@domain.to_a.join(" ")) if @domain - param["nonce"] = HTTPUtils::quote(nonce) - param["opaque"] = HTTPUtils::quote(opaque) if opaque - param["stale"] = stale.to_s - param["algorithm"] = @algorithm - param["qop"] = HTTPUtils::quote(@qop.to_a.join(",")) if @qop - - res[@response_field] = - "#{@auth_scheme} " + param.map{|k,v| "#{k}=#{v}" }.join(", ") - info("%s: %s", @response_field, res[@response_field]) if $DEBUG - raise @auth_exception - end - - private - - # :stopdoc: - - MustParams = ['username','realm','nonce','uri','response'] - MustParamsAuth = ['cnonce','nc'] - - def _authenticate(req, res) - unless digest_credentials = check_scheme(req) - return false - end - - auth_req = split_param_value(digest_credentials) - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - req_params = MustParams + MustParamsAuth - else - req_params = MustParams - end - req_params.each{|key| - unless auth_req.has_key?(key) - error('%s: parameter missing. "%s"', auth_req['username'], key) - raise HTTPStatus::BadRequest - end - } - - if !check_uri(req, auth_req) - raise HTTPStatus::BadRequest - end - - if auth_req['realm'] != @realm - error('%s: realm unmatch. "%s" for "%s"', - auth_req['username'], auth_req['realm'], @realm) - return false - end - - auth_req['algorithm'] ||= 'MD5' - if auth_req['algorithm'].upcase != @algorithm.upcase - error('%s: algorithm unmatch. "%s" for "%s"', - auth_req['username'], auth_req['algorithm'], @algorithm) - return false - end - - if (@qop.nil? && auth_req.has_key?('qop')) || - (@qop && (! @qop.member?(auth_req['qop']))) - error('%s: the qop is not allowed. "%s"', - auth_req['username'], auth_req['qop']) - return false - end - - password = @userdb.get_passwd(@realm, auth_req['username'], @reload_db) - unless password - error('%s: the user is not allowed.', auth_req['username']) - return false - end - - nonce_is_invalid = false - if @use_opaque - info("@opaque = %s", @opaque.inspect) if $DEBUG - if !(opaque = auth_req['opaque']) - error('%s: opaque is not given.', auth_req['username']) - nonce_is_invalid = true - elsif !(opaque_struct = @opaques[opaque]) - error('%s: invalid opaque is given.', auth_req['username']) - nonce_is_invalid = true - elsif !check_opaque(opaque_struct, req, auth_req) - @opaques.delete(auth_req['opaque']) - nonce_is_invalid = true - end - elsif !check_nonce(req, auth_req) - nonce_is_invalid = true - end - - if /-sess$/i =~ auth_req['algorithm'] - ha1 = hexdigest(password, auth_req['nonce'], auth_req['cnonce']) - else - ha1 = password - end - - if auth_req['qop'] == "auth" || auth_req['qop'] == nil - ha2 = hexdigest(req.request_method, auth_req['uri']) - ha2_res = hexdigest("", auth_req['uri']) - elsif auth_req['qop'] == "auth-int" - body_digest = @h.new - req.body { |chunk| body_digest.update(chunk) } - body_digest = body_digest.hexdigest - ha2 = hexdigest(req.request_method, auth_req['uri'], body_digest) - ha2_res = hexdigest("", auth_req['uri'], body_digest) - end - - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - param2 = ['nonce', 'nc', 'cnonce', 'qop'].map{|key| - auth_req[key] - }.join(':') - digest = hexdigest(ha1, param2, ha2) - digest_res = hexdigest(ha1, param2, ha2_res) - else - digest = hexdigest(ha1, auth_req['nonce'], ha2) - digest_res = hexdigest(ha1, auth_req['nonce'], ha2_res) - end - - if digest != auth_req['response'] - error("%s: digest unmatch.", auth_req['username']) - return false - elsif nonce_is_invalid - error('%s: digest is valid, but nonce is not valid.', - auth_req['username']) - return :nonce_is_stale - elsif @use_auth_info_header - auth_info = { - 'nextnonce' => generate_next_nonce(req), - 'rspauth' => digest_res - } - if @use_opaque - opaque_struct.time = req.request_time - opaque_struct.nonce = auth_info['nextnonce'] - opaque_struct.nc = "%08x" % (auth_req['nc'].hex + 1) - end - if auth_req['qop'] == "auth" || auth_req['qop'] == "auth-int" - ['qop','cnonce','nc'].each{|key| - auth_info[key] = auth_req[key] - } - end - res[@resp_info_field] = auth_info.keys.map{|key| - if key == 'nc' - key + '=' + auth_info[key] - else - key + "=" + HTTPUtils::quote(auth_info[key]) - end - }.join(', ') - end - info('%s: authentication succeeded.', auth_req['username']) - req.user = auth_req['username'] - return true - end - - def split_param_value(string) - ret = {} - string.scan(/\G\s*([\w\-.*%!]+)=\s*(?:\"((?>\\.|[^\"])*)\"|([^,\"]*))\s*,?/) do - ret[$1] = $3 || $2.gsub(/\\(.)/, "\\1") - end - ret - end - - def generate_next_nonce(req) - now = "%012d" % req.request_time.to_i - pk = hexdigest(now, @instance_key)[0,32] - nonce = [now + ":" + pk].pack("m0") # it has 60 length of chars. - nonce - end - - def check_nonce(req, auth_req) - username = auth_req['username'] - nonce = auth_req['nonce'] - - pub_time, pk = nonce.unpack("m*")[0].split(":", 2) - if (!pub_time || !pk) - error("%s: empty nonce is given", username) - return false - elsif (hexdigest(pub_time, @instance_key)[0,32] != pk) - error("%s: invalid private-key: %s for %s", - username, hexdigest(pub_time, @instance_key)[0,32], pk) - return false - end - - diff_time = req.request_time.to_i - pub_time.to_i - if (diff_time < 0) - error("%s: difference of time-stamp is negative.", username) - return false - elsif diff_time > @nonce_expire_period - error("%s: nonce is expired.", username) - return false - end - - return true - end - - def generate_opaque(req) - @mutex.synchronize{ - now = req.request_time - if now - @last_nonce_expire > @nonce_expire_delta - @opaques.delete_if{|key,val| - (now - val.time) > @nonce_expire_period - } - @last_nonce_expire = now - end - begin - opaque = Utils::random_string(16) - end while @opaques[opaque] - @opaques[opaque] = OpaqueInfo.new(now, nil, '00000001') - opaque - } - end - - def check_opaque(opaque_struct, req, auth_req) - if (@use_next_nonce && auth_req['nonce'] != opaque_struct.nonce) - error('%s: nonce unmatched. "%s" for "%s"', - auth_req['username'], auth_req['nonce'], opaque_struct.nonce) - return false - elsif !check_nonce(req, auth_req) - return false - end - if (@check_nc && auth_req['nc'] != opaque_struct.nc) - error('%s: nc unmatched."%s" for "%s"', - auth_req['username'], auth_req['nc'], opaque_struct.nc) - return false - end - true - end - - def check_uri(req, auth_req) - uri = auth_req['uri'] - if uri != req.request_uri.to_s && uri != req.unparsed_uri && - (@internet_explorer_hack && uri != req.path) - error('%s: uri unmatch. "%s" for "%s"', auth_req['username'], - auth_req['uri'], req.request_uri.to_s) - return false - end - true - end - - def hexdigest(*args) - @h.hexdigest(args.join(":")) - end - - # :startdoc: - end - - ## - # Digest authentication for proxy servers. See DigestAuth for details. - - class ProxyDigestAuth < DigestAuth - include ProxyAuthenticator - - private - def check_uri(req, auth_req) # :nodoc: - return true - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htdigest.rb b/tool/lib/webrick/httpauth/htdigest.rb deleted file mode 100644 index 93b18e2c75..0000000000 --- a/tool/lib/webrick/httpauth/htdigest.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htdigest.rb -- Apache compatible htdigest file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htdigest.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ - -require_relative 'userdb' -require_relative 'digestauth' -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htdigest accesses apache-compatible digest password files. Passwords are - # matched to a realm where they are valid. For security, the path for a - # digest password database should be stored outside of the paths available - # to the HTTP server. - # - # Htdigest is intended for use with WEBrick::HTTPAuth::DigestAuth and - # stores passwords using cryptographic hashes. - # - # htpasswd = WEBrick::HTTPAuth::Htdigest.new 'my_password_file' - # htpasswd.set_passwd 'my realm', 'username', 'password' - # htpasswd.flush - - class Htdigest - include UserDB - - ## - # Open a digest password database at +path+ - - def initialize(path) - @path = path - @mtime = Time.at(0) - @digest = Hash.new - @mutex = Thread::Mutex::new - @auth_type = DigestAuth - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reloads passwords from the database - - def reload - mtime = File::mtime(@path) - if mtime > @mtime - @digest.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - user, realm, pass = line.split(/:/, 3) - unless @digest[realm] - @digest[realm] = Hash.new - end - @digest[realm][user] = pass - end - } - @mtime = mtime - end - end - - ## - # Flush the password database. If +output+ is given the database will - # be written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htpasswd", File::dirname(output)) - renamed = false - begin - each{|item| tmp.puts(item.join(":")) } - tmp.close - File::rename(tmp.path, output) - renamed = true - ensure - tmp.close - File.unlink(tmp.path) if !renamed - end - end - - ## - # Retrieves a password from the database for +user+ in +realm+. If - # +reload_db+ is true the database will be reloaded first. - - def get_passwd(realm, user, reload_db) - reload() if reload_db - if hash = @digest[realm] - hash[user] - end - end - - ## - # Sets a password in the database for +user+ in +realm+ to +pass+. - - def set_passwd(realm, user, pass) - @mutex.synchronize{ - unless @digest[realm] - @digest[realm] = Hash.new - end - @digest[realm][user] = make_passwd(realm, user, pass) - } - end - - ## - # Removes a password from the database for +user+ in +realm+. - - def delete_passwd(realm, user) - if hash = @digest[realm] - hash.delete(user) - end - end - - ## - # Iterate passwords in the database. - - def each # :yields: [user, realm, password_hash] - @digest.keys.sort.each{|realm| - hash = @digest[realm] - hash.keys.sort.each{|user| - yield([user, realm, hash[user]]) - } - } - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htgroup.rb b/tool/lib/webrick/httpauth/htgroup.rb deleted file mode 100644 index e06c441b18..0000000000 --- a/tool/lib/webrick/httpauth/htgroup.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htgroup.rb -- Apache compatible htgroup file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htgroup.rb,v 1.1 2003/02/16 22:22:56 gotoyuzo Exp $ - -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htgroup accesses apache-compatible group files. Htgroup can be used to - # provide group-based authentication for users. Currently Htgroup is not - # directly integrated with any authenticators in WEBrick. For security, - # the path for a digest password database should be stored outside of the - # paths available to the HTTP server. - # - # Example: - # - # htgroup = WEBrick::HTTPAuth::Htgroup.new 'my_group_file' - # htgroup.add 'superheroes', %w[spiderman batman] - # - # htgroup.members('superheroes').include? 'magneto' # => false - - class Htgroup - - ## - # Open a group database at +path+ - - def initialize(path) - @path = path - @mtime = Time.at(0) - @group = Hash.new - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reload groups from the database - - def reload - if (mtime = File::mtime(@path)) > @mtime - @group.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - group, members = line.split(/:\s*/) - @group[group] = members.split(/\s+/) - end - } - @mtime = mtime - end - end - - ## - # Flush the group database. If +output+ is given the database will be - # written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htgroup", File::dirname(output)) - begin - @group.keys.sort.each{|group| - tmp.puts(format("%s: %s", group, self.members(group).join(" "))) - } - ensure - tmp.close - if $! - File.unlink(tmp.path) - else - return File.rename(tmp.path, output) - end - end - end - - ## - # Retrieve the list of members from +group+ - - def members(group) - reload - @group[group] || [] - end - - ## - # Add an Array of +members+ to +group+ - - def add(group, members) - @group[group] = members(group) | members - end - end - end -end diff --git a/tool/lib/webrick/httpauth/htpasswd.rb b/tool/lib/webrick/httpauth/htpasswd.rb deleted file mode 100644 index abca30532e..0000000000 --- a/tool/lib/webrick/httpauth/htpasswd.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: false -# -# httpauth/htpasswd -- Apache compatible htpasswd file -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: htpasswd.rb,v 1.4 2003/07/22 19:20:45 gotoyuzo Exp $ - -require_relative 'userdb' -require_relative 'basicauth' -require 'tempfile' - -module WEBrick - module HTTPAuth - - ## - # Htpasswd accesses apache-compatible password files. Passwords are - # matched to a realm where they are valid. For security, the path for a - # password database should be stored outside of the paths available to the - # HTTP server. - # - # Htpasswd is intended for use with WEBrick::HTTPAuth::BasicAuth. - # - # To create an Htpasswd database with a single user: - # - # htpasswd = WEBrick::HTTPAuth::Htpasswd.new 'my_password_file' - # htpasswd.set_passwd 'my realm', 'username', 'password' - # htpasswd.flush - - class Htpasswd - include UserDB - - ## - # Open a password database at +path+ - - def initialize(path, password_hash: nil) - @path = path - @mtime = Time.at(0) - @passwd = Hash.new - @auth_type = BasicAuth - @password_hash = password_hash - - case @password_hash - when nil - # begin - # require "string/crypt" - # rescue LoadError - # warn("Unable to load string/crypt, proceeding with deprecated use of String#crypt, consider using password_hash: :bcrypt") - # end - @password_hash = :crypt - when :crypt - # require "string/crypt" - when :bcrypt - require "bcrypt" - else - raise ArgumentError, "only :crypt and :bcrypt are supported for password_hash keyword argument" - end - - File.open(@path,"a").close unless File.exist?(@path) - reload - end - - ## - # Reload passwords from the database - - def reload - mtime = File::mtime(@path) - if mtime > @mtime - @passwd.clear - File.open(@path){|io| - while line = io.gets - line.chomp! - case line - when %r!\A[^:]+:[a-zA-Z0-9./]{13}\z! - if @password_hash == :bcrypt - raise StandardError, ".htpasswd file contains crypt password, only bcrypt passwords supported" - end - user, pass = line.split(":") - when %r!\A[^:]+:\$2[aby]\$\d{2}\$.{53}\z! - if @password_hash == :crypt - raise StandardError, ".htpasswd file contains bcrypt password, only crypt passwords supported" - end - user, pass = line.split(":") - when /:\$/, /:{SHA}/ - raise NotImplementedError, - 'MD5, SHA1 .htpasswd file not supported' - else - raise StandardError, 'bad .htpasswd file' - end - @passwd[user] = pass - end - } - @mtime = mtime - end - end - - ## - # Flush the password database. If +output+ is given the database will - # be written there instead of to the original path. - - def flush(output=nil) - output ||= @path - tmp = Tempfile.create("htpasswd", File::dirname(output)) - renamed = false - begin - each{|item| tmp.puts(item.join(":")) } - tmp.close - File::rename(tmp.path, output) - renamed = true - ensure - tmp.close - File.unlink(tmp.path) if !renamed - end - end - - ## - # Retrieves a password from the database for +user+ in +realm+. If - # +reload_db+ is true the database will be reloaded first. - - def get_passwd(realm, user, reload_db) - reload() if reload_db - @passwd[user] - end - - ## - # Sets a password in the database for +user+ in +realm+ to +pass+. - - def set_passwd(realm, user, pass) - if @password_hash == :bcrypt - # Cost of 5 to match Apache default, and because the - # bcrypt default of 10 will introduce significant delays - # for every request. - @passwd[user] = BCrypt::Password.create(pass, :cost=>5) - else - @passwd[user] = make_passwd(realm, user, pass) - end - end - - ## - # Removes a password from the database for +user+ in +realm+. - - def delete_passwd(realm, user) - @passwd.delete(user) - end - - ## - # Iterate passwords in the database. - - def each # :yields: [user, password] - @passwd.keys.sort.each{|user| - yield([user, @passwd[user]]) - } - end - end - end -end diff --git a/tool/lib/webrick/httpauth/userdb.rb b/tool/lib/webrick/httpauth/userdb.rb deleted file mode 100644 index 7a17715cdf..0000000000 --- a/tool/lib/webrick/httpauth/userdb.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: false -#-- -# httpauth/userdb.rb -- UserDB mix-in module. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: userdb.rb,v 1.2 2003/02/20 07:15:48 gotoyuzo Exp $ - -module WEBrick - module HTTPAuth - - ## - # User database mixin for HTTPAuth. This mixin dispatches user record - # access to the underlying auth_type for this database. - - module UserDB - - ## - # The authentication type. - # - # WEBrick::HTTPAuth::BasicAuth or WEBrick::HTTPAuth::DigestAuth are - # built-in. - - attr_accessor :auth_type - - ## - # Creates an obscured password in +realm+ with +user+ and +password+ - # using the auth_type of this database. - - def make_passwd(realm, user, pass) - @auth_type::make_passwd(realm, user, pass) - end - - ## - # Sets a password in +realm+ with +user+ and +password+ for the - # auth_type of this database. - - def set_passwd(realm, user, pass) - self[user] = pass - end - - ## - # Retrieves a password in +realm+ for +user+ for the auth_type of this - # database. +reload_db+ is a dummy value. - - def get_passwd(realm, user, reload_db=false) - make_passwd(realm, user, self[user]) - end - end - end -end diff --git a/tool/lib/webrick/httpproxy.rb b/tool/lib/webrick/httpproxy.rb deleted file mode 100644 index 7607c3df88..0000000000 --- a/tool/lib/webrick/httpproxy.rb +++ /dev/null @@ -1,354 +0,0 @@ -# frozen_string_literal: false -# -# httpproxy.rb -- HTTPProxy Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 GOTO Kentaro -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpproxy.rb,v 1.18 2003/03/08 18:58:10 gotoyuzo Exp $ -# $kNotwork: straw.rb,v 1.3 2002/02/12 15:13:07 gotoken Exp $ - -require_relative "httpserver" -require "net/http" - -module WEBrick - - NullReader = Object.new # :nodoc: - class << NullReader # :nodoc: - def read(*args) - nil - end - alias gets read - end - - FakeProxyURI = Object.new # :nodoc: - class << FakeProxyURI # :nodoc: - def method_missing(meth, *args) - if %w(scheme host port path query userinfo).member?(meth.to_s) - return nil - end - super - end - end - - # :startdoc: - - ## - # An HTTP Proxy server which proxies GET, HEAD and POST requests. - # - # To create a simple proxy server: - # - # require 'webrick' - # require 'webrick/httpproxy' - # - # proxy = WEBrick::HTTPProxyServer.new Port: 8000 - # - # trap 'INT' do proxy.shutdown end - # trap 'TERM' do proxy.shutdown end - # - # proxy.start - # - # See ::new for proxy-specific configuration items. - # - # == Modifying proxied responses - # - # To modify content the proxy server returns use the +:ProxyContentHandler+ - # option: - # - # handler = proc do |req, res| - # if res['content-type'] == 'text/plain' then - # res.body << "\nThis content was proxied!\n" - # end - # end - # - # proxy = - # WEBrick::HTTPProxyServer.new Port: 8000, ProxyContentHandler: handler - - class HTTPProxyServer < HTTPServer - - ## - # Proxy server configurations. The proxy server handles the following - # configuration items in addition to those supported by HTTPServer: - # - # :ProxyAuthProc:: Called with a request and response to authorize a - # request - # :ProxyVia:: Appended to the via header - # :ProxyURI:: The proxy server's URI - # :ProxyContentHandler:: Called with a request and response and allows - # modification of the response - # :ProxyTimeout:: Sets the proxy timeouts to 30 seconds for open and 60 - # seconds for read operations - - def initialize(config={}, default=Config::HTTP) - super(config, default) - c = @config - @via = "#{c[:HTTPVersion]} #{c[:ServerName]}:#{c[:Port]}" - end - - # :stopdoc: - def service(req, res) - if req.request_method == "CONNECT" - do_CONNECT(req, res) - elsif req.unparsed_uri =~ %r!^http://! - proxy_service(req, res) - else - super(req, res) - end - end - - def proxy_auth(req, res) - if proc = @config[:ProxyAuthProc] - proc.call(req, res) - end - req.header.delete("proxy-authorization") - end - - def proxy_uri(req, res) - # should return upstream proxy server's URI - return @config[:ProxyURI] - end - - def proxy_service(req, res) - # Proxy Authentication - proxy_auth(req, res) - - begin - public_send("do_#{req.request_method}", req, res) - rescue NoMethodError - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - rescue => err - logger.debug("#{err.class}: #{err.message}") - raise HTTPStatus::ServiceUnavailable, err.message - end - - # Process contents - if handler = @config[:ProxyContentHandler] - handler.call(req, res) - end - end - - def do_CONNECT(req, res) - # Proxy Authentication - proxy_auth(req, res) - - ua = Thread.current[:WEBrickSocket] # User-Agent - raise HTTPStatus::InternalServerError, - "[BUG] cannot get socket" unless ua - - host, port = req.unparsed_uri.split(":", 2) - # Proxy authentication for upstream proxy server - if proxy = proxy_uri(req, res) - proxy_request_line = "CONNECT #{host}:#{port} HTTP/1.0" - if proxy.userinfo - credentials = "Basic " + [proxy.userinfo].pack("m0") - end - host, port = proxy.host, proxy.port - end - - begin - @logger.debug("CONNECT: upstream proxy is `#{host}:#{port}'.") - os = TCPSocket.new(host, port) # origin server - - if proxy - @logger.debug("CONNECT: sending a Request-Line") - os << proxy_request_line << CRLF - @logger.debug("CONNECT: > #{proxy_request_line}") - if credentials - @logger.debug("CONNECT: sending credentials") - os << "Proxy-Authorization: " << credentials << CRLF - end - os << CRLF - proxy_status_line = os.gets(LF) - @logger.debug("CONNECT: read Status-Line from the upstream server") - @logger.debug("CONNECT: < #{proxy_status_line}") - if %r{^HTTP/\d+\.\d+\s+200\s*} =~ proxy_status_line - while line = os.gets(LF) - break if /\A(#{CRLF}|#{LF})\z/om =~ line - end - else - raise HTTPStatus::BadGateway - end - end - @logger.debug("CONNECT #{host}:#{port}: succeeded") - res.status = HTTPStatus::RC_OK - rescue => ex - @logger.debug("CONNECT #{host}:#{port}: failed `#{ex.message}'") - res.set_error(ex) - raise HTTPStatus::EOFError - ensure - if handler = @config[:ProxyContentHandler] - handler.call(req, res) - end - res.send_response(ua) - access_log(@config, req, res) - - # Should clear request-line not to send the response twice. - # see: HTTPServer#run - req.parse(NullReader) rescue nil - end - - begin - while fds = IO::select([ua, os]) - if fds[0].member?(ua) - buf = ua.readpartial(1024); - @logger.debug("CONNECT: #{buf.bytesize} byte from User-Agent") - os.write(buf) - elsif fds[0].member?(os) - buf = os.readpartial(1024); - @logger.debug("CONNECT: #{buf.bytesize} byte from #{host}:#{port}") - ua.write(buf) - end - end - rescue - os.close - @logger.debug("CONNECT #{host}:#{port}: closed") - end - - raise HTTPStatus::EOFError - end - - def do_GET(req, res) - perform_proxy_request(req, res, Net::HTTP::Get) - end - - def do_HEAD(req, res) - perform_proxy_request(req, res, Net::HTTP::Head) - end - - def do_POST(req, res) - perform_proxy_request(req, res, Net::HTTP::Post, req.body_reader) - end - - def do_OPTIONS(req, res) - res['allow'] = "GET,HEAD,POST,OPTIONS,CONNECT" - end - - private - - # Some header fields should not be transferred. - HopByHop = %w( connection keep-alive proxy-authenticate upgrade - proxy-authorization te trailers transfer-encoding ) - ShouldNotTransfer = %w( set-cookie proxy-connection ) - def split_field(f) f ? f.split(/,\s+/).collect{|i| i.downcase } : [] end - - def choose_header(src, dst) - connections = split_field(src['connection']) - src.each{|key, value| - key = key.downcase - if HopByHop.member?(key) || # RFC2616: 13.5.1 - connections.member?(key) || # RFC2616: 14.10 - ShouldNotTransfer.member?(key) # pragmatics - @logger.debug("choose_header: `#{key}: #{value}'") - next - end - dst[key] = value - } - end - - # Net::HTTP is stupid about the multiple header fields. - # Here is workaround: - def set_cookie(src, dst) - if str = src['set-cookie'] - cookies = [] - str.split(/,\s*/).each{|token| - if /^[^=]+;/o =~ token - cookies[-1] << ", " << token - elsif /=/o =~ token - cookies << token - else - cookies[-1] << ", " << token - end - } - dst.cookies.replace(cookies) - end - end - - def set_via(h) - if @config[:ProxyVia] - if h['via'] - h['via'] << ", " << @via - else - h['via'] = @via - end - end - end - - def setup_proxy_header(req, res) - # Choose header fields to transfer - header = Hash.new - choose_header(req, header) - set_via(header) - return header - end - - def setup_upstream_proxy_authentication(req, res, header) - if upstream = proxy_uri(req, res) - if upstream.userinfo - header['proxy-authorization'] = - "Basic " + [upstream.userinfo].pack("m0") - end - return upstream - end - return FakeProxyURI - end - - def create_net_http(uri, upstream) - Net::HTTP.new(uri.host, uri.port, upstream.host, upstream.port) - end - - def perform_proxy_request(req, res, req_class, body_stream = nil) - uri = req.request_uri - path = uri.path.dup - path << "?" << uri.query if uri.query - header = setup_proxy_header(req, res) - upstream = setup_upstream_proxy_authentication(req, res, header) - - body_tmp = [] - http = create_net_http(uri, upstream) - req_fib = Fiber.new do - http.start do - if @config[:ProxyTimeout] - ################################## these issues are - http.open_timeout = 30 # secs # necessary (maybe because - http.read_timeout = 60 # secs # Ruby's bug, but why?) - ################################## - end - if body_stream && req['transfer-encoding'] =~ /\bchunked\b/i - header['Transfer-Encoding'] = 'chunked' - end - http_req = req_class.new(path, header) - http_req.body_stream = body_stream if body_stream - http.request(http_req) do |response| - # Persistent connection requirements are mysterious for me. - # So I will close the connection in every response. - res['proxy-connection'] = "close" - res['connection'] = "close" - - # stream Net::HTTP::HTTPResponse to WEBrick::HTTPResponse - res.status = response.code.to_i - res.chunked = response.chunked? - choose_header(response, res) - set_cookie(response, res) - set_via(res) - response.read_body do |buf| - body_tmp << buf - Fiber.yield # wait for res.body Proc#call - end - end # http.request - end - end - req_fib.resume # read HTTP response headers and first chunk of the body - res.body = ->(socket) do - while buf = body_tmp.shift - socket.write(buf) - buf.clear - req_fib.resume # continue response.read_body - end - end - end - # :stopdoc: - end -end diff --git a/tool/lib/webrick/httprequest.rb b/tool/lib/webrick/httprequest.rb deleted file mode 100644 index d34eac7ecf..0000000000 --- a/tool/lib/webrick/httprequest.rb +++ /dev/null @@ -1,636 +0,0 @@ -# frozen_string_literal: false -# -# httprequest.rb -- HTTPRequest Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httprequest.rb,v 1.64 2003/07/13 17:18:22 gotoyuzo Exp $ - -require 'fiber' -require 'uri' -require_relative 'httpversion' -require_relative 'httpstatus' -require_relative 'httputils' -require_relative 'cookie' - -module WEBrick - - ## - # An HTTP request. This is consumed by service and do_* methods in - # WEBrick servlets - - class HTTPRequest - - BODY_CONTAINABLE_METHODS = [ "POST", "PUT" ] # :nodoc: - - # :section: Request line - - ## - # The complete request line such as: - # - # GET / HTTP/1.1 - - attr_reader :request_line - - ## - # The request method, GET, POST, PUT, etc. - - attr_reader :request_method - - ## - # The unparsed URI of the request - - attr_reader :unparsed_uri - - ## - # The HTTP version of the request - - attr_reader :http_version - - # :section: Request-URI - - ## - # The parsed URI of the request - - attr_reader :request_uri - - ## - # The request path - - attr_reader :path - - ## - # The script name (CGI variable) - - attr_accessor :script_name - - ## - # The path info (CGI variable) - - attr_accessor :path_info - - ## - # The query from the URI of the request - - attr_accessor :query_string - - # :section: Header and entity body - - ## - # The raw header of the request - - attr_reader :raw_header - - ## - # The parsed header of the request - - attr_reader :header - - ## - # The parsed request cookies - - attr_reader :cookies - - ## - # The Accept header value - - attr_reader :accept - - ## - # The Accept-Charset header value - - attr_reader :accept_charset - - ## - # The Accept-Encoding header value - - attr_reader :accept_encoding - - ## - # The Accept-Language header value - - attr_reader :accept_language - - # :section: - - ## - # The remote user (CGI variable) - - attr_accessor :user - - ## - # The socket address of the server - - attr_reader :addr - - ## - # The socket address of the client - - attr_reader :peeraddr - - ## - # Hash of request attributes - - attr_reader :attributes - - ## - # Is this a keep-alive connection? - - attr_reader :keep_alive - - ## - # The local time this request was received - - attr_reader :request_time - - ## - # Creates a new HTTP request. WEBrick::Config::HTTP is the default - # configuration. - - def initialize(config) - @config = config - @buffer_size = @config[:InputBufferSize] - @logger = config[:Logger] - - @request_line = @request_method = - @unparsed_uri = @http_version = nil - - @request_uri = @host = @port = @path = nil - @script_name = @path_info = nil - @query_string = nil - @query = nil - @form_data = nil - - @raw_header = Array.new - @header = nil - @cookies = [] - @accept = [] - @accept_charset = [] - @accept_encoding = [] - @accept_language = [] - @body = "" - - @addr = @peeraddr = nil - @attributes = {} - @user = nil - @keep_alive = false - @request_time = nil - - @remaining_size = nil - @socket = nil - - @forwarded_proto = @forwarded_host = @forwarded_port = - @forwarded_server = @forwarded_for = nil - end - - ## - # Parses a request from +socket+. This is called internally by - # WEBrick::HTTPServer. - - def parse(socket=nil) - @socket = socket - begin - @peeraddr = socket.respond_to?(:peeraddr) ? socket.peeraddr : [] - @addr = socket.respond_to?(:addr) ? socket.addr : [] - rescue Errno::ENOTCONN - raise HTTPStatus::EOFError - end - - read_request_line(socket) - if @http_version.major > 0 - read_header(socket) - @header['cookie'].each{|cookie| - @cookies += Cookie::parse(cookie) - } - @accept = HTTPUtils.parse_qvalues(self['accept']) - @accept_charset = HTTPUtils.parse_qvalues(self['accept-charset']) - @accept_encoding = HTTPUtils.parse_qvalues(self['accept-encoding']) - @accept_language = HTTPUtils.parse_qvalues(self['accept-language']) - end - return if @request_method == "CONNECT" - return if @unparsed_uri == "*" - - begin - setup_forwarded_info - @request_uri = parse_uri(@unparsed_uri) - @path = HTTPUtils::unescape(@request_uri.path) - @path = HTTPUtils::normalize_path(@path) - @host = @request_uri.host - @port = @request_uri.port - @query_string = @request_uri.query - @script_name = "" - @path_info = @path.dup - rescue - raise HTTPStatus::BadRequest, "bad URI `#{@unparsed_uri}'." - end - - if /\Aclose\z/io =~ self["connection"] - @keep_alive = false - elsif /\Akeep-alive\z/io =~ self["connection"] - @keep_alive = true - elsif @http_version < "1.1" - @keep_alive = false - else - @keep_alive = true - end - end - - ## - # Generate HTTP/1.1 100 continue response if the client expects it, - # otherwise does nothing. - - def continue # :nodoc: - if self['expect'] == '100-continue' && @config[:HTTPVersion] >= "1.1" - @socket << "HTTP/#{@config[:HTTPVersion]} 100 continue#{CRLF}#{CRLF}" - @header.delete('expect') - end - end - - ## - # Returns the request body. - - def body(&block) # :yields: body_chunk - block ||= Proc.new{|chunk| @body << chunk } - read_body(@socket, block) - @body.empty? ? nil : @body - end - - ## - # Prepares the HTTPRequest object for use as the - # source for IO.copy_stream - - def body_reader - @body_tmp = [] - @body_rd = Fiber.new do - body do |buf| - @body_tmp << buf - Fiber.yield - end - end - @body_rd.resume # grab the first chunk and yield - self - end - - # for IO.copy_stream. - def readpartial(size, buf = ''.b) # :nodoc - res = @body_tmp.shift or raise EOFError, 'end of file reached' - if res.length > size - @body_tmp.unshift(res[size..-1]) - res = res[0..size - 1] - end - buf.replace(res) - res.clear - # get more chunks - check alive? because we can take a partial chunk - @body_rd.resume if @body_rd.alive? - buf - end - - ## - # Request query as a Hash - - def query - unless @query - parse_query() - end - @query - end - - ## - # The content-length header - - def content_length - return Integer(self['content-length']) - end - - ## - # The content-type header - - def content_type - return self['content-type'] - end - - ## - # Retrieves +header_name+ - - def [](header_name) - if @header - value = @header[header_name.downcase] - value.empty? ? nil : value.join(", ") - end - end - - ## - # Iterates over the request headers - - def each - if @header - @header.each{|k, v| - value = @header[k] - yield(k, value.empty? ? nil : value.join(", ")) - } - end - end - - ## - # The host this request is for - - def host - return @forwarded_host || @host - end - - ## - # The port this request is for - - def port - return @forwarded_port || @port - end - - ## - # The server name this request is for - - def server_name - return @forwarded_server || @config[:ServerName] - end - - ## - # The client's IP address - - def remote_ip - return self["client-ip"] || @forwarded_for || @peeraddr[3] - end - - ## - # Is this an SSL request? - - def ssl? - return @request_uri.scheme == "https" - end - - ## - # Should the connection this request was made on be kept alive? - - def keep_alive? - @keep_alive - end - - def to_s # :nodoc: - ret = @request_line.dup - @raw_header.each{|line| ret << line } - ret << CRLF - ret << body if body - ret - end - - ## - # Consumes any remaining body and updates keep-alive status - - def fixup() # :nodoc: - begin - body{|chunk| } # read remaining body - rescue HTTPStatus::Error => ex - @logger.error("HTTPRequest#fixup: #{ex.class} occurred.") - @keep_alive = false - rescue => ex - @logger.error(ex) - @keep_alive = false - end - end - - # This method provides the metavariables defined by the revision 3 - # of "The WWW Common Gateway Interface Version 1.1" - # To browse the current document of CGI Version 1.1, see below: - # http://tools.ietf.org/html/rfc3875 - - def meta_vars - meta = Hash.new - - cl = self["Content-Length"] - ct = self["Content-Type"] - meta["CONTENT_LENGTH"] = cl if cl.to_i > 0 - meta["CONTENT_TYPE"] = ct.dup if ct - meta["GATEWAY_INTERFACE"] = "CGI/1.1" - meta["PATH_INFO"] = @path_info ? @path_info.dup : "" - #meta["PATH_TRANSLATED"] = nil # no plan to be provided - meta["QUERY_STRING"] = @query_string ? @query_string.dup : "" - meta["REMOTE_ADDR"] = @peeraddr[3] - meta["REMOTE_HOST"] = @peeraddr[2] - #meta["REMOTE_IDENT"] = nil # no plan to be provided - meta["REMOTE_USER"] = @user - meta["REQUEST_METHOD"] = @request_method.dup - meta["REQUEST_URI"] = @request_uri.to_s - meta["SCRIPT_NAME"] = @script_name.dup - meta["SERVER_NAME"] = @host - meta["SERVER_PORT"] = @port.to_s - meta["SERVER_PROTOCOL"] = "HTTP/" + @config[:HTTPVersion].to_s - meta["SERVER_SOFTWARE"] = @config[:ServerSoftware].dup - - self.each{|key, val| - next if /^content-type$/i =~ key - next if /^content-length$/i =~ key - name = "HTTP_" + key - name.gsub!(/-/o, "_") - name.upcase! - meta[name] = val - } - - meta - end - - private - - # :stopdoc: - - MAX_URI_LENGTH = 2083 # :nodoc: - - # same as Mongrel, Thin and Puma - MAX_HEADER_LENGTH = (112 * 1024) # :nodoc: - - def read_request_line(socket) - @request_line = read_line(socket, MAX_URI_LENGTH) if socket - raise HTTPStatus::EOFError unless @request_line - - @request_bytes = @request_line.bytesize - if @request_bytes >= MAX_URI_LENGTH and @request_line[-1, 1] != LF - raise HTTPStatus::RequestURITooLarge - end - - @request_time = Time.now - if /^(\S+)\s+(\S++)(?:\s+HTTP\/(\d+\.\d+))?\r?\n/mo =~ @request_line - @request_method = $1 - @unparsed_uri = $2 - @http_version = HTTPVersion.new($3 ? $3 : "0.9") - else - rl = @request_line.sub(/\x0d?\x0a\z/o, '') - raise HTTPStatus::BadRequest, "bad Request-Line `#{rl}'." - end - end - - def read_header(socket) - if socket - while line = read_line(socket) - break if /\A(#{CRLF}|#{LF})\z/om =~ line - if (@request_bytes += line.bytesize) > MAX_HEADER_LENGTH - raise HTTPStatus::RequestEntityTooLarge, 'headers too large' - end - @raw_header << line - end - end - @header = HTTPUtils::parse_header(@raw_header.join) - end - - def parse_uri(str, scheme="http") - if @config[:Escape8bitURI] - str = HTTPUtils::escape8bit(str) - end - str.sub!(%r{\A/+}o, '/') - uri = URI::parse(str) - return uri if uri.absolute? - if @forwarded_host - host, port = @forwarded_host, @forwarded_port - elsif self["host"] - pattern = /\A(#{URI::REGEXP::PATTERN::HOST})(?::(\d+))?\z/n - host, port = *self['host'].scan(pattern)[0] - elsif @addr.size > 0 - host, port = @addr[2], @addr[1] - else - host, port = @config[:ServerName], @config[:Port] - end - uri.scheme = @forwarded_proto || scheme - uri.host = host - uri.port = port ? port.to_i : nil - return URI::parse(uri.to_s) - end - - def read_body(socket, block) - return unless socket - if tc = self['transfer-encoding'] - case tc - when /\Achunked\z/io then read_chunked(socket, block) - else raise HTTPStatus::NotImplemented, "Transfer-Encoding: #{tc}." - end - elsif self['content-length'] || @remaining_size - @remaining_size ||= self['content-length'].to_i - while @remaining_size > 0 - sz = [@buffer_size, @remaining_size].min - break unless buf = read_data(socket, sz) - @remaining_size -= buf.bytesize - block.call(buf) - end - if @remaining_size > 0 && @socket.eof? - raise HTTPStatus::BadRequest, "invalid body size." - end - elsif BODY_CONTAINABLE_METHODS.member?(@request_method) && !@socket.eof - raise HTTPStatus::LengthRequired - end - return @body - end - - def read_chunk_size(socket) - line = read_line(socket) - if /^([0-9a-fA-F]+)(?:;(\S+))?/ =~ line - chunk_size = $1.hex - chunk_ext = $2 - [ chunk_size, chunk_ext ] - else - raise HTTPStatus::BadRequest, "bad chunk `#{line}'." - end - end - - def read_chunked(socket, block) - chunk_size, = read_chunk_size(socket) - while chunk_size > 0 - begin - sz = [ chunk_size, @buffer_size ].min - data = read_data(socket, sz) # read chunk-data - if data.nil? || data.bytesize != sz - raise HTTPStatus::BadRequest, "bad chunk data size." - end - block.call(data) - end while (chunk_size -= sz) > 0 - - read_line(socket) # skip CRLF - chunk_size, = read_chunk_size(socket) - end - read_header(socket) # trailer + CRLF - @header.delete("transfer-encoding") - @remaining_size = 0 - end - - def _read_data(io, method, *arg) - begin - WEBrick::Utils.timeout(@config[:RequestTimeout]){ - return io.__send__(method, *arg) - } - rescue Errno::ECONNRESET - return nil - rescue Timeout::Error - raise HTTPStatus::RequestTimeout - end - end - - def read_line(io, size=4096) - _read_data(io, :gets, LF, size) - end - - def read_data(io, size) - _read_data(io, :read, size) - end - - def parse_query() - begin - if @request_method == "GET" || @request_method == "HEAD" - @query = HTTPUtils::parse_query(@query_string) - elsif self['content-type'] =~ /^application\/x-www-form-urlencoded/ - @query = HTTPUtils::parse_query(body) - elsif self['content-type'] =~ /^multipart\/form-data; boundary=(.+)/ - boundary = HTTPUtils::dequote($1) - @query = HTTPUtils::parse_form_data(body, boundary) - else - @query = Hash.new - end - rescue => ex - raise HTTPStatus::BadRequest, ex.message - end - end - - PrivateNetworkRegexp = / - ^unknown$| - ^((::ffff:)?127.0.0.1|::1)$| - ^(::ffff:)?(10|172\.(1[6-9]|2[0-9]|3[01])|192\.168)\. - /ixo - - # It's said that all X-Forwarded-* headers will contain more than one - # (comma-separated) value if the original request already contained one of - # these headers. Since we could use these values as Host header, we choose - # the initial(first) value. (apr_table_mergen() adds new value after the - # existing value with ", " prefix) - def setup_forwarded_info - if @forwarded_server = self["x-forwarded-server"] - @forwarded_server = @forwarded_server.split(",", 2).first - end - if @forwarded_proto = self["x-forwarded-proto"] - @forwarded_proto = @forwarded_proto.split(",", 2).first - end - if host_port = self["x-forwarded-host"] - host_port = host_port.split(",", 2).first - if host_port =~ /\A(\[[0-9a-fA-F:]+\])(?::(\d+))?\z/ - @forwarded_host = $1 - tmp = $2 - else - @forwarded_host, tmp = host_port.split(":", 2) - end - @forwarded_port = (tmp || (@forwarded_proto == "https" ? 443 : 80)).to_i - end - if addrs = self["x-forwarded-for"] - addrs = addrs.split(",").collect(&:strip) - addrs.reject!{|ip| PrivateNetworkRegexp =~ ip } - @forwarded_for = addrs.first - end - end - - # :startdoc: - end -end diff --git a/tool/lib/webrick/httpresponse.rb b/tool/lib/webrick/httpresponse.rb deleted file mode 100644 index ba4494ab74..0000000000 --- a/tool/lib/webrick/httpresponse.rb +++ /dev/null @@ -1,564 +0,0 @@ -# frozen_string_literal: false -# -# httpresponse.rb -- HTTPResponse Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpresponse.rb,v 1.45 2003/07/11 11:02:25 gotoyuzo Exp $ - -require 'time' -require 'uri' -require_relative 'httpversion' -require_relative 'htmlutils' -require_relative 'httputils' -require_relative 'httpstatus' - -module WEBrick - ## - # An HTTP response. This is filled in by the service or do_* methods of a - # WEBrick HTTP Servlet. - - class HTTPResponse - class InvalidHeader < StandardError - end - - ## - # HTTP Response version - - attr_reader :http_version - - ## - # Response status code (200) - - attr_reader :status - - ## - # Response header - - attr_reader :header - - ## - # Response cookies - - attr_reader :cookies - - ## - # Response reason phrase ("OK") - - attr_accessor :reason_phrase - - ## - # Body may be: - # * a String; - # * an IO-like object that responds to +#read+ and +#readpartial+; - # * a Proc-like object that responds to +#call+. - # - # In the latter case, either #chunked= should be set to +true+, - # or <code>header['content-length']</code> explicitly provided. - # Example: - # - # server.mount_proc '/' do |req, res| - # res.chunked = true - # # or - # # res.header['content-length'] = 10 - # res.body = proc { |out| out.write(Time.now.to_s) } - # end - - attr_accessor :body - - ## - # Request method for this response - - attr_accessor :request_method - - ## - # Request URI for this response - - attr_accessor :request_uri - - ## - # Request HTTP version for this response - - attr_accessor :request_http_version - - ## - # Filename of the static file in this response. Only used by the - # FileHandler servlet. - - attr_accessor :filename - - ## - # Is this a keep-alive response? - - attr_accessor :keep_alive - - ## - # Configuration for this response - - attr_reader :config - - ## - # Bytes sent in this response - - attr_reader :sent_size - - ## - # Creates a new HTTP response object. WEBrick::Config::HTTP is the - # default configuration. - - def initialize(config) - @config = config - @buffer_size = config[:OutputBufferSize] - @logger = config[:Logger] - @header = Hash.new - @status = HTTPStatus::RC_OK - @reason_phrase = nil - @http_version = HTTPVersion::convert(@config[:HTTPVersion]) - @body = '' - @keep_alive = true - @cookies = [] - @request_method = nil - @request_uri = nil - @request_http_version = @http_version # temporary - @chunked = false - @filename = nil - @sent_size = 0 - @bodytempfile = nil - end - - ## - # The response's HTTP status line - - def status_line - "HTTP/#@http_version #@status #@reason_phrase".rstrip << CRLF - end - - ## - # Sets the response's status to the +status+ code - - def status=(status) - @status = status - @reason_phrase = HTTPStatus::reason_phrase(status) - end - - ## - # Retrieves the response header +field+ - - def [](field) - @header[field.downcase] - end - - ## - # Sets the response header +field+ to +value+ - - def []=(field, value) - @chunked = value.to_s.downcase == 'chunked' if field.downcase == 'transfer-encoding' - @header[field.downcase] = value.to_s - end - - ## - # The content-length header - - def content_length - if len = self['content-length'] - return Integer(len) - end - end - - ## - # Sets the content-length header to +len+ - - def content_length=(len) - self['content-length'] = len.to_s - end - - ## - # The content-type header - - def content_type - self['content-type'] - end - - ## - # Sets the content-type header to +type+ - - def content_type=(type) - self['content-type'] = type - end - - ## - # Iterates over each header in the response - - def each - @header.each{|field, value| yield(field, value) } - end - - ## - # Will this response body be returned using chunked transfer-encoding? - - def chunked? - @chunked - end - - ## - # Enables chunked transfer encoding. - - def chunked=(val) - @chunked = val ? true : false - end - - ## - # Will this response's connection be kept alive? - - def keep_alive? - @keep_alive - end - - ## - # Sends the response on +socket+ - - def send_response(socket) # :nodoc: - begin - setup_header() - send_header(socket) - send_body(socket) - rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ENOTCONN => ex - @logger.debug(ex) - @keep_alive = false - rescue Exception => ex - @logger.error(ex) - @keep_alive = false - end - end - - ## - # Sets up the headers for sending - - def setup_header() # :nodoc: - @reason_phrase ||= HTTPStatus::reason_phrase(@status) - @header['server'] ||= @config[:ServerSoftware] - @header['date'] ||= Time.now.httpdate - - # HTTP/0.9 features - if @request_http_version < "1.0" - @http_version = HTTPVersion.new("0.9") - @keep_alive = false - end - - # HTTP/1.0 features - if @request_http_version < "1.1" - if chunked? - @chunked = false - ver = @request_http_version.to_s - msg = "chunked is set for an HTTP/#{ver} request. (ignored)" - @logger.warn(msg) - end - end - - # Determine the message length (RFC2616 -- 4.4 Message Length) - if @status == 304 || @status == 204 || HTTPStatus::info?(@status) - @header.delete('content-length') - @body = "" - elsif chunked? - @header["transfer-encoding"] = "chunked" - @header.delete('content-length') - elsif %r{^multipart/byteranges} =~ @header['content-type'] - @header.delete('content-length') - elsif @header['content-length'].nil? - if @body.respond_to? :readpartial - elsif @body.respond_to? :call - make_body_tempfile - else - @header['content-length'] = (@body ? @body.bytesize : 0).to_s - end - end - - # Keep-Alive connection. - if @header['connection'] == "close" - @keep_alive = false - elsif keep_alive? - if chunked? || @header['content-length'] || @status == 304 || @status == 204 || HTTPStatus.info?(@status) - @header['connection'] = "Keep-Alive" - else - msg = "Could not determine content-length of response body. Set content-length of the response or set Response#chunked = true" - @logger.warn(msg) - @header['connection'] = "close" - @keep_alive = false - end - else - @header['connection'] = "close" - end - - # Location is a single absoluteURI. - if location = @header['location'] - if @request_uri - @header['location'] = @request_uri.merge(location).to_s - end - end - end - - def make_body_tempfile # :nodoc: - return if @bodytempfile - bodytempfile = Tempfile.create("webrick") - if @body.nil? - # nothing - elsif @body.respond_to? :readpartial - IO.copy_stream(@body, bodytempfile) - @body.close - elsif @body.respond_to? :call - @body.call(bodytempfile) - else - bodytempfile.write @body - end - bodytempfile.rewind - @body = @bodytempfile = bodytempfile - @header['content-length'] = bodytempfile.stat.size.to_s - end - - def remove_body_tempfile # :nodoc: - if @bodytempfile - @bodytempfile.close - File.unlink @bodytempfile.path - @bodytempfile = nil - end - end - - - ## - # Sends the headers on +socket+ - - def send_header(socket) # :nodoc: - if @http_version.major > 0 - data = status_line() - @header.each{|key, value| - tmp = key.gsub(/\bwww|^te$|\b\w/){ $&.upcase } - data << "#{tmp}: #{check_header(value)}" << CRLF - } - @cookies.each{|cookie| - data << "Set-Cookie: " << check_header(cookie.to_s) << CRLF - } - data << CRLF - socket.write(data) - end - rescue InvalidHeader => e - @header.clear - @cookies.clear - set_error e - retry - end - - ## - # Sends the body on +socket+ - - def send_body(socket) # :nodoc: - if @body.respond_to? :readpartial then - send_body_io(socket) - elsif @body.respond_to?(:call) then - send_body_proc(socket) - else - send_body_string(socket) - end - end - - ## - # Redirects to +url+ with a WEBrick::HTTPStatus::Redirect +status+. - # - # Example: - # - # res.set_redirect WEBrick::HTTPStatus::TemporaryRedirect - - def set_redirect(status, url) - url = URI(url).to_s - @body = "<HTML><A HREF=\"#{url}\">#{url}</A>.</HTML>\n" - @header['location'] = url - raise status - end - - ## - # Creates an error page for exception +ex+ with an optional +backtrace+ - - def set_error(ex, backtrace=false) - case ex - when HTTPStatus::Status - @keep_alive = false if HTTPStatus::error?(ex.code) - self.status = ex.code - else - @keep_alive = false - self.status = HTTPStatus::RC_INTERNAL_SERVER_ERROR - end - @header['content-type'] = "text/html; charset=ISO-8859-1" - - if respond_to?(:create_error_page) - create_error_page() - return - end - - if @request_uri - host, port = @request_uri.host, @request_uri.port - else - host, port = @config[:ServerName], @config[:Port] - end - - error_body(backtrace, ex, host, port) - end - - private - - def check_header(header_value) - header_value = header_value.to_s - if /[\r\n]/ =~ header_value - raise InvalidHeader - else - header_value - end - end - - # :stopdoc: - - def error_body(backtrace, ex, host, port) - @body = '' - @body << <<-_end_of_html_ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN"> -<HTML> - <HEAD><TITLE>#{HTMLUtils::escape(@reason_phrase)}</TITLE></HEAD> - <BODY> - <H1>#{HTMLUtils::escape(@reason_phrase)}</H1> - #{HTMLUtils::escape(ex.message)} - <HR> - _end_of_html_ - - if backtrace && $DEBUG - @body << "backtrace of `#{HTMLUtils::escape(ex.class.to_s)}' " - @body << "#{HTMLUtils::escape(ex.message)}" - @body << "<PRE>" - ex.backtrace.each{|line| @body << "\t#{line}\n"} - @body << "</PRE><HR>" - end - - @body << <<-_end_of_html_ - <ADDRESS> - #{HTMLUtils::escape(@config[:ServerSoftware])} at - #{host}:#{port} - </ADDRESS> - </BODY> -</HTML> - _end_of_html_ - end - - def send_body_io(socket) - begin - if @request_method == "HEAD" - # do nothing - elsif chunked? - buf = '' - begin - @body.readpartial(@buffer_size, buf) - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - socket.write(data) - data.clear - @sent_size += size - rescue EOFError - break - end while true - buf.clear - socket.write("0#{CRLF}#{CRLF}") - else - if %r{\Abytes (\d+)-(\d+)/\d+\z} =~ @header['content-range'] - offset = $1.to_i - size = $2.to_i - offset + 1 - else - offset = nil - size = @header['content-length'] - size = size.to_i if size - end - begin - @sent_size = IO.copy_stream(@body, socket, size, offset) - rescue NotImplementedError - @body.seek(offset, IO::SEEK_SET) - @sent_size = IO.copy_stream(@body, socket, size) - end - end - ensure - @body.close - end - remove_body_tempfile - end - - def send_body_string(socket) - if @request_method == "HEAD" - # do nothing - elsif chunked? - body ? @body.bytesize : 0 - while buf = @body[@sent_size, @buffer_size] - break if buf.empty? - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - buf.clear - socket.write(data) - @sent_size += size - end - socket.write("0#{CRLF}#{CRLF}") - else - if @body && @body.bytesize > 0 - socket.write(@body) - @sent_size = @body.bytesize - end - end - end - - def send_body_proc(socket) - if @request_method == "HEAD" - # do nothing - elsif chunked? - @body.call(ChunkedWrapper.new(socket, self)) - socket.write("0#{CRLF}#{CRLF}") - else - size = @header['content-length'].to_i - if @bodytempfile - @bodytempfile.rewind - IO.copy_stream(@bodytempfile, socket) - else - @body.call(socket) - end - @sent_size = size - end - end - - class ChunkedWrapper - def initialize(socket, resp) - @socket = socket - @resp = resp - end - - def write(buf) - return 0 if buf.empty? - socket = @socket - @resp.instance_eval { - size = buf.bytesize - data = "#{size.to_s(16)}#{CRLF}#{buf}#{CRLF}" - socket.write(data) - data.clear - @sent_size += size - size - } - end - - def <<(*buf) - write(buf) - self - end - end - - # preserved for compatibility with some 3rd-party handlers - def _write_data(socket, data) - socket << data - end - - # :startdoc: - end - -end diff --git a/tool/lib/webrick/https.rb b/tool/lib/webrick/https.rb deleted file mode 100644 index b0a49bc40b..0000000000 --- a/tool/lib/webrick/https.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: false -# -# https.rb -- SSL/TLS enhancement for HTTPServer -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: https.rb,v 1.15 2003/07/22 19:20:42 gotoyuzo Exp $ - -require_relative 'ssl' -require_relative 'httpserver' - -module WEBrick - module Config - HTTP.update(SSL) - end - - ## - #-- - # Adds SSL functionality to WEBrick::HTTPRequest - - class HTTPRequest - - ## - # HTTP request SSL cipher - - attr_reader :cipher - - ## - # HTTP request server certificate - - attr_reader :server_cert - - ## - # HTTP request client certificate - - attr_reader :client_cert - - # :stopdoc: - - alias orig_parse parse - - def parse(socket=nil) - if socket.respond_to?(:cert) - @server_cert = socket.cert || @config[:SSLCertificate] - @client_cert = socket.peer_cert - @client_cert_chain = socket.peer_cert_chain - @cipher = socket.cipher - end - orig_parse(socket) - end - - alias orig_parse_uri parse_uri - - def parse_uri(str, scheme="https") - if server_cert - return orig_parse_uri(str, scheme) - end - return orig_parse_uri(str) - end - private :parse_uri - - alias orig_meta_vars meta_vars - - def meta_vars - meta = orig_meta_vars - if server_cert - meta["HTTPS"] = "on" - meta["SSL_SERVER_CERT"] = @server_cert.to_pem - meta["SSL_CLIENT_CERT"] = @client_cert ? @client_cert.to_pem : "" - if @client_cert_chain - @client_cert_chain.each_with_index{|cert, i| - meta["SSL_CLIENT_CERT_CHAIN_#{i}"] = cert.to_pem - } - end - meta["SSL_CIPHER"] = @cipher[0] - meta["SSL_PROTOCOL"] = @cipher[1] - meta["SSL_CIPHER_USEKEYSIZE"] = @cipher[2].to_s - meta["SSL_CIPHER_ALGKEYSIZE"] = @cipher[3].to_s - end - meta - end - - # :startdoc: - end - - ## - #-- - # Fake WEBrick::HTTPRequest for lookup_server - - class SNIRequest - - ## - # The SNI hostname - - attr_reader :host - - ## - # The socket address of the server - - attr_reader :addr - - ## - # The port this request is for - - attr_reader :port - - ## - # Creates a new SNIRequest. - - def initialize(sslsocket, hostname) - @host = hostname - @addr = sslsocket.addr - @port = @addr[1] - end - end - - - ## - #-- - # Adds SSL functionality to WEBrick::HTTPServer - - class HTTPServer < ::WEBrick::GenericServer - ## - # ServerNameIndication callback - - def ssl_servername_callback(sslsocket, hostname = nil) - req = SNIRequest.new(sslsocket, hostname) - server = lookup_server(req) - server ? server.ssl_context : nil - end - - # :stopdoc: - - ## - # Check whether +server+ is also SSL server. - # Also +server+'s SSL context will be created. - - alias orig_virtual_host virtual_host - - def virtual_host(server) - if @config[:SSLEnable] && !server.ssl_context - raise ArgumentError, "virtual host must set SSLEnable to true" - end - orig_virtual_host(server) - end - - # :startdoc: - end -end diff --git a/tool/lib/webrick/httpserver.rb b/tool/lib/webrick/httpserver.rb deleted file mode 100644 index e85d059319..0000000000 --- a/tool/lib/webrick/httpserver.rb +++ /dev/null @@ -1,294 +0,0 @@ -# frozen_string_literal: false -# -# httpserver.rb -- HTTPServer Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpserver.rb,v 1.63 2002/10/01 17:16:32 gotoyuzo Exp $ - -require 'io/wait' -require_relative 'server' -require_relative 'httputils' -require_relative 'httpstatus' -require_relative 'httprequest' -require_relative 'httpresponse' -require_relative 'httpservlet' -require_relative 'accesslog' - -module WEBrick - class HTTPServerError < ServerError; end - - ## - # An HTTP Server - - class HTTPServer < ::WEBrick::GenericServer - ## - # Creates a new HTTP server according to +config+ - # - # An HTTP server uses the following attributes: - # - # :AccessLog:: An array of access logs. See WEBrick::AccessLog - # :BindAddress:: Local address for the server to bind to - # :DocumentRoot:: Root path to serve files from - # :DocumentRootOptions:: Options for the default HTTPServlet::FileHandler - # :HTTPVersion:: The HTTP version of this server - # :Port:: Port to listen on - # :RequestCallback:: Called with a request and response before each - # request is serviced. - # :RequestTimeout:: Maximum time to wait between requests - # :ServerAlias:: Array of alternate names for this server for virtual - # hosting - # :ServerName:: Name for this server for virtual hosting - - def initialize(config={}, default=Config::HTTP) - super(config, default) - @http_version = HTTPVersion::convert(@config[:HTTPVersion]) - - @mount_tab = MountTable.new - if @config[:DocumentRoot] - mount("/", HTTPServlet::FileHandler, @config[:DocumentRoot], - @config[:DocumentRootOptions]) - end - - unless @config[:AccessLog] - @config[:AccessLog] = [ - [ $stderr, AccessLog::COMMON_LOG_FORMAT ], - [ $stderr, AccessLog::REFERER_LOG_FORMAT ] - ] - end - - @virtual_hosts = Array.new - end - - ## - # Processes requests on +sock+ - - def run(sock) - while true - req = create_request(@config) - res = create_response(@config) - server = self - begin - timeout = @config[:RequestTimeout] - while timeout > 0 - break if sock.to_io.wait_readable(0.5) - break if @status != :Running - timeout -= 0.5 - end - raise HTTPStatus::EOFError if timeout <= 0 || @status != :Running - raise HTTPStatus::EOFError if sock.eof? - req.parse(sock) - res.request_method = req.request_method - res.request_uri = req.request_uri - res.request_http_version = req.http_version - res.keep_alive = req.keep_alive? - server = lookup_server(req) || self - if callback = server[:RequestCallback] - callback.call(req, res) - elsif callback = server[:RequestHandler] - msg = ":RequestHandler is deprecated, please use :RequestCallback" - @logger.warn(msg) - callback.call(req, res) - end - server.service(req, res) - rescue HTTPStatus::EOFError, HTTPStatus::RequestTimeout => ex - res.set_error(ex) - rescue HTTPStatus::Error => ex - @logger.error(ex.message) - res.set_error(ex) - rescue HTTPStatus::Status => ex - res.status = ex.code - rescue StandardError => ex - @logger.error(ex) - res.set_error(ex, true) - ensure - if req.request_line - if req.keep_alive? && res.keep_alive? - req.fixup() - end - res.send_response(sock) - server.access_log(@config, req, res) - end - end - break if @http_version < "1.1" - break unless req.keep_alive? - break unless res.keep_alive? - end - end - - ## - # Services +req+ and fills in +res+ - - def service(req, res) - if req.unparsed_uri == "*" - if req.request_method == "OPTIONS" - do_OPTIONS(req, res) - raise HTTPStatus::OK - end - raise HTTPStatus::NotFound, "`#{req.unparsed_uri}' not found." - end - - servlet, options, script_name, path_info = search_servlet(req.path) - raise HTTPStatus::NotFound, "`#{req.path}' not found." unless servlet - req.script_name = script_name - req.path_info = path_info - si = servlet.get_instance(self, *options) - @logger.debug(format("%s is invoked.", si.class.name)) - si.service(req, res) - end - - ## - # The default OPTIONS request handler says GET, HEAD, POST and OPTIONS - # requests are allowed. - - def do_OPTIONS(req, res) - res["allow"] = "GET,HEAD,POST,OPTIONS" - end - - ## - # Mounts +servlet+ on +dir+ passing +options+ to the servlet at creation - # time - - def mount(dir, servlet, *options) - @logger.debug(sprintf("%s is mounted on %s.", servlet.inspect, dir)) - @mount_tab[dir] = [ servlet, options ] - end - - ## - # Mounts +proc+ or +block+ on +dir+ and calls it with a - # WEBrick::HTTPRequest and WEBrick::HTTPResponse - - def mount_proc(dir, proc=nil, &block) - proc ||= block - raise HTTPServerError, "must pass a proc or block" unless proc - mount(dir, HTTPServlet::ProcHandler.new(proc)) - end - - ## - # Unmounts +dir+ - - def unmount(dir) - @logger.debug(sprintf("unmount %s.", dir)) - @mount_tab.delete(dir) - end - alias umount unmount - - ## - # Finds a servlet for +path+ - - def search_servlet(path) - script_name, path_info = @mount_tab.scan(path) - servlet, options = @mount_tab[script_name] - if servlet - [ servlet, options, script_name, path_info ] - end - end - - ## - # Adds +server+ as a virtual host. - - def virtual_host(server) - @virtual_hosts << server - @virtual_hosts = @virtual_hosts.sort_by{|s| - num = 0 - num -= 4 if s[:BindAddress] - num -= 2 if s[:Port] - num -= 1 if s[:ServerName] - num - } - end - - ## - # Finds the appropriate virtual host to handle +req+ - - def lookup_server(req) - @virtual_hosts.find{|s| - (s[:BindAddress].nil? || req.addr[3] == s[:BindAddress]) && - (s[:Port].nil? || req.port == s[:Port]) && - ((s[:ServerName].nil? || req.host == s[:ServerName]) || - (!s[:ServerAlias].nil? && s[:ServerAlias].find{|h| h === req.host})) - } - end - - ## - # Logs +req+ and +res+ in the access logs. +config+ is used for the - # server name. - - def access_log(config, req, res) - param = AccessLog::setup_params(config, req, res) - @config[:AccessLog].each{|logger, fmt| - logger << AccessLog::format(fmt+"\n", param) - } - end - - ## - # Creates the HTTPRequest used when handling the HTTP - # request. Can be overridden by subclasses. - def create_request(with_webrick_config) - HTTPRequest.new(with_webrick_config) - end - - ## - # Creates the HTTPResponse used when handling the HTTP - # request. Can be overridden by subclasses. - def create_response(with_webrick_config) - HTTPResponse.new(with_webrick_config) - end - - ## - # Mount table for the path a servlet is mounted on in the directory space - # of the server. Users of WEBrick can only access this indirectly via - # WEBrick::HTTPServer#mount, WEBrick::HTTPServer#unmount and - # WEBrick::HTTPServer#search_servlet - - class MountTable # :nodoc: - def initialize - @tab = Hash.new - compile - end - - def [](dir) - dir = normalize(dir) - @tab[dir] - end - - def []=(dir, val) - dir = normalize(dir) - @tab[dir] = val - compile - val - end - - def delete(dir) - dir = normalize(dir) - res = @tab.delete(dir) - compile - res - end - - def scan(path) - @scanner =~ path - [ $&, $' ] - end - - private - - def compile - k = @tab.keys - k.sort! - k.reverse! - k.collect!{|path| Regexp.escape(path) } - @scanner = Regexp.new("\\A(" + k.join("|") +")(?=/|\\z)") - end - - def normalize(dir) - ret = dir ? dir.dup : "" - ret.sub!(%r|/+\z|, "") - ret - end - end - end -end diff --git a/tool/lib/webrick/httpservlet.rb b/tool/lib/webrick/httpservlet.rb deleted file mode 100644 index da49a1405b..0000000000 --- a/tool/lib/webrick/httpservlet.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: false -# -# httpservlet.rb -- HTTPServlet Utility File -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpservlet.rb,v 1.21 2003/02/23 12:24:46 gotoyuzo Exp $ - -require_relative 'httpservlet/abstract' -require_relative 'httpservlet/filehandler' -require_relative 'httpservlet/cgihandler' -require_relative 'httpservlet/erbhandler' -require_relative 'httpservlet/prochandler' - -module WEBrick - module HTTPServlet - FileHandler.add_handler("cgi", CGIHandler) - FileHandler.add_handler("rhtml", ERBHandler) - end -end diff --git a/tool/lib/webrick/httpservlet/abstract.rb b/tool/lib/webrick/httpservlet/abstract.rb deleted file mode 100644 index bccb091861..0000000000 --- a/tool/lib/webrick/httpservlet/abstract.rb +++ /dev/null @@ -1,152 +0,0 @@ -# frozen_string_literal: false -# -# httpservlet.rb -- HTTPServlet Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: abstract.rb,v 1.24 2003/07/11 11:16:46 gotoyuzo Exp $ - -require_relative '../htmlutils' -require_relative '../httputils' -require_relative '../httpstatus' - -module WEBrick - module HTTPServlet - class HTTPServletError < StandardError; end - - ## - # AbstractServlet allows HTTP server modules to be reused across multiple - # servers and allows encapsulation of functionality. - # - # By default a servlet will respond to GET, HEAD (through an alias to GET) - # and OPTIONS requests. - # - # By default a new servlet is initialized for every request. A servlet - # instance can be reused by overriding ::get_instance in the - # AbstractServlet subclass. - # - # == A Simple Servlet - # - # class Simple < WEBrick::HTTPServlet::AbstractServlet - # def do_GET request, response - # status, content_type, body = do_stuff_with request - # - # response.status = status - # response['Content-Type'] = content_type - # response.body = body - # end - # - # def do_stuff_with request - # return 200, 'text/plain', 'you got a page' - # end - # end - # - # This servlet can be mounted on a server at a given path: - # - # server.mount '/simple', Simple - # - # == Servlet Configuration - # - # Servlets can be configured via initialize. The first argument is the - # HTTP server the servlet is being initialized for. - # - # class Configurable < Simple - # def initialize server, color, size - # super server - # @color = color - # @size = size - # end - # - # def do_stuff_with request - # content = "<p " \ - # %q{style="color: #{@color}; font-size: #{@size}"} \ - # ">Hello, World!" - # - # return 200, "text/html", content - # end - # end - # - # This servlet must be provided two arguments at mount time: - # - # server.mount '/configurable', Configurable, 'red', '2em' - - class AbstractServlet - - ## - # Factory for servlet instances that will handle a request from +server+ - # using +options+ from the mount point. By default a new servlet - # instance is created for every call. - - def self.get_instance(server, *options) - self.new(server, *options) - end - - ## - # Initializes a new servlet for +server+ using +options+ which are - # stored as-is in +@options+. +@logger+ is also provided. - - def initialize(server, *options) - @server = @config = server - @logger = @server[:Logger] - @options = options - end - - ## - # Dispatches to a +do_+ method based on +req+ if such a method is - # available. (+do_GET+ for a GET request). Raises a MethodNotAllowed - # exception if the method is not implemented. - - def service(req, res) - method_name = "do_" + req.request_method.gsub(/-/, "_") - if respond_to?(method_name) - __send__(method_name, req, res) - else - raise HTTPStatus::MethodNotAllowed, - "unsupported method `#{req.request_method}'." - end - end - - ## - # Raises a NotFound exception - - def do_GET(req, res) - raise HTTPStatus::NotFound, "not found." - end - - ## - # Dispatches to do_GET - - def do_HEAD(req, res) - do_GET(req, res) - end - - ## - # Returns the allowed HTTP request methods - - def do_OPTIONS(req, res) - m = self.methods.grep(/\Ado_([A-Z]+)\z/) {$1} - m.sort! - res["allow"] = m.join(",") - end - - private - - ## - # Redirects to a path ending in / - - def redirect_to_directory_uri(req, res) - if req.path[-1] != ?/ - location = WEBrick::HTTPUtils.escape_path(req.path + "/") - if req.query_string && req.query_string.bytesize > 0 - location << "?" << req.query_string - end - res.set_redirect(HTTPStatus::MovedPermanently, location) - end - end - end - - end -end diff --git a/tool/lib/webrick/httpservlet/cgi_runner.rb b/tool/lib/webrick/httpservlet/cgi_runner.rb deleted file mode 100644 index 0398c16749..0000000000 --- a/tool/lib/webrick/httpservlet/cgi_runner.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: false -# -# cgi_runner.rb -- CGI launcher. -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cgi_runner.rb,v 1.9 2002/09/25 11:33:15 gotoyuzo Exp $ - -def sysread(io, size) - buf = "" - while size > 0 - tmp = io.sysread(size) - buf << tmp - size -= tmp.bytesize - end - return buf -end - -STDIN.binmode - -len = sysread(STDIN, 8).to_i -out = sysread(STDIN, len) -STDOUT.reopen(File.open(out, "w")) - -len = sysread(STDIN, 8).to_i -err = sysread(STDIN, len) -STDERR.reopen(File.open(err, "w")) - -len = sysread(STDIN, 8).to_i -dump = sysread(STDIN, len) -hash = Marshal.restore(dump) -ENV.keys.each{|name| ENV.delete(name) } -hash.each{|k, v| ENV[k] = v if v } - -dir = File::dirname(ENV["SCRIPT_FILENAME"]) -Dir::chdir dir - -if ARGV[0] - argv = ARGV.dup - argv << ENV["SCRIPT_FILENAME"] - exec(*argv) - # NOTREACHED -end -exec ENV["SCRIPT_FILENAME"] diff --git a/tool/lib/webrick/httpservlet/cgihandler.rb b/tool/lib/webrick/httpservlet/cgihandler.rb deleted file mode 100644 index 4457770b7a..0000000000 --- a/tool/lib/webrick/httpservlet/cgihandler.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: false -# -# cgihandler.rb -- CGIHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: cgihandler.rb,v 1.27 2003/03/21 19:56:01 gotoyuzo Exp $ - -require 'rbconfig' -require 'tempfile' -require_relative '../config' -require_relative 'abstract' - -module WEBrick - module HTTPServlet - - ## - # Servlet for handling CGI scripts - # - # Example: - # - # server.mount('/cgi/my_script', WEBrick::HTTPServlet::CGIHandler, - # '/path/to/my_script') - - class CGIHandler < AbstractServlet - Ruby = RbConfig.ruby # :nodoc: - CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: - CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb".freeze].freeze # :nodoc: - - ## - # Creates a new CGI script servlet for the script at +name+ - - def initialize(server, name) - super(server, name) - @script_filename = name - @tempdir = server[:TempDir] - interpreter = server[:CGIInterpreter] - if interpreter.is_a?(Array) - @cgicmd = CGIRunnerArray + interpreter - else - @cgicmd = "#{CGIRunner} #{interpreter}" - end - end - - # :stopdoc: - - def do_GET(req, res) - cgi_in = IO::popen(@cgicmd, "wb") - cgi_out = Tempfile.new("webrick.cgiout.", @tempdir, mode: IO::BINARY) - cgi_out.set_encoding("ASCII-8BIT") - cgi_err = Tempfile.new("webrick.cgierr.", @tempdir, mode: IO::BINARY) - cgi_err.set_encoding("ASCII-8BIT") - begin - cgi_in.sync = true - meta = req.meta_vars - meta["SCRIPT_FILENAME"] = @script_filename - meta["PATH"] = @config[:CGIPathEnv] - meta.delete("HTTP_PROXY") - if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM - meta["SystemRoot"] = ENV["SystemRoot"] - end - dump = Marshal.dump(meta) - - cgi_in.write("%8d" % cgi_out.path.bytesize) - cgi_in.write(cgi_out.path) - cgi_in.write("%8d" % cgi_err.path.bytesize) - cgi_in.write(cgi_err.path) - cgi_in.write("%8d" % dump.bytesize) - cgi_in.write(dump) - - req.body { |chunk| cgi_in.write(chunk) } - ensure - cgi_in.close - status = $?.exitstatus - sleep 0.1 if /mswin|bccwin|mingw/ =~ RUBY_PLATFORM - data = cgi_out.read - cgi_out.close(true) - if errmsg = cgi_err.read - if errmsg.bytesize > 0 - @logger.error("CGIHandler: #{@script_filename}:\n" + errmsg) - end - end - cgi_err.close(true) - end - - if status != 0 - @logger.error("CGIHandler: #{@script_filename} exit with #{status}") - end - - data = "" unless data - raw_header, body = data.split(/^[\xd\xa]+/, 2) - raise HTTPStatus::InternalServerError, - "Premature end of script headers: #{@script_filename}" if body.nil? - - begin - header = HTTPUtils::parse_header(raw_header) - if /^(\d+)/ =~ header['status'][0] - res.status = $1.to_i - header.delete('status') - end - if header.has_key?('location') - # RFC 3875 6.2.3, 6.2.4 - res.status = 302 unless (300...400) === res.status - end - if header.has_key?('set-cookie') - header['set-cookie'].each{|k| - res.cookies << Cookie.parse_set_cookie(k) - } - header.delete('set-cookie') - end - header.each{|key, val| res[key] = val.join(", ") } - rescue => ex - raise HTTPStatus::InternalServerError, ex.message - end - res.body = body - end - alias do_POST do_GET - - # :startdoc: - end - - end -end diff --git a/tool/lib/webrick/httpservlet/erbhandler.rb b/tool/lib/webrick/httpservlet/erbhandler.rb deleted file mode 100644 index cd09e5f216..0000000000 --- a/tool/lib/webrick/httpservlet/erbhandler.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: false -# -# erbhandler.rb -- ERBHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: erbhandler.rb,v 1.25 2003/02/24 19:25:31 gotoyuzo Exp $ - -require_relative 'abstract' - -require 'erb' - -module WEBrick - module HTTPServlet - - ## - # ERBHandler evaluates an ERB file and returns the result. This handler - # is automatically used if there are .rhtml files in a directory served by - # the FileHandler. - # - # ERBHandler supports GET and POST methods. - # - # The ERB file is evaluated with the local variables +servlet_request+ and - # +servlet_response+ which are a WEBrick::HTTPRequest and - # WEBrick::HTTPResponse respectively. - # - # Example .rhtml file: - # - # Request to <%= servlet_request.request_uri %> - # - # Query params <%= servlet_request.query.inspect %> - - class ERBHandler < AbstractServlet - - ## - # Creates a new ERBHandler on +server+ that will evaluate and serve the - # ERB file +name+ - - def initialize(server, name) - super(server, name) - @script_filename = name - end - - ## - # Handles GET requests - - def do_GET(req, res) - unless defined?(ERB) - @logger.warn "#{self.class}: ERB not defined." - raise HTTPStatus::Forbidden, "ERBHandler cannot work." - end - begin - data = File.open(@script_filename, &:read) - res.body = evaluate(ERB.new(data), req, res) - res['content-type'] ||= - HTTPUtils::mime_type(@script_filename, @config[:MimeTypes]) - rescue StandardError - raise - rescue Exception => ex - @logger.error(ex) - raise HTTPStatus::InternalServerError, ex.message - end - end - - ## - # Handles POST requests - - alias do_POST do_GET - - private - - ## - # Evaluates +erb+ providing +servlet_request+ and +servlet_response+ as - # local variables. - - def evaluate(erb, servlet_request, servlet_response) - Module.new.module_eval{ - servlet_request.meta_vars - servlet_request.query - erb.result(binding) - } - end - end - end -end diff --git a/tool/lib/webrick/httpservlet/filehandler.rb b/tool/lib/webrick/httpservlet/filehandler.rb deleted file mode 100644 index 010df0e918..0000000000 --- a/tool/lib/webrick/httpservlet/filehandler.rb +++ /dev/null @@ -1,552 +0,0 @@ -# frozen_string_literal: false -# -# filehandler.rb -- FileHandler Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: filehandler.rb,v 1.44 2003/06/07 01:34:51 gotoyuzo Exp $ - -require 'time' - -require_relative '../htmlutils' -require_relative '../httputils' -require_relative '../httpstatus' - -module WEBrick - module HTTPServlet - - ## - # Servlet for serving a single file. You probably want to use the - # FileHandler servlet instead as it handles directories and fancy indexes. - # - # Example: - # - # server.mount('/my_page.txt', WEBrick::HTTPServlet::DefaultFileHandler, - # '/path/to/my_page.txt') - # - # This servlet handles If-Modified-Since and Range requests. - - class DefaultFileHandler < AbstractServlet - - ## - # Creates a DefaultFileHandler instance for the file at +local_path+. - - def initialize(server, local_path) - super(server, local_path) - @local_path = local_path - end - - # :stopdoc: - - def do_GET(req, res) - st = File::stat(@local_path) - mtime = st.mtime - res['etag'] = sprintf("%x-%x-%x", st.ino, st.size, st.mtime.to_i) - - if not_modified?(req, res, mtime, res['etag']) - res.body = '' - raise HTTPStatus::NotModified - elsif req['range'] - make_partial_content(req, res, @local_path, st.size) - raise HTTPStatus::PartialContent - else - mtype = HTTPUtils::mime_type(@local_path, @config[:MimeTypes]) - res['content-type'] = mtype - res['content-length'] = st.size.to_s - res['last-modified'] = mtime.httpdate - res.body = File.open(@local_path, "rb") - end - end - - def not_modified?(req, res, mtime, etag) - if ir = req['if-range'] - begin - if Time.httpdate(ir) >= mtime - return true - end - rescue - if HTTPUtils::split_header_value(ir).member?(res['etag']) - return true - end - end - end - - if (ims = req['if-modified-since']) && Time.parse(ims) >= mtime - return true - end - - if (inm = req['if-none-match']) && - HTTPUtils::split_header_value(inm).member?(res['etag']) - return true - end - - return false - end - - # returns a lambda for webrick/httpresponse.rb send_body_proc - def multipart_body(body, parts, boundary, mtype, filesize) - lambda do |socket| - begin - begin - first = parts.shift - last = parts.shift - socket.write( - "--#{boundary}#{CRLF}" \ - "Content-Type: #{mtype}#{CRLF}" \ - "Content-Range: bytes #{first}-#{last}/#{filesize}#{CRLF}" \ - "#{CRLF}" - ) - - begin - IO.copy_stream(body, socket, last - first + 1, first) - rescue NotImplementedError - body.seek(first, IO::SEEK_SET) - IO.copy_stream(body, socket, last - first + 1) - end - socket.write(CRLF) - end while parts[0] - socket.write("--#{boundary}--#{CRLF}") - ensure - body.close - end - end - end - - def make_partial_content(req, res, filename, filesize) - mtype = HTTPUtils::mime_type(filename, @config[:MimeTypes]) - unless ranges = HTTPUtils::parse_range_header(req['range']) - raise HTTPStatus::BadRequest, - "Unrecognized range-spec: \"#{req['range']}\"" - end - File.open(filename, "rb"){|io| - if ranges.size > 1 - time = Time.now - boundary = "#{time.sec}_#{time.usec}_#{Process::pid}" - parts = [] - ranges.each {|range| - prange = prepare_range(range, filesize) - next if prange[0] < 0 - parts.concat(prange) - } - raise HTTPStatus::RequestRangeNotSatisfiable if parts.empty? - res["content-type"] = "multipart/byteranges; boundary=#{boundary}" - if req.http_version < '1.1' - res['connection'] = 'close' - else - res.chunked = true - end - res.body = multipart_body(io.dup, parts, boundary, mtype, filesize) - elsif range = ranges[0] - first, last = prepare_range(range, filesize) - raise HTTPStatus::RequestRangeNotSatisfiable if first < 0 - res['content-type'] = mtype - res['content-range'] = "bytes #{first}-#{last}/#{filesize}" - res['content-length'] = (last - first + 1).to_s - res.body = io.dup - else - raise HTTPStatus::BadRequest - end - } - end - - def prepare_range(range, filesize) - first = range.first < 0 ? filesize + range.first : range.first - return -1, -1 if first < 0 || first >= filesize - last = range.last < 0 ? filesize + range.last : range.last - last = filesize - 1 if last >= filesize - return first, last - end - - # :startdoc: - end - - ## - # Serves a directory including fancy indexing and a variety of other - # options. - # - # Example: - # - # server.mount('/assets', WEBrick::HTTPServlet::FileHandler, - # '/path/to/assets') - - class FileHandler < AbstractServlet - HandlerTable = Hash.new # :nodoc: - - ## - # Allow custom handling of requests for files with +suffix+ by class - # +handler+ - - def self.add_handler(suffix, handler) - HandlerTable[suffix] = handler - end - - ## - # Remove custom handling of requests for files with +suffix+ - - def self.remove_handler(suffix) - HandlerTable.delete(suffix) - end - - ## - # Creates a FileHandler servlet on +server+ that serves files starting - # at directory +root+ - # - # +options+ may be a Hash containing keys from - # WEBrick::Config::FileHandler or +true+ or +false+. - # - # If +options+ is true or false then +:FancyIndexing+ is enabled or - # disabled respectively. - - def initialize(server, root, options={}, default=Config::FileHandler) - @config = server.config - @logger = @config[:Logger] - @root = File.expand_path(root) - if options == true || options == false - options = { :FancyIndexing => options } - end - @options = default.dup.update(options) - end - - # :stopdoc: - - def set_filesystem_encoding(str) - enc = Encoding.find('filesystem') - if enc == Encoding::US_ASCII - str.b - else - str.dup.force_encoding(enc) - end - end - - def service(req, res) - # if this class is mounted on "/" and /~username is requested. - # we're going to override path information before invoking service. - if defined?(Etc) && @options[:UserDir] && req.script_name.empty? - if %r|^(/~([^/]+))| =~ req.path_info - script_name, user = $1, $2 - path_info = $' - begin - passwd = Etc::getpwnam(user) - @root = File::join(passwd.dir, @options[:UserDir]) - req.script_name = script_name - req.path_info = path_info - rescue - @logger.debug "#{self.class}#do_GET: getpwnam(#{user}) failed" - end - end - end - prevent_directory_traversal(req, res) - super(req, res) - end - - def do_GET(req, res) - unless exec_handler(req, res) - set_dir_list(req, res) - end - end - - def do_POST(req, res) - unless exec_handler(req, res) - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - def do_OPTIONS(req, res) - unless exec_handler(req, res) - super(req, res) - end - end - - # ToDo - # RFC2518: HTTP Extensions for Distributed Authoring -- WEBDAV - # - # PROPFIND PROPPATCH MKCOL DELETE PUT COPY MOVE - # LOCK UNLOCK - - # RFC3253: Versioning Extensions to WebDAV - # (Web Distributed Authoring and Versioning) - # - # VERSION-CONTROL REPORT CHECKOUT CHECK_IN UNCHECKOUT - # MKWORKSPACE UPDATE LABEL MERGE ACTIVITY - - private - - def trailing_pathsep?(path) - # check for trailing path separator: - # File.dirname("/aaaa/bbbb/") #=> "/aaaa") - # File.dirname("/aaaa/bbbb/x") #=> "/aaaa/bbbb") - # File.dirname("/aaaa/bbbb") #=> "/aaaa") - # File.dirname("/aaaa/bbbbx") #=> "/aaaa") - return File.dirname(path) != File.dirname(path+"x") - end - - def prevent_directory_traversal(req, res) - # Preventing directory traversal on Windows platforms; - # Backslashes (0x5c) in path_info are not interpreted as special - # character in URI notation. So the value of path_info should be - # normalize before accessing to the filesystem. - - # dirty hack for filesystem encoding; in nature, File.expand_path - # should not be used for path normalization. [Bug #3345] - path = req.path_info.dup.force_encoding(Encoding.find("filesystem")) - if trailing_pathsep?(req.path_info) - # File.expand_path removes the trailing path separator. - # Adding a character is a workaround to save it. - # File.expand_path("/aaa/") #=> "/aaa" - # File.expand_path("/aaa/" + "x") #=> "/aaa/x" - expanded = File.expand_path(path + "x") - expanded.chop! # remove trailing "x" - else - expanded = File.expand_path(path) - end - expanded.force_encoding(req.path_info.encoding) - req.path_info = expanded - end - - def exec_handler(req, res) - raise HTTPStatus::NotFound, "`#{req.path}' not found." unless @root - if set_filename(req, res) - handler = get_handler(req, res) - call_callback(:HandlerCallback, req, res) - h = handler.get_instance(@config, res.filename) - h.service(req, res) - return true - end - call_callback(:HandlerCallback, req, res) - return false - end - - def get_handler(req, res) - suffix1 = (/\.(\w+)\z/ =~ res.filename) && $1.downcase - if /\.(\w+)\.([\w\-]+)\z/ =~ res.filename - if @options[:AcceptableLanguages].include?($2.downcase) - suffix2 = $1.downcase - end - end - handler_table = @options[:HandlerTable] - return handler_table[suffix1] || handler_table[suffix2] || - HandlerTable[suffix1] || HandlerTable[suffix2] || - DefaultFileHandler - end - - def set_filename(req, res) - res.filename = @root - path_info = req.path_info.scan(%r|/[^/]*|) - - path_info.unshift("") # dummy for checking @root dir - while base = path_info.first - base = set_filesystem_encoding(base) - break if base == "/" - break unless File.directory?(File.expand_path(res.filename + base)) - shift_path_info(req, res, path_info) - call_callback(:DirectoryCallback, req, res) - end - - if base = path_info.first - base = set_filesystem_encoding(base) - if base == "/" - if file = search_index_file(req, res) - shift_path_info(req, res, path_info, file) - call_callback(:FileCallback, req, res) - return true - end - shift_path_info(req, res, path_info) - elsif file = search_file(req, res, base) - shift_path_info(req, res, path_info, file) - call_callback(:FileCallback, req, res) - return true - else - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - return false - end - - def check_filename(req, res, name) - if nondisclosure_name?(name) || windows_ambiguous_name?(name) - @logger.warn("the request refers nondisclosure name `#{name}'.") - raise HTTPStatus::NotFound, "`#{req.path}' not found." - end - end - - def shift_path_info(req, res, path_info, base=nil) - tmp = path_info.shift - base = base || set_filesystem_encoding(tmp) - req.path_info = path_info.join - req.script_name << base - res.filename = File.expand_path(res.filename + base) - check_filename(req, res, File.basename(res.filename)) - end - - def search_index_file(req, res) - @config[:DirectoryIndex].each{|index| - if file = search_file(req, res, "/"+index) - return file - end - } - return nil - end - - def search_file(req, res, basename) - langs = @options[:AcceptableLanguages] - path = res.filename + basename - if File.file?(path) - return basename - elsif langs.size > 0 - req.accept_language.each{|lang| - path_with_lang = path + ".#{lang}" - if langs.member?(lang) && File.file?(path_with_lang) - return basename + ".#{lang}" - end - } - (langs - req.accept_language).each{|lang| - path_with_lang = path + ".#{lang}" - if File.file?(path_with_lang) - return basename + ".#{lang}" - end - } - end - return nil - end - - def call_callback(callback_name, req, res) - if cb = @options[callback_name] - cb.call(req, res) - end - end - - def windows_ambiguous_name?(name) - return true if /[. ]+\z/ =~ name - return true if /::\$DATA\z/ =~ name - return false - end - - def nondisclosure_name?(name) - @options[:NondisclosureName].each{|pattern| - if File.fnmatch(pattern, name, File::FNM_CASEFOLD) - return true - end - } - return false - end - - def set_dir_list(req, res) - redirect_to_directory_uri(req, res) - unless @options[:FancyIndexing] - raise HTTPStatus::Forbidden, "no access permission to `#{req.path}'" - end - local_path = res.filename - list = Dir::entries(local_path).collect{|name| - next if name == "." || name == ".." - next if nondisclosure_name?(name) - next if windows_ambiguous_name?(name) - st = (File::stat(File.join(local_path, name)) rescue nil) - if st.nil? - [ name, nil, -1 ] - elsif st.directory? - [ name + "/", st.mtime, -1 ] - else - [ name, st.mtime, st.size ] - end - } - list.compact! - - query = req.query - - d0 = nil - idx = nil - %w[N M S].each_with_index do |q, i| - if d = query.delete(q) - idx ||= i - d0 ||= d - end - end - d0 ||= "A" - idx ||= 0 - d1 = (d0 == "A") ? "D" : "A" - - if d0 == "A" - list.sort!{|a,b| a[idx] <=> b[idx] } - else - list.sort!{|a,b| b[idx] <=> a[idx] } - end - - namewidth = query["NameWidth"] - if namewidth == "*" - namewidth = nil - elsif !namewidth or (namewidth = namewidth.to_i) < 2 - namewidth = 25 - end - query = query.inject('') {|s, (k, v)| s << '&' << HTMLUtils::escape("#{k}=#{v}")} - - type = "text/html" - case enc = Encoding.find('filesystem') - when Encoding::US_ASCII, Encoding::ASCII_8BIT - else - type << "; charset=\"#{enc.name}\"" - end - res['content-type'] = type - - title = "Index of #{HTMLUtils::escape(req.path)}" - res.body = <<-_end_of_html_ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN"> -<HTML> - <HEAD> - <TITLE>#{title}</TITLE> - <style type="text/css"> - <!-- - .name, .mtime { text-align: left; } - .size { text-align: right; } - td { text-overflow: ellipsis; white-space: nowrap; overflow: hidden; } - table { border-collapse: collapse; } - tr th { border-bottom: 2px groove; } - //--> - </style> - </HEAD> - <BODY> - <H1>#{title}</H1> - _end_of_html_ - - res.body << "<TABLE width=\"100%\"><THEAD><TR>\n" - res.body << "<TH class=\"name\"><A HREF=\"?N=#{d1}#{query}\">Name</A></TH>" - res.body << "<TH class=\"mtime\"><A HREF=\"?M=#{d1}#{query}\">Last modified</A></TH>" - res.body << "<TH class=\"size\"><A HREF=\"?S=#{d1}#{query}\">Size</A></TH>\n" - res.body << "</TR></THEAD>\n" - res.body << "<TBODY>\n" - - query.sub!(/\A&/, '?') - list.unshift [ "..", File::mtime(local_path+"/.."), -1 ] - list.each{ |name, time, size| - if name == ".." - dname = "Parent Directory" - elsif namewidth and name.size > namewidth - dname = name[0...(namewidth - 2)] << '..' - else - dname = name - end - s = "<TR><TD class=\"name\"><A HREF=\"#{HTTPUtils::escape(name)}#{query if name.end_with?('/')}\">#{HTMLUtils::escape(dname)}</A></TD>" - s << "<TD class=\"mtime\">" << (time ? time.strftime("%Y/%m/%d %H:%M") : "") << "</TD>" - s << "<TD class=\"size\">" << (size >= 0 ? size.to_s : "-") << "</TD></TR>\n" - res.body << s - } - res.body << "</TBODY></TABLE>" - res.body << "<HR>" - - res.body << <<-_end_of_html_ - <ADDRESS> - #{HTMLUtils::escape(@config[:ServerSoftware])}<BR> - at #{req.host}:#{req.port} - </ADDRESS> - </BODY> -</HTML> - _end_of_html_ - end - - # :startdoc: - end - end -end diff --git a/tool/lib/webrick/httpservlet/prochandler.rb b/tool/lib/webrick/httpservlet/prochandler.rb deleted file mode 100644 index 599ffc4340..0000000000 --- a/tool/lib/webrick/httpservlet/prochandler.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: false -# -# prochandler.rb -- ProcHandler Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: prochandler.rb,v 1.7 2002/09/21 12:23:42 gotoyuzo Exp $ - -require_relative 'abstract' - -module WEBrick - module HTTPServlet - - ## - # Mounts a proc at a path that accepts a request and response. - # - # Instead of mounting this servlet with WEBrick::HTTPServer#mount use - # WEBrick::HTTPServer#mount_proc: - # - # server.mount_proc '/' do |req, res| - # res.body = 'it worked!' - # res.status = 200 - # end - - class ProcHandler < AbstractServlet - # :stopdoc: - def get_instance(server, *options) - self - end - - def initialize(proc) - @proc = proc - end - - def do_GET(request, response) - @proc.call(request, response) - end - - alias do_POST do_GET - # :startdoc: - end - - end -end diff --git a/tool/lib/webrick/httpstatus.rb b/tool/lib/webrick/httpstatus.rb deleted file mode 100644 index c811f21964..0000000000 --- a/tool/lib/webrick/httpstatus.rb +++ /dev/null @@ -1,194 +0,0 @@ -# frozen_string_literal: false -#-- -# httpstatus.rb -- HTTPStatus Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpstatus.rb,v 1.11 2003/03/24 20:18:55 gotoyuzo Exp $ - -require_relative 'accesslog' - -module WEBrick - - ## - # This module is used to manager HTTP status codes. - # - # See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html for more - # information. - module HTTPStatus - - ## - # Root of the HTTP status class hierarchy - class Status < StandardError - class << self - attr_reader :code, :reason_phrase # :nodoc: - end - - # Returns the HTTP status code - def code() self::class::code end - - # Returns the HTTP status description - def reason_phrase() self::class::reason_phrase end - - alias to_i code # :nodoc: - end - - # Root of the HTTP info statuses - class Info < Status; end - # Root of the HTTP success statuses - class Success < Status; end - # Root of the HTTP redirect statuses - class Redirect < Status; end - # Root of the HTTP error statuses - class Error < Status; end - # Root of the HTTP client error statuses - class ClientError < Error; end - # Root of the HTTP server error statuses - class ServerError < Error; end - - class EOFError < StandardError; end - - # HTTP status codes and descriptions - StatusMessage = { # :nodoc: - 100 => 'Continue', - 101 => 'Switching Protocols', - 200 => 'OK', - 201 => 'Created', - 202 => 'Accepted', - 203 => 'Non-Authoritative Information', - 204 => 'No Content', - 205 => 'Reset Content', - 206 => 'Partial Content', - 207 => 'Multi-Status', - 300 => 'Multiple Choices', - 301 => 'Moved Permanently', - 302 => 'Found', - 303 => 'See Other', - 304 => 'Not Modified', - 305 => 'Use Proxy', - 307 => 'Temporary Redirect', - 400 => 'Bad Request', - 401 => 'Unauthorized', - 402 => 'Payment Required', - 403 => 'Forbidden', - 404 => 'Not Found', - 405 => 'Method Not Allowed', - 406 => 'Not Acceptable', - 407 => 'Proxy Authentication Required', - 408 => 'Request Timeout', - 409 => 'Conflict', - 410 => 'Gone', - 411 => 'Length Required', - 412 => 'Precondition Failed', - 413 => 'Request Entity Too Large', - 414 => 'Request-URI Too Large', - 415 => 'Unsupported Media Type', - 416 => 'Request Range Not Satisfiable', - 417 => 'Expectation Failed', - 422 => 'Unprocessable Entity', - 423 => 'Locked', - 424 => 'Failed Dependency', - 426 => 'Upgrade Required', - 428 => 'Precondition Required', - 429 => 'Too Many Requests', - 431 => 'Request Header Fields Too Large', - 451 => 'Unavailable For Legal Reasons', - 500 => 'Internal Server Error', - 501 => 'Not Implemented', - 502 => 'Bad Gateway', - 503 => 'Service Unavailable', - 504 => 'Gateway Timeout', - 505 => 'HTTP Version Not Supported', - 507 => 'Insufficient Storage', - 511 => 'Network Authentication Required', - } - - # Maps a status code to the corresponding Status class - CodeToError = {} # :nodoc: - - # Creates a status or error class for each status code and - # populates the CodeToError map. - StatusMessage.each{|code, message| - message.freeze - var_name = message.gsub(/[ \-]/,'_').upcase - err_name = message.gsub(/[ \-]/,'') - - case code - when 100...200; parent = Info - when 200...300; parent = Success - when 300...400; parent = Redirect - when 400...500; parent = ClientError - when 500...600; parent = ServerError - end - - const_set("RC_#{var_name}", code) - err_class = Class.new(parent) - err_class.instance_variable_set(:@code, code) - err_class.instance_variable_set(:@reason_phrase, message) - const_set(err_name, err_class) - CodeToError[code] = err_class - } - - ## - # Returns the description corresponding to the HTTP status +code+ - # - # WEBrick::HTTPStatus.reason_phrase 404 - # => "Not Found" - def reason_phrase(code) - StatusMessage[code.to_i] - end - - ## - # Is +code+ an informational status? - def info?(code) - code.to_i >= 100 and code.to_i < 200 - end - - ## - # Is +code+ a successful status? - def success?(code) - code.to_i >= 200 and code.to_i < 300 - end - - ## - # Is +code+ a redirection status? - def redirect?(code) - code.to_i >= 300 and code.to_i < 400 - end - - ## - # Is +code+ an error status? - def error?(code) - code.to_i >= 400 and code.to_i < 600 - end - - ## - # Is +code+ a client error status? - def client_error?(code) - code.to_i >= 400 and code.to_i < 500 - end - - ## - # Is +code+ a server error status? - def server_error?(code) - code.to_i >= 500 and code.to_i < 600 - end - - ## - # Returns the status class corresponding to +code+ - # - # WEBrick::HTTPStatus[302] - # => WEBrick::HTTPStatus::NotFound - # - def self.[](code) - CodeToError[code] - end - - module_function :reason_phrase - module_function :info?, :success?, :redirect?, :error? - module_function :client_error?, :server_error? - end -end diff --git a/tool/lib/webrick/httputils.rb b/tool/lib/webrick/httputils.rb deleted file mode 100644 index f1b9ddf9f0..0000000000 --- a/tool/lib/webrick/httputils.rb +++ /dev/null @@ -1,512 +0,0 @@ -# frozen_string_literal: false -# -# httputils.rb -- HTTPUtils Module -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httputils.rb,v 1.34 2003/06/05 21:34:08 gotoyuzo Exp $ - -require 'socket' -require 'tempfile' - -module WEBrick - CR = "\x0d" # :nodoc: - LF = "\x0a" # :nodoc: - CRLF = "\x0d\x0a" # :nodoc: - - ## - # HTTPUtils provides utility methods for working with the HTTP protocol. - # - # This module is generally used internally by WEBrick - - module HTTPUtils - - ## - # Normalizes a request path. Raises an exception if the path cannot be - # normalized. - - def normalize_path(path) - raise "abnormal path `#{path}'" if path[0] != ?/ - ret = path.dup - - ret.gsub!(%r{/+}o, '/') # // => / - while ret.sub!(%r'/\.(?:/|\Z)', '/'); end # /. => / - while ret.sub!(%r'/(?!\.\./)[^/]+/\.\.(?:/|\Z)', '/'); end # /foo/.. => /foo - - raise "abnormal path `#{path}'" if %r{/\.\.(/|\Z)} =~ ret - ret - end - module_function :normalize_path - - ## - # Default mime types - - DefaultMimeTypes = { - "ai" => "application/postscript", - "asc" => "text/plain", - "avi" => "video/x-msvideo", - "bin" => "application/octet-stream", - "bmp" => "image/bmp", - "class" => "application/octet-stream", - "cer" => "application/pkix-cert", - "crl" => "application/pkix-crl", - "crt" => "application/x-x509-ca-cert", - #"crl" => "application/x-pkcs7-crl", - "css" => "text/css", - "dms" => "application/octet-stream", - "doc" => "application/msword", - "dvi" => "application/x-dvi", - "eps" => "application/postscript", - "etx" => "text/x-setext", - "exe" => "application/octet-stream", - "gif" => "image/gif", - "htm" => "text/html", - "html" => "text/html", - "jpe" => "image/jpeg", - "jpeg" => "image/jpeg", - "jpg" => "image/jpeg", - "js" => "application/javascript", - "json" => "application/json", - "lha" => "application/octet-stream", - "lzh" => "application/octet-stream", - "mjs" => "application/javascript", - "mov" => "video/quicktime", - "mpe" => "video/mpeg", - "mpeg" => "video/mpeg", - "mpg" => "video/mpeg", - "pbm" => "image/x-portable-bitmap", - "pdf" => "application/pdf", - "pgm" => "image/x-portable-graymap", - "png" => "image/png", - "pnm" => "image/x-portable-anymap", - "ppm" => "image/x-portable-pixmap", - "ppt" => "application/vnd.ms-powerpoint", - "ps" => "application/postscript", - "qt" => "video/quicktime", - "ras" => "image/x-cmu-raster", - "rb" => "text/plain", - "rd" => "text/plain", - "rtf" => "application/rtf", - "sgm" => "text/sgml", - "sgml" => "text/sgml", - "svg" => "image/svg+xml", - "tif" => "image/tiff", - "tiff" => "image/tiff", - "txt" => "text/plain", - "wasm" => "application/wasm", - "xbm" => "image/x-xbitmap", - "xhtml" => "text/html", - "xls" => "application/vnd.ms-excel", - "xml" => "text/xml", - "xpm" => "image/x-xpixmap", - "xwd" => "image/x-xwindowdump", - "zip" => "application/zip", - } - - ## - # Loads Apache-compatible mime.types in +file+. - - def load_mime_types(file) - # note: +file+ may be a "| command" for now; some people may - # rely on this, but currently we do not use this method by default. - open(file){ |io| - hash = Hash.new - io.each{ |line| - next if /^#/ =~ line - line.chomp! - mimetype, ext0 = line.split(/\s+/, 2) - next unless ext0 - next if ext0.empty? - ext0.split(/\s+/).each{ |ext| hash[ext] = mimetype } - } - hash - } - end - module_function :load_mime_types - - ## - # Returns the mime type of +filename+ from the list in +mime_tab+. If no - # mime type was found application/octet-stream is returned. - - def mime_type(filename, mime_tab) - suffix1 = (/\.(\w+)$/ =~ filename && $1.downcase) - suffix2 = (/\.(\w+)\.[\w\-]+$/ =~ filename && $1.downcase) - mime_tab[suffix1] || mime_tab[suffix2] || "application/octet-stream" - end - module_function :mime_type - - ## - # Parses an HTTP header +raw+ into a hash of header fields with an Array - # of values. - - def parse_header(raw) - header = Hash.new([].freeze) - field = nil - raw.each_line{|line| - case line - when /^([A-Za-z0-9!\#$%&'*+\-.^_`|~]+):\s*(.*?)\s*\z/om - field, value = $1, $2 - field.downcase! - header[field] = [] unless header.has_key?(field) - header[field] << value - when /^\s+(.*?)\s*\z/om - value = $1 - unless field - raise HTTPStatus::BadRequest, "bad header '#{line}'." - end - header[field][-1] << " " << value - else - raise HTTPStatus::BadRequest, "bad header '#{line}'." - end - } - header.each{|key, values| - values.each(&:strip!) - } - header - end - module_function :parse_header - - ## - # Splits a header value +str+ according to HTTP specification. - - def split_header_value(str) - str.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",]+)+) - (?:,\s*|\Z)'xn).flatten - end - module_function :split_header_value - - ## - # Parses a Range header value +ranges_specifier+ - - def parse_range_header(ranges_specifier) - if /^bytes=(.*)/ =~ ranges_specifier - byte_range_set = split_header_value($1) - byte_range_set.collect{|range_spec| - case range_spec - when /^(\d+)-(\d+)/ then $1.to_i .. $2.to_i - when /^(\d+)-/ then $1.to_i .. -1 - when /^-(\d+)/ then -($1.to_i) .. -1 - else return nil - end - } - end - end - module_function :parse_range_header - - ## - # Parses q values in +value+ as used in Accept headers. - - def parse_qvalues(value) - tmp = [] - if value - parts = value.split(/,\s*/) - parts.each {|part| - if m = %r{^([^\s,]+?)(?:;\s*q=(\d+(?:\.\d+)?))?$}.match(part) - val = m[1] - q = (m[2] or 1).to_f - tmp.push([val, q]) - end - } - tmp = tmp.sort_by{|val, q| -q} - tmp.collect!{|val, q| val} - end - return tmp - end - module_function :parse_qvalues - - ## - # Removes quotes and escapes from +str+ - - def dequote(str) - ret = (/\A"(.*)"\Z/ =~ str) ? $1 : str.dup - ret.gsub!(/\\(.)/, "\\1") - ret - end - module_function :dequote - - ## - # Quotes and escapes quotes in +str+ - - def quote(str) - '"' << str.gsub(/[\\\"]/o, "\\\1") << '"' - end - module_function :quote - - ## - # Stores multipart form data. FormData objects are created when - # WEBrick::HTTPUtils.parse_form_data is called. - - class FormData < String - EmptyRawHeader = [].freeze # :nodoc: - EmptyHeader = {}.freeze # :nodoc: - - ## - # The name of the form data part - - attr_accessor :name - - ## - # The filename of the form data part - - attr_accessor :filename - - attr_accessor :next_data # :nodoc: - protected :next_data - - ## - # Creates a new FormData object. - # - # +args+ is an Array of form data entries. One FormData will be created - # for each entry. - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you - - def initialize(*args) - @name = @filename = @next_data = nil - if args.empty? - @raw_header = [] - @header = nil - super("") - else - @raw_header = EmptyRawHeader - @header = EmptyHeader - super(args.shift) - unless args.empty? - @next_data = self.class.new(*args) - end - end - end - - ## - # Retrieves the header at the first entry in +key+ - - def [](*key) - begin - @header[key[0].downcase].join(", ") - rescue StandardError, NameError - super - end - end - - ## - # Adds +str+ to this FormData which may be the body, a header or a - # header entry. - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you - - def <<(str) - if @header - super - elsif str == CRLF - @header = HTTPUtils::parse_header(@raw_header.join) - if cd = self['content-disposition'] - if /\s+name="(.*?)"/ =~ cd then @name = $1 end - if /\s+filename="(.*?)"/ =~ cd then @filename = $1 end - end - else - @raw_header << str - end - self - end - - ## - # Adds +data+ at the end of the chain of entries - # - # This is called by WEBrick::HTTPUtils.parse_form_data for you. - - def append_data(data) - tmp = self - while tmp - unless tmp.next_data - tmp.next_data = data - break - end - tmp = tmp.next_data - end - self - end - - ## - # Yields each entry in this FormData - - def each_data - tmp = self - while tmp - next_data = tmp.next_data - yield(tmp) - tmp = next_data - end - end - - ## - # Returns all the FormData as an Array - - def list - ret = [] - each_data{|data| - ret << data.to_s - } - ret - end - - ## - # A FormData will behave like an Array - - alias :to_ary :list - - ## - # This FormData's body - - def to_s - String.new(self) - end - end - - ## - # Parses the query component of a URI in +str+ - - def parse_query(str) - query = Hash.new - if str - str.split(/[&;]/).each{|x| - next if x.empty? - key, val = x.split(/=/,2) - key = unescape_form(key) - val = unescape_form(val.to_s) - val = FormData.new(val) - val.name = key - if query.has_key?(key) - query[key].append_data(val) - next - end - query[key] = val - } - end - query - end - module_function :parse_query - - ## - # Parses form data in +io+ with the given +boundary+ - - def parse_form_data(io, boundary) - boundary_regexp = /\A--#{Regexp.quote(boundary)}(--)?#{CRLF}\z/ - form_data = Hash.new - return form_data unless io - data = nil - io.each_line{|line| - if boundary_regexp =~ line - if data - data.chop! - key = data.name - if form_data.has_key?(key) - form_data[key].append_data(data) - else - form_data[key] = data - end - end - data = FormData.new - next - else - if data - data << line - end - end - } - return form_data - end - module_function :parse_form_data - - ##### - - reserved = ';/?:@&=+$,' - num = '0123456789' - lowalpha = 'abcdefghijklmnopqrstuvwxyz' - upalpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - mark = '-_.!~*\'()' - unreserved = num + lowalpha + upalpha + mark - control = (0x0..0x1f).collect{|c| c.chr }.join + "\x7f" - space = " " - delims = '<>#%"' - unwise = '{}|\\^[]`' - nonascii = (0x80..0xff).collect{|c| c.chr }.join - - module_function - - # :stopdoc: - - def _make_regex(str) /([#{Regexp.escape(str)}])/n end - def _make_regex!(str) /([^#{Regexp.escape(str)}])/n end - def _escape(str, regex) - str = str.b - str.gsub!(regex) {"%%%02X" % $1.ord} - # %-escaped string should contain US-ASCII only - str.force_encoding(Encoding::US_ASCII) - end - def _unescape(str, regex) - str = str.b - str.gsub!(regex) {$1.hex.chr} - # encoding of %-unescaped string is unknown - str - end - - UNESCAPED = _make_regex(control+space+delims+unwise+nonascii) - UNESCAPED_FORM = _make_regex(reserved+control+delims+unwise+nonascii) - NONASCII = _make_regex(nonascii) - ESCAPED = /%([0-9a-fA-F]{2})/ - UNESCAPED_PCHAR = _make_regex!(unreserved+":@&=+$,") - - # :startdoc: - - ## - # Escapes HTTP reserved and unwise characters in +str+ - - def escape(str) - _escape(str, UNESCAPED) - end - - ## - # Unescapes HTTP reserved and unwise characters in +str+ - - def unescape(str) - _unescape(str, ESCAPED) - end - - ## - # Escapes form reserved characters in +str+ - - def escape_form(str) - ret = _escape(str, UNESCAPED_FORM) - ret.gsub!(/ /, "+") - ret - end - - ## - # Unescapes form reserved characters in +str+ - - def unescape_form(str) - _unescape(str.gsub(/\+/, " "), ESCAPED) - end - - ## - # Escapes path +str+ - - def escape_path(str) - result = "" - str.scan(%r{/([^/]*)}).each{|i| - result << "/" << _escape(i[0], UNESCAPED_PCHAR) - } - return result - end - - ## - # Escapes 8 bit characters in +str+ - - def escape8bit(str) - _escape(str, NONASCII) - end - end -end diff --git a/tool/lib/webrick/httpversion.rb b/tool/lib/webrick/httpversion.rb deleted file mode 100644 index 8a251944a2..0000000000 --- a/tool/lib/webrick/httpversion.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: false -#-- -# HTTPVersion.rb -- presentation of HTTP version -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: httpversion.rb,v 1.5 2002/09/21 12:23:37 gotoyuzo Exp $ - -module WEBrick - - ## - # Represents an HTTP protocol version - - class HTTPVersion - include Comparable - - ## - # The major protocol version number - - attr_accessor :major - - ## - # The minor protocol version number - - attr_accessor :minor - - ## - # Converts +version+ into an HTTPVersion - - def self.convert(version) - version.is_a?(self) ? version : new(version) - end - - ## - # Creates a new HTTPVersion from +version+. - - def initialize(version) - case version - when HTTPVersion - @major, @minor = version.major, version.minor - when String - if /^(\d+)\.(\d+)$/ =~ version - @major, @minor = $1.to_i, $2.to_i - end - end - if @major.nil? || @minor.nil? - raise ArgumentError, - format("cannot convert %s into %s", version.class, self.class) - end - end - - ## - # Compares this version with +other+ according to the HTTP specification - # rules. - - def <=>(other) - unless other.is_a?(self.class) - other = self.class.new(other) - end - if (ret = @major <=> other.major) == 0 - return @minor <=> other.minor - end - return ret - end - - ## - # The HTTP version as show in the HTTP request and response. For example, - # "1.1" - - def to_s - format("%d.%d", @major, @minor) - end - end -end diff --git a/tool/lib/webrick/log.rb b/tool/lib/webrick/log.rb deleted file mode 100644 index 2c1fdfe602..0000000000 --- a/tool/lib/webrick/log.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: false -#-- -# log.rb -- Log Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: log.rb,v 1.26 2002/10/06 17:06:10 gotoyuzo Exp $ - -module WEBrick - - ## - # A generic logging class - - class BasicLog - - # Fatal log level which indicates a server crash - - FATAL = 1 - - # Error log level which indicates a recoverable error - - ERROR = 2 - - # Warning log level which indicates a possible problem - - WARN = 3 - - # Information log level which indicates possibly useful information - - INFO = 4 - - # Debugging error level for messages used in server development or - # debugging - - DEBUG = 5 - - # log-level, messages above this level will be logged - attr_accessor :level - - ## - # Initializes a new logger for +log_file+ that outputs messages at +level+ - # or higher. +log_file+ can be a filename, an IO-like object that - # responds to #<< or nil which outputs to $stderr. - # - # If no level is given INFO is chosen by default - - def initialize(log_file=nil, level=nil) - @level = level || INFO - case log_file - when String - @log = File.open(log_file, "a+") - @log.sync = true - @opened = true - when NilClass - @log = $stderr - else - @log = log_file # requires "<<". (see BasicLog#log) - end - end - - ## - # Closes the logger (also closes the log device associated to the logger) - def close - @log.close if @opened - @log = nil - end - - ## - # Logs +data+ at +level+ if the given level is above the current log - # level. - - def log(level, data) - if @log && level <= @level - data += "\n" if /\n\Z/ !~ data - @log << data - end - end - - ## - # Synonym for log(INFO, obj.to_s) - def <<(obj) - log(INFO, obj.to_s) - end - - # Shortcut for logging a FATAL message - def fatal(msg) log(FATAL, "FATAL " << format(msg)); end - # Shortcut for logging an ERROR message - def error(msg) log(ERROR, "ERROR " << format(msg)); end - # Shortcut for logging a WARN message - def warn(msg) log(WARN, "WARN " << format(msg)); end - # Shortcut for logging an INFO message - def info(msg) log(INFO, "INFO " << format(msg)); end - # Shortcut for logging a DEBUG message - def debug(msg) log(DEBUG, "DEBUG " << format(msg)); end - - # Will the logger output FATAL messages? - def fatal?; @level >= FATAL; end - # Will the logger output ERROR messages? - def error?; @level >= ERROR; end - # Will the logger output WARN messages? - def warn?; @level >= WARN; end - # Will the logger output INFO messages? - def info?; @level >= INFO; end - # Will the logger output DEBUG messages? - def debug?; @level >= DEBUG; end - - private - - ## - # Formats +arg+ for the logger - # - # * If +arg+ is an Exception, it will format the error message and - # the back trace. - # * If +arg+ responds to #to_str, it will return it. - # * Otherwise it will return +arg+.inspect. - def format(arg) - if arg.is_a?(Exception) - "#{arg.class}: #{AccessLog.escape(arg.message)}\n\t" << - arg.backtrace.join("\n\t") << "\n" - elsif arg.respond_to?(:to_str) - AccessLog.escape(arg.to_str) - else - arg.inspect - end - end - end - - ## - # A logging class that prepends a timestamp to each message. - - class Log < BasicLog - # Format of the timestamp which is applied to each logged line. The - # default is <tt>"[%Y-%m-%d %H:%M:%S]"</tt> - attr_accessor :time_format - - ## - # Same as BasicLog#initialize - # - # You can set the timestamp format through #time_format - def initialize(log_file=nil, level=nil) - super(log_file, level) - @time_format = "[%Y-%m-%d %H:%M:%S]" - end - - ## - # Same as BasicLog#log - def log(level, data) - tmp = Time.now.strftime(@time_format) - tmp << " " << data - super(level, tmp) - end - end -end diff --git a/tool/lib/webrick/server.rb b/tool/lib/webrick/server.rb deleted file mode 100644 index fd6b7a61b5..0000000000 --- a/tool/lib/webrick/server.rb +++ /dev/null @@ -1,381 +0,0 @@ -# frozen_string_literal: false -# -# server.rb -- GenericServer Class -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000, 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: server.rb,v 1.62 2003/07/22 19:20:43 gotoyuzo Exp $ - -require 'socket' -require_relative 'config' -require_relative 'log' - -module WEBrick - - ## - # Server error exception - - class ServerError < StandardError; end - - ## - # Base server class - - class SimpleServer - - ## - # A SimpleServer only yields when you start it - - def SimpleServer.start - yield - end - end - - ## - # A generic module for daemonizing a process - - class Daemon - - ## - # Performs the standard operations for daemonizing a process. Runs a - # block, if given. - - def Daemon.start - Process.daemon - File.umask(0) - yield if block_given? - end - end - - ## - # Base TCP server class. You must subclass GenericServer and provide a #run - # method. - - class GenericServer - - ## - # The server status. One of :Stop, :Running or :Shutdown - - attr_reader :status - - ## - # The server configuration - - attr_reader :config - - ## - # The server logger. This is independent from the HTTP access log. - - attr_reader :logger - - ## - # Tokens control the number of outstanding clients. The - # <code>:MaxClients</code> configuration sets this. - - attr_reader :tokens - - ## - # Sockets listening for connections. - - attr_reader :listeners - - ## - # Creates a new generic server from +config+. The default configuration - # comes from +default+. - - def initialize(config={}, default=Config::General) - @config = default.dup.update(config) - @status = :Stop - @config[:Logger] ||= Log::new - @logger = @config[:Logger] - - @tokens = Thread::SizedQueue.new(@config[:MaxClients]) - @config[:MaxClients].times{ @tokens.push(nil) } - - webrickv = WEBrick::VERSION - rubyv = "#{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]" - @logger.info("WEBrick #{webrickv}") - @logger.info("ruby #{rubyv}") - - @listeners = [] - @shutdown_pipe = nil - unless @config[:DoNotListen] - raise ArgumentError, "Port must an integer" unless @config[:Port].to_s == @config[:Port].to_i.to_s - - @config[:Port] = @config[:Port].to_i - if @config[:Listen] - warn(":Listen option is deprecated; use GenericServer#listen", uplevel: 1) - end - listen(@config[:BindAddress], @config[:Port]) - if @config[:Port] == 0 - @config[:Port] = @listeners[0].addr[1] - end - end - end - - ## - # Retrieves +key+ from the configuration - - def [](key) - @config[key] - end - - ## - # Adds listeners from +address+ and +port+ to the server. See - # WEBrick::Utils::create_listeners for details. - - def listen(address, port) - @listeners += Utils::create_listeners(address, port) - end - - ## - # Starts the server and runs the +block+ for each connection. This method - # does not return until the server is stopped from a signal handler or - # another thread using #stop or #shutdown. - # - # If the block raises a subclass of StandardError the exception is logged - # and ignored. If an IOError or Errno::EBADF exception is raised the - # exception is ignored. If an Exception subclass is raised the exception - # is logged and re-raised which stops the server. - # - # To completely shut down a server call #shutdown from ensure: - # - # server = WEBrick::GenericServer.new - # # or WEBrick::HTTPServer.new - # - # begin - # server.start - # ensure - # server.shutdown - # end - - def start(&block) - raise ServerError, "already started." if @status != :Stop - server_type = @config[:ServerType] || SimpleServer - - setup_shutdown_pipe - - server_type.start{ - @logger.info \ - "#{self.class}#start: pid=#{$$} port=#{@config[:Port]}" - @status = :Running - call_callback(:StartCallback) - - shutdown_pipe = @shutdown_pipe - - thgroup = ThreadGroup.new - begin - while @status == :Running - begin - sp = shutdown_pipe[0] - if svrs = IO.select([sp, *@listeners]) - if svrs[0].include? sp - # swallow shutdown pipe - buf = String.new - nil while String === - sp.read_nonblock([sp.nread, 8].max, buf, exception: false) - break - end - svrs[0].each{|svr| - @tokens.pop # blocks while no token is there. - if sock = accept_client(svr) - unless config[:DoNotReverseLookup].nil? - sock.do_not_reverse_lookup = !!config[:DoNotReverseLookup] - end - th = start_thread(sock, &block) - th[:WEBrickThread] = true - thgroup.add(th) - else - @tokens.push(nil) - end - } - end - rescue Errno::EBADF, Errno::ENOTSOCK, IOError => ex - # if the listening socket was closed in GenericServer#shutdown, - # IO::select raise it. - rescue StandardError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - rescue Exception => ex - @logger.fatal ex - raise - end - end - ensure - cleanup_shutdown_pipe(shutdown_pipe) - cleanup_listener - @status = :Shutdown - @logger.info "going to shutdown ..." - thgroup.list.each{|th| th.join if th[:WEBrickThread] } - call_callback(:StopCallback) - @logger.info "#{self.class}#start done." - @status = :Stop - end - } - end - - ## - # Stops the server from accepting new connections. - - def stop - if @status == :Running - @status = :Shutdown - end - - alarm_shutdown_pipe {|f| f.write_nonblock("\0")} - end - - ## - # Shuts down the server and all listening sockets. New listeners must be - # provided to restart the server. - - def shutdown - stop - - alarm_shutdown_pipe(&:close) - end - - ## - # You must subclass GenericServer and implement \#run which accepts a TCP - # client socket - - def run(sock) - @logger.fatal "run() must be provided by user." - end - - private - - # :stopdoc: - - ## - # Accepts a TCP client socket from the TCP server socket +svr+ and returns - # the client socket. - - def accept_client(svr) - case sock = svr.to_io.accept_nonblock(exception: false) - when :wait_readable - nil - else - if svr.respond_to?(:start_immediately) - sock = OpenSSL::SSL::SSLSocket.new(sock, ssl_context) - sock.sync_close = true - # we cannot do OpenSSL::SSL::SSLSocket#accept here because - # a slow client can prevent us from accepting connections - # from other clients - end - sock - end - rescue Errno::ECONNRESET, Errno::ECONNABORTED, - Errno::EPROTO, Errno::EINVAL - nil - rescue StandardError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - nil - end - - ## - # Starts a server thread for the client socket +sock+ that runs the given - # +block+. - # - # Sets the socket to the <code>:WEBrickSocket</code> thread local variable - # in the thread. - # - # If any errors occur in the block they are logged and handled. - - def start_thread(sock, &block) - Thread.start{ - begin - Thread.current[:WEBrickSocket] = sock - begin - addr = sock.peeraddr - @logger.debug "accept: #{addr[3]}:#{addr[1]}" - rescue SocketError - @logger.debug "accept: <address unknown>" - raise - end - if sock.respond_to?(:sync_close=) && @config[:SSLStartImmediately] - WEBrick::Utils.timeout(@config[:RequestTimeout]) do - begin - sock.accept # OpenSSL::SSL::SSLSocket#accept - rescue Errno::ECONNRESET, Errno::ECONNABORTED, - Errno::EPROTO, Errno::EINVAL - Thread.exit - end - end - end - call_callback(:AcceptCallback, sock) - block ? block.call(sock) : run(sock) - rescue Errno::ENOTCONN - @logger.debug "Errno::ENOTCONN raised" - rescue ServerError => ex - msg = "#{ex.class}: #{ex.message}\n\t#{ex.backtrace[0]}" - @logger.error msg - rescue Exception => ex - @logger.error ex - ensure - @tokens.push(nil) - Thread.current[:WEBrickSocket] = nil - if addr - @logger.debug "close: #{addr[3]}:#{addr[1]}" - else - @logger.debug "close: <address unknown>" - end - sock.close - end - } - end - - ## - # Calls the callback +callback_name+ from the configuration with +args+ - - def call_callback(callback_name, *args) - @config[callback_name]&.call(*args) - end - - def setup_shutdown_pipe - return @shutdown_pipe ||= IO.pipe - end - - def cleanup_shutdown_pipe(shutdown_pipe) - @shutdown_pipe = nil - shutdown_pipe&.each(&:close) - end - - def alarm_shutdown_pipe - _, pipe = @shutdown_pipe # another thread may modify @shutdown_pipe. - if pipe - if !pipe.closed? - begin - yield pipe - rescue IOError # closed by another thread. - end - end - end - end - - def cleanup_listener - @listeners.each{|s| - if @logger.debug? - addr = s.addr - @logger.debug("close TCPSocket(#{addr[2]}, #{addr[1]})") - end - begin - s.shutdown - rescue Errno::ENOTCONN - # when `Errno::ENOTCONN: Socket is not connected' on some platforms, - # call #close instead of #shutdown. - # (ignore @config[:ShutdownSocketWithoutClose]) - s.close - else - unless @config[:ShutdownSocketWithoutClose] - s.close - end - end - } - @listeners.clear - end - end # end of GenericServer -end diff --git a/tool/lib/webrick/ssl.rb b/tool/lib/webrick/ssl.rb deleted file mode 100644 index e448095a12..0000000000 --- a/tool/lib/webrick/ssl.rb +++ /dev/null @@ -1,215 +0,0 @@ -# frozen_string_literal: false -# -# ssl.rb -- SSL/TLS enhancement for GenericServer -# -# Copyright (c) 2003 GOTOU Yuuzou All rights reserved. -# -# $Id$ - -require 'webrick' -require 'openssl' - -module WEBrick - module Config - svrsoft = General[:ServerSoftware] - osslv = ::OpenSSL::OPENSSL_VERSION.split[1] - - ## - # Default SSL server configuration. - # - # WEBrick can automatically create a self-signed certificate if - # <code>:SSLCertName</code> is set. For more information on the various - # SSL options see OpenSSL::SSL::SSLContext. - # - # :ServerSoftware :: - # The server software name used in the Server: header. - # :SSLEnable :: false, - # Enable SSL for this server. Defaults to false. - # :SSLCertificate :: - # The SSL certificate for the server. - # :SSLPrivateKey :: - # The SSL private key for the server certificate. - # :SSLClientCA :: nil, - # Array of certificates that will be sent to the client. - # :SSLExtraChainCert :: nil, - # Array of certificates that will be added to the certificate chain - # :SSLCACertificateFile :: nil, - # Path to a CA certificate file - # :SSLCACertificatePath :: nil, - # Path to a directory containing CA certificates - # :SSLCertificateStore :: nil, - # OpenSSL::X509::Store used for certificate validation of the client - # :SSLTmpDhCallback :: nil, - # Callback invoked when DH parameters are required. - # :SSLVerifyClient :: - # Sets whether the client is verified. This defaults to VERIFY_NONE - # which is typical for an HTTPS server. - # :SSLVerifyDepth :: - # Number of CA certificates to walk when verifying a certificate chain - # :SSLVerifyCallback :: - # Custom certificate verification callback - # :SSLServerNameCallback:: - # Custom servername indication callback - # :SSLTimeout :: - # Maximum session lifetime - # :SSLOptions :: - # Various SSL options - # :SSLCiphers :: - # Ciphers to be used - # :SSLStartImmediately :: - # Immediately start SSL upon connection? Defaults to true - # :SSLCertName :: - # SSL certificate name. Must be set to enable automatic certificate - # creation. - # :SSLCertComment :: - # Comment used during automatic certificate creation. - - SSL = { - :ServerSoftware => "#{svrsoft} OpenSSL/#{osslv}", - :SSLEnable => false, - :SSLCertificate => nil, - :SSLPrivateKey => nil, - :SSLClientCA => nil, - :SSLExtraChainCert => nil, - :SSLCACertificateFile => nil, - :SSLCACertificatePath => nil, - :SSLCertificateStore => nil, - :SSLTmpDhCallback => nil, - :SSLVerifyClient => ::OpenSSL::SSL::VERIFY_NONE, - :SSLVerifyDepth => nil, - :SSLVerifyCallback => nil, # custom verification - :SSLTimeout => nil, - :SSLOptions => nil, - :SSLCiphers => nil, - :SSLStartImmediately => true, - # Must specify if you use auto generated certificate. - :SSLCertName => nil, - :SSLCertComment => "Generated by Ruby/OpenSSL" - } - General.update(SSL) - end - - module Utils - ## - # Creates a self-signed certificate with the given number of +bits+, - # the issuer +cn+ and a +comment+ to be stored in the certificate. - - def create_self_signed_cert(bits, cn, comment) - rsa = OpenSSL::PKey::RSA.new(bits){|p, n| - case p - when 0; $stderr.putc "." # BN_generate_prime - when 1; $stderr.putc "+" # BN_generate_prime - when 2; $stderr.putc "*" # searching good prime, - # n = #of try, - # but also data from BN_generate_prime - when 3; $stderr.putc "\n" # found good prime, n==0 - p, n==1 - q, - # but also data from BN_generate_prime - else; $stderr.putc "*" # BN_generate_prime - end - } - cert = OpenSSL::X509::Certificate.new - cert.version = 2 - cert.serial = 1 - name = (cn.kind_of? String) ? OpenSSL::X509::Name.parse(cn) - : OpenSSL::X509::Name.new(cn) - cert.subject = name - cert.issuer = name - cert.not_before = Time.now - cert.not_after = Time.now + (365*24*60*60) - cert.public_key = rsa.public_key - - ef = OpenSSL::X509::ExtensionFactory.new(nil,cert) - ef.issuer_certificate = cert - cert.extensions = [ - ef.create_extension("basicConstraints","CA:FALSE"), - ef.create_extension("keyUsage", "keyEncipherment, digitalSignature, keyAgreement, dataEncipherment"), - ef.create_extension("subjectKeyIdentifier", "hash"), - ef.create_extension("extendedKeyUsage", "serverAuth"), - ef.create_extension("nsComment", comment), - ] - aki = ef.create_extension("authorityKeyIdentifier", - "keyid:always,issuer:always") - cert.add_extension(aki) - cert.sign(rsa, "SHA256") - - return [ cert, rsa ] - end - module_function :create_self_signed_cert - end - - ## - #-- - # Updates WEBrick::GenericServer with SSL functionality - - class GenericServer - - ## - # SSL context for the server when run in SSL mode - - def ssl_context # :nodoc: - @ssl_context ||= begin - if @config[:SSLEnable] - ssl_context = setup_ssl_context(@config) - @logger.info("\n" + @config[:SSLCertificate].to_text) - ssl_context - end - end - end - - undef listen - - ## - # Updates +listen+ to enable SSL when the SSL configuration is active. - - def listen(address, port) # :nodoc: - listeners = Utils::create_listeners(address, port) - if @config[:SSLEnable] - listeners.collect!{|svr| - ssvr = ::OpenSSL::SSL::SSLServer.new(svr, ssl_context) - ssvr.start_immediately = @config[:SSLStartImmediately] - ssvr - } - end - @listeners += listeners - setup_shutdown_pipe - end - - ## - # Sets up an SSL context for +config+ - - def setup_ssl_context(config) # :nodoc: - unless config[:SSLCertificate] - cn = config[:SSLCertName] - comment = config[:SSLCertComment] - cert, key = Utils::create_self_signed_cert(2048, cn, comment) - config[:SSLCertificate] = cert - config[:SSLPrivateKey] = key - end - ctx = OpenSSL::SSL::SSLContext.new - ctx.key = config[:SSLPrivateKey] - ctx.cert = config[:SSLCertificate] - ctx.client_ca = config[:SSLClientCA] - ctx.extra_chain_cert = config[:SSLExtraChainCert] - ctx.ca_file = config[:SSLCACertificateFile] - ctx.ca_path = config[:SSLCACertificatePath] - ctx.cert_store = config[:SSLCertificateStore] - ctx.tmp_dh_callback = config[:SSLTmpDhCallback] - ctx.verify_mode = config[:SSLVerifyClient] - ctx.verify_depth = config[:SSLVerifyDepth] - ctx.verify_callback = config[:SSLVerifyCallback] - ctx.servername_cb = config[:SSLServerNameCallback] || proc { |args| ssl_servername_callback(*args) } - ctx.timeout = config[:SSLTimeout] - ctx.options = config[:SSLOptions] - ctx.ciphers = config[:SSLCiphers] - ctx - end - - ## - # ServerNameIndication callback - - def ssl_servername_callback(sslsocket, hostname = nil) - # default - end - - end -end diff --git a/tool/lib/webrick/utils.rb b/tool/lib/webrick/utils.rb deleted file mode 100644 index a96d6f03fd..0000000000 --- a/tool/lib/webrick/utils.rb +++ /dev/null @@ -1,265 +0,0 @@ -# frozen_string_literal: false -# -# utils.rb -- Miscellaneous utilities -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2001 TAKAHASHI Masayoshi, GOTOU Yuuzou -# Copyright (c) 2002 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: utils.rb,v 1.10 2003/02/16 22:22:54 gotoyuzo Exp $ - -require 'socket' -require 'io/nonblock' -require 'etc' - -module WEBrick - module Utils - ## - # Sets IO operations on +io+ to be non-blocking - def set_non_blocking(io) - io.nonblock = true if io.respond_to?(:nonblock=) - end - module_function :set_non_blocking - - ## - # Sets the close on exec flag for +io+ - def set_close_on_exec(io) - io.close_on_exec = true if io.respond_to?(:close_on_exec=) - end - module_function :set_close_on_exec - - ## - # Changes the process's uid and gid to the ones of +user+ - def su(user) - if pw = Etc.getpwnam(user) - Process::initgroups(user, pw.gid) - Process::Sys::setgid(pw.gid) - Process::Sys::setuid(pw.uid) - else - warn("WEBrick::Utils::su doesn't work on this platform", uplevel: 1) - end - end - module_function :su - - ## - # The server hostname - def getservername - Socket::gethostname - end - module_function :getservername - - ## - # Creates TCP server sockets bound to +address+:+port+ and returns them. - # - # It will create IPV4 and IPV6 sockets on all interfaces. - def create_listeners(address, port) - unless port - raise ArgumentError, "must specify port" - end - sockets = Socket.tcp_server_sockets(address, port) - sockets = sockets.map {|s| - s.autoclose = false - ts = TCPServer.for_fd(s.fileno) - s.close - ts - } - return sockets - end - module_function :create_listeners - - ## - # Characters used to generate random strings - RAND_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + - "0123456789" + - "abcdefghijklmnopqrstuvwxyz" - - ## - # Generates a random string of length +len+ - def random_string(len) - rand_max = RAND_CHARS.bytesize - ret = "" - len.times{ ret << RAND_CHARS[rand(rand_max)] } - ret - end - module_function :random_string - - ########### - - require "timeout" - require "singleton" - - ## - # Class used to manage timeout handlers across multiple threads. - # - # Timeout handlers should be managed by using the class methods which are - # synchronized. - # - # id = TimeoutHandler.register(10, Timeout::Error) - # begin - # sleep 20 - # puts 'foo' - # ensure - # TimeoutHandler.cancel(id) - # end - # - # will raise Timeout::Error - # - # id = TimeoutHandler.register(10, Timeout::Error) - # begin - # sleep 5 - # puts 'foo' - # ensure - # TimeoutHandler.cancel(id) - # end - # - # will print 'foo' - # - class TimeoutHandler - include Singleton - - ## - # Mutex used to synchronize access across threads - TimeoutMutex = Thread::Mutex.new # :nodoc: - - ## - # Registers a new timeout handler - # - # +time+:: Timeout in seconds - # +exception+:: Exception to raise when timeout elapsed - def TimeoutHandler.register(seconds, exception) - at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds - instance.register(Thread.current, at, exception) - end - - ## - # Cancels the timeout handler +id+ - def TimeoutHandler.cancel(id) - instance.cancel(Thread.current, id) - end - - def self.terminate - instance.terminate - end - - ## - # Creates a new TimeoutHandler. You should use ::register and ::cancel - # instead of creating the timeout handler directly. - def initialize - TimeoutMutex.synchronize{ - @timeout_info = Hash.new - } - @queue = Thread::Queue.new - @watcher = nil - end - - # :nodoc: - private \ - def watch - to_interrupt = [] - while true - now = Process.clock_gettime(Process::CLOCK_MONOTONIC) - wakeup = nil - to_interrupt.clear - TimeoutMutex.synchronize{ - @timeout_info.each {|thread, ary| - next unless ary - ary.each{|info| - time, exception = *info - if time < now - to_interrupt.push [thread, info.object_id, exception] - elsif !wakeup || time < wakeup - wakeup = time - end - } - } - } - to_interrupt.each {|arg| interrupt(*arg)} - if !wakeup - @queue.pop - elsif (wakeup -= now) > 0 - begin - (th = Thread.start {@queue.pop}).join(wakeup) - ensure - th&.kill&.join - end - end - @queue.clear - end - end - - # :nodoc: - private \ - def watcher - (w = @watcher)&.alive? and return w # usual case - TimeoutMutex.synchronize{ - (w = @watcher)&.alive? and next w # pathological check - @watcher = Thread.start(&method(:watch)) - } - end - - ## - # Interrupts the timeout handler +id+ and raises +exception+ - def interrupt(thread, id, exception) - if cancel(thread, id) && thread.alive? - thread.raise(exception, "execution timeout") - end - end - - ## - # Registers a new timeout handler - # - # +time+:: Timeout in seconds - # +exception+:: Exception to raise when timeout elapsed - def register(thread, time, exception) - info = nil - TimeoutMutex.synchronize{ - (@timeout_info[thread] ||= []) << (info = [time, exception]) - } - @queue.push nil - watcher - return info.object_id - end - - ## - # Cancels the timeout handler +id+ - def cancel(thread, id) - TimeoutMutex.synchronize{ - if ary = @timeout_info[thread] - ary.delete_if{|info| info.object_id == id } - if ary.empty? - @timeout_info.delete(thread) - end - return true - end - return false - } - end - - ## - def terminate - TimeoutMutex.synchronize{ - @timeout_info.clear - @watcher&.kill&.join - } - end - end - - ## - # Executes the passed block and raises +exception+ if execution takes more - # than +seconds+. - # - # If +seconds+ is zero or nil, simply executes the block - def timeout(seconds, exception=Timeout::Error) - return yield if seconds.nil? or seconds.zero? - # raise ThreadError, "timeout within critical session" if Thread.critical - id = TimeoutHandler.register(seconds, exception) - begin - yield(seconds) - ensure - TimeoutHandler.cancel(id) - end - end - module_function :timeout - end -end diff --git a/tool/lib/webrick/version.rb b/tool/lib/webrick/version.rb deleted file mode 100644 index b62988bdbb..0000000000 --- a/tool/lib/webrick/version.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: false -#-- -# version.rb -- version and release date -# -# Author: IPR -- Internet Programming with Ruby -- writers -# Copyright (c) 2000 TAKAHASHI Masayoshi, GOTOU YUUZOU -# Copyright (c) 2003 Internet Programming with Ruby writers. All rights -# reserved. -# -# $IPR: version.rb,v 1.74 2003/07/22 19:20:43 gotoyuzo Exp $ - -module WEBrick - - ## - # The WEBrick version - - VERSION = "1.7.0" -end diff --git a/tool/ln_sr.rb b/tool/ln_sr.rb new file mode 100755 index 0000000000..e1b5b6f76b --- /dev/null +++ b/tool/ln_sr.rb @@ -0,0 +1,131 @@ +#!/usr/bin/ruby + +target_directory = true +noop = false +force = false +quiet = false + +until ARGV.empty? + case ARGV[0] + when '-n' + noop = true + when '-f' + force = true + when '-T' + target_directory = false + when '-q' + quiet = true + else + break + end + ARGV.shift +end + +unless ARGV.size == 2 + abort "usage: #{$0} src destdir" +end +src, dest = ARGV + +require 'fileutils' + +include FileUtils +unless respond_to?(:ln_sr) + def ln_sr(src, dest, target_directory: true, force: nil, noop: nil, verbose: nil) + options = "#{force ? 'f' : ''}#{target_directory ? '' : 'T'}" + dest = File.path(dest) + srcs = Array(src) + link = proc do |s, target_dir_p = true| + s = File.path(s) + if target_dir_p + d = File.join(destdirs = dest, File.basename(s)) + else + destdirs = File.dirname(d = dest) + end + destdirs = fu_split_path(File.realpath(destdirs)) + if fu_starting_path?(s) + srcdirs = fu_split_path((File.realdirpath(s) rescue File.expand_path(s))) + base = fu_relative_components_from(srcdirs, destdirs) + s = File.join(*base) + else + srcdirs = fu_clean_components(*fu_split_path(s)) + base = fu_relative_components_from(fu_split_path(Dir.pwd), destdirs) + while srcdirs.first&. == ".." and base.last&.!=("..") and !fu_starting_path?(base.last) + srcdirs.shift + base.pop + end + s = File.join(*base, *srcdirs) + end + fu_output_message "ln -s#{options} #{s} #{d}" if verbose + next if noop + remove_file d, true if force + File.symlink s, d + end + case srcs.size + when 0 + when 1 + link[srcs[0], target_directory && File.directory?(dest)] + else + srcs.each(&link) + end + end + + def fu_split_path(path) + path = File.path(path) + list = [] + until (parent, base = File.split(path); parent == path or parent == ".") + list << base + path = parent + end + list << path + list.reverse! + end + + def fu_relative_components_from(target, base) #:nodoc: + i = 0 + while target[i]&.== base[i] + i += 1 + end + Array.new(base.size-i, '..').concat(target[i..-1]) + end + + def fu_clean_components(*comp) + comp.shift while comp.first == "." + return comp if comp.empty? + clean = [comp.shift] + path = File.join(*clean, "") # ending with File::SEPARATOR + while c = comp.shift + if c == ".." and clean.last != ".." and !(fu_have_symlink? && File.symlink?(path)) + clean.pop + path.sub!(%r((?<=\A|/)[^/]+/\z), "") + else + clean << c + path << c << "/" + end + end + clean + end + + if fu_windows? + def fu_starting_path?(path) + path&.start_with?(%r(\w:|/)) + end + else + def fu_starting_path?(path) + path&.start_with?("/") + end + end +end + +if File.respond_to?(:symlink) + if quiet and File.identical?(src, dest) + exit + end + begin + ln_sr(src, dest, verbose: true, target_directory: target_directory, force: force, noop: noop) + rescue NotImplementedError, Errno::EPERM, Errno::EACCES + else + exit + end +end + +cp_r(src, dest) diff --git a/tool/lrama/LEGAL.md b/tool/lrama/LEGAL.md new file mode 100644 index 0000000000..a3ef848514 --- /dev/null +++ b/tool/lrama/LEGAL.md @@ -0,0 +1,12 @@ +# LEGAL NOTICE INFORMATION + +All the files in this distribution are covered under the MIT License except some files +mentioned below. + +## GNU General Public License version 3 + +These files are licensed under the GNU General Public License version 3 or later. See these files for more information. + +* template/bison/_yacc.h +* template/bison/yacc.c +* template/bison/yacc.h diff --git a/tool/lrama/MIT b/tool/lrama/MIT new file mode 100644 index 0000000000..b23d5210d5 --- /dev/null +++ b/tool/lrama/MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 Yuichiro Kaneko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md new file mode 100644 index 0000000000..f71118a913 --- /dev/null +++ b/tool/lrama/NEWS.md @@ -0,0 +1,1032 @@ +# NEWS for Lrama + +## Lrama 0.7.1 (2025-12-24) + +### Optimize IELR + +Optimized performance to a level that allows for IELR testing in practical applications. + +https://github.com/ruby/lrama/pull/595 +https://github.com/ruby/lrama/pull/605 +https://github.com/ruby/lrama/pull/685 +https://github.com/ruby/lrama/pull/700 + +### Introduce counterexamples timeout + +Counterexample searches can sometimes take a long time, so we've added a timeout to abort the process after a set period. The current limits are: + +* 10 seconds per case +* 120 seconds total (cumulative) + +Please note that these are hard-coded and cannot be modified by the user in the current version. + +https://github.com/ruby/lrama/pull/623 + +### Optimize Counterexamples + +Optimized counterexample search performance. + +https://github.com/ruby/lrama/pull/607 +https://github.com/ruby/lrama/pull/610 +https://github.com/ruby/lrama/pull/614 +https://github.com/ruby/lrama/pull/622 +https://github.com/ruby/lrama/pull/627 +https://github.com/ruby/lrama/pull/629 +https://github.com/ruby/lrama/pull/659 + +### Support parameterized rule's arguments include inline + +Allow to use %inline directive with Parameterized rules arguments. When an inline rule is used as an argument to a Parameterized rule, it expands inline at the point of use. + +```yacc +%rule %inline op : '+' + | '-' + ; +%% +operation : op? + ; +``` + +This expands to: + +```yacc +operation : /* empty */ + | '+' + | '-' + ; +``` + +https://github.com/ruby/lrama/pull/637 + +### Render conflicts of each state on output file + +Added token information for conflicts in the output file. +These information are useful when a state has many actions. + +``` +State 1 + + 4 class: keyword_class • tSTRING "end" + 5 $@1: ε • [tSTRING] + 7 class: keyword_class • $@1 tSTRING '!' "end" $@2 + 8 $@3: ε • [tSTRING] + 10 class: keyword_class • $@3 tSTRING '?' "end" $@4 + + Conflict on tSTRING. shift/reduce($@1) + Conflict on tSTRING. shift/reduce($@3) + Conflict on tSTRING. reduce($@1)/reduce($@3) + + tSTRING shift, and go to state 6 + + tSTRING reduce using rule 5 ($@1) + tSTRING reduce using rule 8 ($@3) + + $@1 go to state 7 + $@3 go to state 8 +``` + +https://github.com/ruby/lrama/pull/541 + +### Render the origin of conflicted tokens on output file + +For example, for the grammar file like below: + +``` +%% + +program: expr + ; + +expr: expr '+' expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where `"plus"` (`'+'`) look ahead tokens come from: + +``` +State 6 + + 2 expr: expr • "plus" expr + 2 | expr "plus" expr • ["end of file", "plus"] + + Conflict on "plus". shift/reduce(expr) + "plus" comes from state 0 goto by expr + "plus" comes from state 5 goto by expr +``` + +state 0 and state 5 look like below: + +``` +State 0 + + 0 $accept: • program "end of file" + 1 program: • expr + 2 expr: • expr "plus" expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + program go to state 2 + expr go to state 3 + +State 5 + + 2 expr: • expr "plus" expr + 2 | expr "plus" • expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + expr go to state 6 +``` + +https://github.com/ruby/lrama/pull/726 + +### Render precedences usage information on output file + +For example, for the grammar file like below: + +``` +%left tPLUS +%right tUPLUS + +%% + +program: expr ; + +expr: tUPLUS expr + | expr tPLUS expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where these precedences are used to resolve conflicts: + +``` +Precedences + precedence on "unary+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + precedence on "+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + state 8. Conflict between reduce by "expr -> expr tPLUS expr" and shift "+" resolved as reduce (%left "+"). +``` + +https://github.com/ruby/lrama/pull/741 + +### Add support for reporting Rule Usage Frequency + +Support to report rule usage frequency statistics for analyzing grammar characteristics. +Run `exe/lrama --report=rules` to show how frequently each terminal and non-terminal symbol is used in the grammar rules. + +```console +$ exe/lrama --report=rules sample/calc.y +Rule Usage Frequency + 0 tSTRING (4 times) + 1 keyword_class (3 times) + 2 keyword_end (3 times) + 3 '+' (2 times) + 4 string (2 times) + 5 string_1 (2 times) + 6 '!' (1 times) + 7 '-' (1 times) + 8 '?' (1 times) + 9 EOI (1 times) + 10 class (1 times) + 11 program (1 times) + 12 string_2 (1 times) + 13 strings_1 (1 times) + 14 strings_2 (1 times) + 15 tNUMBER (1 times) +``` + +This feature provides insights into the language characteristics by showing: +- Which symbols are most frequently used in the grammar +- The distribution of terminal and non-terminal usage +- Potential areas for grammar optimization or refactoring + +The frequency statistics help developers understand the grammar structure and can be useful for: +- Grammar complexity analysis +- Performance optimization hints +- Language design decisions +- Documentation and educational purposes + +https://github.com/ruby/lrama/pull/677 + +### Render Split States information on output file + +For example, for the grammar file like below: + +``` +%token a +%token b +%token c +%define lr.type ielr + +%precedence tLOWEST +%precedence a +%precedence tHIGHEST + +%% + +S: a A B a + | b A B b + ; + +A: a C D E + ; + +B: c + | // empty + ; + +C: D + ; + +D: a + ; + +E: a + | %prec tHIGHEST // empty + ; + +%% +``` + +Lrama generates output file which describes where which new states are created when IELR is enabled: + +``` +Split States + + State 19 is split from state 4 + State 20 is split from state 9 + State 21 is split from state 14 +``` + +https://github.com/ruby/lrama/pull/624 + +### Add ioption support to the Standard library + +Support `ioption` (inline option) rule, which is expanded inline without creating intermediate rules. + +Unlike the regular `option` rule that generates a separate rule, `ioption` directly expands at the point of use: + +```yacc +program: ioption(number) expr + +// Expanded inline to: + +program: expr + | number expr +``` + +This differs from the regular `option` which would generate: + +```yacc +program: option(number) expr + +// Expanded to: + +program: option_number expr +option_number: %empty + | number +``` + +The `ioption` rule provides more compact grammar generation by avoiding intermediate rule creation, which can be beneficial for reducing the parser's rule count and potentially improving performance. + +This feature is inspired by Menhir's standard library and maintains compatibility with [Menhir's `ioption` behavior](https://github.com/let-def/menhir/blob/e8ba7bef219acd355798072c42abbd11335ecf09/src/standard.mly#L33-L41). + +https://github.com/ruby/lrama/pull/666 + +### Syntax Diagrams + +Lrama provides an API for generating HTML syntax diagrams. These visual diagrams are highly useful as grammar development tools and can also serve as a form of automatic self-documentation. + + + +If you use syntax diagrams, you add `--diagram` option. + +```console +$ exe/lrama --diagram sample.y +``` + +https://github.com/ruby/lrama/pull/523 + +### Support `--profile` option + +You can profile parser generation process without modification for Lrama source code. +Currently `--profile=call-stack` and `--profile=memory` are supported. + +```console +$ exe/lrama --profile=call-stack sample/calc.y +``` + +Then "tmp/stackprof-cpu-myapp.dump" is generated. + +https://github.com/ruby/lrama/pull/525 + +### Add support Start-Symbol: `%start` + +https://github.com/ruby/lrama/pull/576 + +## Lrama 0.7.0 (2025-01-21) + +### [EXPERIMENTAL] Support the generation of the IELR(1) parser described in this paper + +Support the generation of the IELR(1) parser described in this paper. +https://www.sciencedirect.com/science/article/pii/S0167642309001191 + +If you use IELR(1) parser, you can write the following directive in your grammar file. + +```yacc +%define lr.type ielr +``` + +But, currently IELR(1) parser is experimental feature. If you find any bugs, please report it to us. Thank you. + +### Support `-t` option as same as `--debug` option + +Support to `-t` option as same as `--debug` option. +These options align with Bison behavior. So same as `--debug` option. + +### Trace only explicit rules + +Support to trace only explicit rules. +If you use `--trace=rules` option, it shows include mid-rule actions. If you want to show only explicit rules, you can use `--trace=only-explicit-rules` option. + +Example: + +```yacc +%{ +%} +%union { + int i; +} +%token <i> number +%type <i> program +%% +program : number { printf("%d", $1); } number { $$ = $1 + $3; } + ; +%% +``` + +Result of `--trace=rules`: + +```console +$ exe/lrama --trace=rules sample.y +Grammar rules: +$accept -> program YYEOF +$@1 -> ε +program -> number $@1 number +``` + +Result of `--trace=only-explicit-rules`: + +```console +$ exe/lrama --trace=explicit-rules sample.y +Grammar rules: +$accept -> program YYEOF +program -> number number +``` + +## Lrama 0.6.11 (2024-12-23) + +### Add support for %type declarations using %nterm in Nonterminal Symbols + +Allow to use `%nterm` in Nonterminal Symbols for `%type` declarations. + +```yacc +%nterm <type> nonterminal… +``` + +This directive is also supported for compatibility with Bison, and only non-terminal symbols are allowed. In other words, definitions like the following will result in an error: + +```yacc +%{ +// Prologue +%} + +%token EOI 0 "EOI" +%nterm EOI + +%% + +program: /* empty */ + ; +``` + +It show an error message like the following: + +```command +❯ exe/lrama nterm.y +nterm.y:6:7: symbol EOI redeclared as a nonterminal +%nterm EOI + ^^^ +``` + +## Lrama 0.6.10 (2024-09-11) + +### Aliased Named References for actions of RHS in Parameterizing rules + +Allow to use aliased named references for actions of RHS in Parameterizing rules. + +```yacc +%rule sum(X, Y): X[summand] '+' Y[addend] { $$ = $summand + $addend } + ; +``` + +https://github.com/ruby/lrama/pull/410 + + +### Named References for actions of RHS in Parameterizing rules caller side + +Allow to use named references for actions of RHS in Parameterizing rules caller side. + +```yacc +opt_nl: '\n'?[nl] <str> { $$ = $nl; } + ; +``` + +https://github.com/ruby/lrama/pull/414 + +### Widen the definable position of Parameterizing rules + +Allow to define Parameterizing rules in the middle of the grammar. + +```yacc +%rule defined_option(X): /* empty */ + | X + ; + +%% + +program : defined_option(number) <i> + | defined_list(number) <i> + ; + +%rule defined_list(X): /* empty */ /* <--- here */ + | defined_list(X) number + ; +``` + +https://github.com/ruby/lrama/pull/420 + +### Report unused terminal symbols + +Support to report unused terminal symbols. +Run `exe/lrama --report=terms` to show unused terminal symbols. + +```console +$ exe/lrama --report=terms sample/calc.y + 11 Unused Terms + 0 YYerror + 1 YYUNDEF + 2 '\\\\' + 3 '\\13' + 4 keyword_class2 + 5 tNUMBER + 6 tPLUS + 7 tMINUS + 8 tEQ + 9 tEQEQ + 10 '>' +``` +https://github.com/ruby/lrama/pull/439 + +### Report unused rules + +Support to report unused rules. +Run `exe/lrama --report=rules` to show unused rules. + +```console +$ exe/lrama --report=rules sample/calc.y + 3 Unused Rules + 0 unused_option + 1 unused_list + 2 unused_nonempty_list +``` + +https://github.com/ruby/lrama/pull/441 + +### Ensure compatibility with Bison for `%locations` directive + +Support `%locations` directive to ensure compatibility with Bison. +Change to `%locations` directive not set by default. + +https://github.com/ruby/lrama/pull/446 + +### Diagnostics report for parameterized rules redefine + +Support to warning redefined parameterized rules. +Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterized rules. + +```console +$ exe/lrama -W sample/calc.y +parameterized rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) +``` + +https://github.com/ruby/lrama/pull/448 + +### Support `-v` and `--verbose` option + +Support to `-v` and `--verbose` option. +These options align with Bison behavior. So same as '--report=state' option. + +https://github.com/ruby/lrama/pull/457 + +## Lrama 0.6.9 (2024-05-02) + +### Callee side tag specification of Parameterizing rules + +Allow to specify tag on callee side of Parameterizing rules. + +```yacc +%union { + int i; +} + +%rule with_tag(X) <i>: X { $$ = $1; } + ; +``` + +### Named References for actions of RHS in Parameterizing rules + +Allow to use named references for actions of RHS in Parameterizing rules. + +```yacc +%rule option(number): /* empty */ + | number { $$ = $number; } + ; +``` + +## Lrama 0.6.8 (2024-04-29) + +### Nested Parameterizing rules with tag + +Allow to nested Parameterizing rules with tag. + +```yacc +%union { + int i; +} + +%rule nested_nested_option(X): /* empty */ + | X + ; + +%rule nested_option(X): /* empty */ + | nested_nested_option(X) <i> + ; + +%rule option(Y): /* empty */ + | nested_option(Y) <i> + ; +``` + +## Lrama 0.6.7 (2024-04-28) + +### RHS of user defined Parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. + +User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined Parameterizing rules. + +``` +%rule with_word_seps(X): /* empty */ + | X ' '+ + ; +``` + +## Lrama 0.6.6 (2024-04-27) + +### Trace actions + +Support trace actions for debugging. +Run `exe/lrama --trace=actions` to show grammar rules with actions. + +```console +$ exe/lrama --trace=actions sample/calc.y +Grammar rules with actions: +$accept -> list, YYEOF {} +list -> ε {} +list -> list, LF {} +list -> list, expr, LF { printf("=> %d\n", $2); } +expr -> NUM {} +expr -> expr, '+', expr { $$ = $1 + $3; } +expr -> expr, '-', expr { $$ = $1 - $3; } +expr -> expr, '*', expr { $$ = $1 * $3; } +expr -> expr, '/', expr { $$ = $1 / $3; } +expr -> '(', expr, ')' { $$ = $2; } +``` + +### Inlining + +Support inlining for rules. +The `%inline` directive causes all references to symbols to be replaced with its definition. + +```yacc +%rule %inline op: PLUS { + } + | TIMES { * } + ; + +%% + +expr : number { $$ = $1; } + | expr op expr { $$ = $1 $2 $3; } + ; +``` + +as same as + +```yacc +expr : number { $$ = $1; } + | expr '+' expr { $$ = $1 + $3; } + | expr '*' expr { $$ = $1 * $3; } + ; +``` + +## Lrama 0.6.5 (2024-03-25) + +### Typed Midrule Actions + +User can specify the type of mid-rule action by tag (`<bar>`) instead of specifying it with in an action. + +```yacc +primary: k_case expr_value terms? + { + $<val>$ = p->case_labels; + p->case_labels = Qnil; + } + case_body + k_end + { + ... + } +``` + +can be written as + +```yacc +primary: k_case expr_value terms? + { + $$ = p->case_labels; + p->case_labels = Qnil; + }<val> + case_body + k_end + { + ... + } +``` + +`%destructor` for midrule action is invoked only when tag is specified by Typed Midrule Actions. + +Difference from Bison's Typed Midrule Actions is that tag is postposed in Lrama however it's preposed in Bison. + +Bison supports this feature from 3.1. + +## Lrama 0.6.4 (2024-03-22) + +### Parameterizing rules (preceded, terminated, delimited) + +Support `preceded`, `terminated` and `delimited` rules. + +```text +program: preceded(opening, X) + +// Expanded to + +program: preceded_opening_X +preceded_opening_X: opening X +``` + +``` +program: terminated(X, closing) + +// Expanded to + +program: terminated_X_closing +terminated_X_closing: X closing +``` + +``` +program: delimited(opening, X, closing) + +// Expanded to + +program: delimited_opening_X_closing +delimited_opening_X_closing: opening X closing +``` + +https://github.com/ruby/lrama/pull/382 + +### Support `%destructor` declaration + +User can set codes for freeing semantic value resources by using `%destructor`. +In general, these resources are freed by actions or after parsing. +However, if syntax error happens in parsing, these codes may not be executed. +Codes associated to `%destructor` are executed when semantic value is popped from the stack by an error. + +```yacc +%token <val1> NUM +%type <val2> expr2 +%type <val3> expr + +%destructor { + printf("destructor for val1: %d\n", $$); +} <val1> // printer for TAG + +%destructor { + printf("destructor for val2: %d\n", $$); +} <val2> + +%destructor { + printf("destructor for expr: %d\n", $$); +} expr // printer for symbol +``` + +Bison supports this feature from 1.75b. + +https://github.com/ruby/lrama/pull/385 + +## Lrama 0.6.3 (2024-02-15) + +### Bring Your Own Stack + +Provide functionalities for Bring Your Own Stack. + +Ruby’s Ripper library requires their own semantic value stack to manage Ruby Objects returned by user defined callback method. Currently Ripper uses semantic value stack (`yyvsa`) which is used by parser to manage Node. This hack introduces some limitation on Ripper. For example, Ripper can not execute semantic analysis depending on Node structure. + +Lrama introduces two features to support another semantic value stack by parser generator users. + +1. Callback entry points + +User can emulate semantic value stack by these callbacks. +Lrama provides these five callbacks. Registered functions are called when each event happens. For example %after-shift function is called when shift happens on original semantic value stack. + +* `%after-shift` function_name +* `%before-reduce` function_name +* `%after-reduce` function_name +* `%after-shift-error-token` function_name +* `%after-pop-stack` function_name + +2. `$:n` variable to access index of each grammar symbols + +User also needs to access semantic value of their stack in grammar action. `$:n` provides the way to access to it. `$:n` is translated to the minus index from the top of the stack. +For example + +```yacc +primary: k_if expr_value then compstmt if_tail k_end + { + /*% ripper: if!($:2, $:4, $:5) %*/ + /* $:2 = -5, $:4 = -3, $:5 = -2. */ + } +``` + +https://github.com/ruby/lrama/pull/367 + +## Lrama 0.6.2 (2024-01-27) + +### %no-stdlib directive + +If `%no-stdlib` directive is set, Lrama doesn't load Lrama standard library for +parameterized rules, stdlib.y. + +https://github.com/ruby/lrama/pull/344 + +## Lrama 0.6.1 (2024-01-13) + +### Nested Parameterizing rules + +Allow to pass an instantiated rule to other Parameterizing rules. + +```yacc +%rule constant(X) : X + ; + +%rule option(Y) : /* empty */ + | Y + ; + +%% + +program : option(constant(number)) // Nested rule + ; +%% +``` + +Allow to use nested Parameterizing rules when define Parameterizing rules. + +```yacc +%rule option(x) : /* empty */ + | X + ; + +%rule double(Y) : Y Y + ; + +%rule double_opt(A) : option(double(A)) // Nested rule + ; + +%% + +program : double_opt(number) + ; + +%% +``` + +https://github.com/ruby/lrama/pull/337 + +## Lrama 0.6.0 (2023-12-25) + +### User defined Parameterizing rules + +Allow to define Parameterizing rule by `%rule` directive. + +```yacc +%rule pair(X, Y): X Y { $$ = $1 + $2; } + ; + +%% + +program: stmt + ; + +stmt: pair(ODD, EVEN) <num> + | pair(EVEN, ODD) <num> + ; +``` + +https://github.com/ruby/lrama/pull/285 + +## Lrama 0.5.11 (2023-12-02) + +### Type specification of Parameterizing rules + +Allow to specify type of rules by specifying tag, `<i>` in below example. +Tag is post-modification style. + +```yacc +%union { + int i; +} + +%% + +program : option(number) <i> + | number_alias? <i> + ; +``` + +https://github.com/ruby/lrama/pull/272 + + +## Lrama 0.5.10 (2023-11-18) + +### Parameterizing rules (option, nonempty_list, list) + +Support function call style Parameterizing rules for `option`, `nonempty_list` and `list`. + +https://github.com/ruby/lrama/pull/197 + +### Parameterizing rules (separated_list) + +Support `separated_list` and `separated_nonempty_list` Parameterizing rules. + +```text +program: separated_list(',', number) + +// Expanded to + +program: separated_list_number +separated_list_number: ε +separated_list_number: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +``` +program: separated_nonempty_list(',', number) + +// Expanded to + +program: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +https://github.com/ruby/lrama/pull/204 + +## Lrama 0.5.9 (2023-11-05) + +### Parameterizing rules (suffix) + +Parameterizing rules are template of rules. +It's very common pattern to write "list" grammar rule like: + +```yacc +opt_args: /* none */ + | args + ; + +args: arg + | args arg +``` + +Lrama supports these suffixes: + +* `?`: option +* `+`: nonempty list +* `*`: list + +Idea of Parameterizing rules comes from Menhir LR(1) parser generator (https://gallium.inria.fr/~fpottier/menhir/manual.html#sec32). + +https://github.com/ruby/lrama/pull/181 + +## Lrama 0.5.7 (2023-10-23) + +### Racc parser + +Replace Lrama's parser from handwritten parser to LR parser generated by Racc. +Lrama uses `--embedded` option to generate LR parser because Racc is changed from default gem to bundled gem by Ruby 3.3 (https://github.com/ruby/lrama/pull/132). + +https://github.com/ruby/lrama/pull/62 + +## Lrama 0.5.4 (2023-08-17) + +### Runtime configuration for error recovery + +Make error recovery function configurable on runtime by two new macros. + +* `YYMAXREPAIR`: Expected to return max length of repair operations. `%parse-param` is passed to this function. +* `YYERROR_RECOVERY_ENABLED`: Expected to return bool value to determine error recovery is enabled or not. `%parse-param` is passed to this function. + +https://github.com/ruby/lrama/pull/74 + +## Lrama 0.5.3 (2023-08-05) + +### Error Recovery + +Support token insert base Error Recovery. +`-e` option is needed to generate parser with error recovery functions. + +https://github.com/ruby/lrama/pull/44 + +## Lrama 0.5.2 (2023-06-14) + +### Named References + +Instead of positional references like `$1` or `$$`, +named references allow to access to symbol by name. + +```yacc +primary: k_class cpath superclass bodystmt k_end + { + $primary = new_class($cpath, $bodystmt, $superclass); + } +``` + +Alias name can be declared. + +```yacc +expr[result]: expr[ex-left] '+' expr[ex.right] + { + $result = $[ex-left] + $[ex.right]; + } +``` + +Bison supports this feature from 2.5. + +### Add parse params to some macros and functions + +`%parse-param` are added to these macros and functions to remove ytab.sed hack from Ruby. + +* `YY_LOCATION_PRINT` +* `YY_SYMBOL_PRINT` +* `yy_stack_print` +* `YY_STACK_PRINT` +* `YY_REDUCE_PRINT` +* `yysyntax_error` + +https://github.com/ruby/lrama/pull/40 + +See also: https://github.com/ruby/ruby/pull/7807 + +## Lrama 0.5.0 (2023-05-17) + +### stdin mode + +When `-` is given as grammar file name, reads the grammar source from STDIN, and takes the next argument as the input file name. This mode helps pre-process a grammar source. + +https://github.com/ruby/lrama/pull/8 + +## Lrama 0.4.0 (2023-05-13) + +This is the first version migrated to Ruby. +This version generates "parse.c" compatible with Bison 3.8.2. diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama new file mode 100755 index 0000000000..710ac0cb96 --- /dev/null +++ b/tool/lrama/exe/lrama @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +$LOAD_PATH << File.join(__dir__, "../lib") +require "lrama" + +Lrama::Command.new(ARGV.dup).run diff --git a/tool/lrama/lib/lrama.rb b/tool/lrama/lib/lrama.rb new file mode 100644 index 0000000000..56ba0044d4 --- /dev/null +++ b/tool/lrama/lib/lrama.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative "lrama/bitmap" +require_relative "lrama/command" +require_relative "lrama/context" +require_relative "lrama/counterexamples" +require_relative "lrama/diagram" +require_relative "lrama/digraph" +require_relative "lrama/erb" +require_relative "lrama/grammar" +require_relative "lrama/lexer" +require_relative "lrama/logger" +require_relative "lrama/option_parser" +require_relative "lrama/options" +require_relative "lrama/output" +require_relative "lrama/parser" +require_relative "lrama/reporter" +require_relative "lrama/state" +require_relative "lrama/states" +require_relative "lrama/tracer" +require_relative "lrama/version" +require_relative "lrama/warnings" diff --git a/tool/lrama/lib/lrama/bitmap.rb b/tool/lrama/lib/lrama/bitmap.rb new file mode 100644 index 0000000000..88b255b012 --- /dev/null +++ b/tool/lrama/lib/lrama/bitmap.rb @@ -0,0 +1,47 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + module Bitmap + # @rbs! + # type bitmap = Integer + + # @rbs (Array[Integer] ary) -> bitmap + def self.from_array(ary) + bit = 0 + + ary.each do |int| + bit |= (1 << int) + end + + bit + end + + # @rbs (Integer int) -> bitmap + def self.from_integer(int) + 1 << int + end + + # @rbs (bitmap int) -> Array[Integer] + def self.to_array(int) + a = [] #: Array[Integer] + i = 0 + + len = int.bit_length + while i < len do + if int[i] == 1 + a << i + end + + i += 1 + end + + a + end + + # @rbs (bitmap int, Integer size) -> Array[bool] + def self.to_bool_array(int, size) + Array.new(size) { |i| int[i] == 1 } + end + end +end diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb new file mode 100644 index 0000000000..17aad1a1c1 --- /dev/null +++ b/tool/lrama/lib/lrama/command.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Lrama + class Command + LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__))) + STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y') + + def initialize(argv) + @logger = Lrama::Logger.new + @options = OptionParser.parse(argv) + @tracer = Tracer.new(STDERR, **@options.trace_opts) + @reporter = Reporter.new(**@options.report_opts) + @warnings = Warnings.new(@logger, @options.warnings) + rescue => e + abort format_error_message(e.message) + end + + def run + Lrama::Reporter::Profile::CallStack.report(@options.profile_opts[:call_stack]) do + Lrama::Reporter::Profile::Memory.report(@options.profile_opts[:memory]) do + execute_command_workflow + end + end + end + + private + + def execute_command_workflow + @tracer.enable_duration + text = read_input + grammar = build_grammar(text) + states, context = compute_status(grammar) + render_reports(states) if @options.report_file + @tracer.trace(grammar) + render_diagram(grammar) + render_output(context, grammar) + states.validate!(@logger) + @warnings.warn(grammar, states) + end + + def read_input + text = @options.y.read + @options.y.close unless @options.y == STDIN + text + end + + def build_grammar(text) + grammar = + Lrama::Parser.new(text, @options.grammar_file, @options.debug, @options.locations, @options.define).parse + merge_stdlib(grammar) + prepare_grammar(grammar) + grammar + rescue => e + raise e if @options.debug + abort format_error_message(e.message) + end + + def format_error_message(message) + return message unless Exception.to_tty? + + message.gsub(/.+/, "\e[1m\\&\e[m") + end + + def merge_stdlib(grammar) + return if grammar.no_stdlib + + stdlib_text = File.read(STDLIB_FILE_PATH) + stdlib_grammar = Lrama::Parser.new( + stdlib_text, + STDLIB_FILE_PATH, + @options.debug, + @options.locations, + @options.define, + ).parse + + grammar.prepend_parameterized_rules(stdlib_grammar.parameterized_rules) + end + + def prepare_grammar(grammar) + grammar.prepare + grammar.validate! + end + + def compute_status(grammar) + states = Lrama::States.new(grammar, @tracer) + states.compute + states.compute_ielr if grammar.ielr_defined? + [states, Lrama::Context.new(states)] + end + + def render_reports(states) + File.open(@options.report_file, "w+") do |f| + @reporter.report(f, states) + end + end + + def render_diagram(grammar) + return unless @options.diagram + + File.open(@options.diagram_file, "w+") do |f| + Lrama::Diagram.render(out: f, grammar: grammar) + end + end + + def render_output(context, grammar) + File.open(@options.outfile, "w+") do |f| + Lrama::Output.new( + out: f, + output_file_path: @options.outfile, + template_name: @options.skeleton, + grammar_file_path: @options.grammar_file, + header_file_path: @options.header_file, + context: context, + grammar: grammar, + error_recovery: @options.error_recovery, + ).render + end + end + end +end diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb new file mode 100644 index 0000000000..eb068c1b9e --- /dev/null +++ b/tool/lrama/lib/lrama/context.rb @@ -0,0 +1,497 @@ +# frozen_string_literal: true + +require_relative "tracer/duration" + +module Lrama + # This is passed to a template + class Context + include Tracer::Duration + + ErrorActionNumber = -Float::INFINITY + BaseMin = -Float::INFINITY + + # TODO: It might be better to pass `states` to Output directly? + attr_reader :states, :yylast, :yypact_ninf, :yytable_ninf, :yydefact, :yydefgoto + + def initialize(states) + @states = states + @yydefact = nil + @yydefgoto = nil + # Array of array + @_actions = [] + + compute_tables + end + + # enum yytokentype + def yytokentype + @states.terms.reject do |term| + 0 < term.token_id && term.token_id < 128 + end.map do |term| + [term.id.s_value, term.token_id, term.display_name] + end.unshift(["YYEMPTY", -2, nil]) + end + + # enum yysymbol_kind_t + def yysymbol_kind_t + @states.symbols.map do |sym| + [sym.enum_name, sym.number, sym.comment] + end.unshift(["YYSYMBOL_YYEMPTY", -2, nil]) + end + + # State number of final (accepted) state + def yyfinal + @states.states.find do |state| + state.items.find do |item| + item.lhs.accept_symbol? && item.end_of_rule? + end + end.id + end + + # Number of terms + def yyntokens + @states.terms.count + end + + # Number of nterms + def yynnts + @states.nterms.count + end + + # Number of rules + def yynrules + @states.rules.count + end + + # Number of states + def yynstates + @states.states.count + end + + # Last token number + def yymaxutok + @states.terms.map(&:token_id).max + end + + # YYTRANSLATE + # + # yytranslate is a mapping from token id to symbol number + def yytranslate + # 2 is YYSYMBOL_YYUNDEF + a = Array.new(yymaxutok, 2) + + @states.terms.each do |term| + a[term.token_id] = term.number + end + + return a + end + + def yytranslate_inverted + a = Array.new(@states.symbols.count, @states.undef_symbol.token_id) + + @states.terms.each do |term| + a[term.number] = term.token_id + end + + return a + end + + # Mapping from rule number to line number of the rule is defined. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyrline + a = [0] + + @states.rules.each do |rule| + a << rule.lineno + end + + return a + end + + # Mapping from symbol number to its name + def yytname + @states.symbols.sort_by(&:number).map do |sym| + sym.display_name + end + end + + def yypact + @base[0...yynstates] + end + + def yypgoto + @base[yynstates..-1] + end + + def yytable + @table + end + + def yycheck + @check + end + + def yystos + @states.states.map do |state| + state.accessing_symbol.number + end + end + + # Mapping from rule number to symbol number of LHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr1 + a = [0] + + @states.rules.each do |rule| + a << rule.lhs.number + end + + return a + end + + # Mapping from rule number to length of RHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr2 + a = [0] + + @states.rules.each do |rule| + a << rule.rhs.count + end + + return a + end + + private + + # Compute these + # + # See also: "src/tables.c" of Bison. + # + # * yydefact + # * yydefgoto + # * yypact and yypgoto + # * yytable + # * yycheck + # * yypact_ninf + # * yytable_ninf + def compute_tables + report_duration(:compute_yydefact) { compute_yydefact } + report_duration(:compute_yydefgoto) { compute_yydefgoto } + report_duration(:sort_actions) { sort_actions } + # debug_sorted_actions + report_duration(:compute_packed_table) { compute_packed_table } + end + + def vectors_count + @states.states.count + @states.nterms.count + end + + # In compressed table, rule 0 is appended as an error case + # and reduce is represented as minus number. + def rule_id_to_action_number(rule_id) + (rule_id + 1) * -1 + end + + # Symbol number is assigned to term first then nterm. + # This method calculates sequence_number for nterm. + def nterm_number_to_sequence_number(nterm_number) + nterm_number - @states.terms.count + end + + # Vector is states + nterms + def nterm_number_to_vector_number(nterm_number) + @states.states.count + (nterm_number - @states.terms.count) + end + + def compute_yydefact + # Default action (shift/reduce/error) for each state. + # Index is state id, value is `rule id + 1` of a default reduction. + @yydefact = Array.new(@states.states.count, 0) + + @states.states.each do |state| + # Action number means + # + # * number = 0, default action + # * number = -Float::INFINITY, error by %nonassoc + # * number > 0, shift then move to state "number" + # * number < 0, reduce by "-number" rule. Rule "number" is already added by 1. + actions = Array.new(@states.terms.count, 0) + + if state.reduces.map(&:selected_look_ahead).any? {|la| !la.empty? } + # Iterate reduces with reverse order so that first rule is used. + state.reduces.reverse_each do |reduce| + reduce.look_ahead.each do |term| + actions[term.number] = rule_id_to_action_number(reduce.rule.id) + end + end + end + + # Shift is selected when S/R conflict exists. + state.selected_term_transitions.each do |shift| + actions[shift.next_sym.number] = shift.to_state.id + end + + state.resolved_conflicts.select do |conflict| + conflict.which == :error + end.each do |conflict| + actions[conflict.symbol.number] = ErrorActionNumber + end + + # If default_reduction_rule, replace default_reduction_rule in + # actions with zero. + if state.default_reduction_rule + actions.map! do |e| + if e == rule_id_to_action_number(state.default_reduction_rule.id) + 0 + else + e + end + end + end + + # If no default_reduction_rule, default behavior is an + # error then replace ErrorActionNumber with zero. + unless state.default_reduction_rule + actions.map! do |e| + if e == ErrorActionNumber + 0 + else + e + end + end + end + + s = actions.each_with_index.map do |n, i| + [i, n] + end.reject do |i, n| + # Remove default_reduction_rule entries + n == 0 + end + + if s.count != 0 + # Entry of @_actions is an array of + # + # * State id + # * Array of tuple, [from, to] where from is term number and to is action. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [state.id, s, s.count, s.last[0] - s.first[0] + 1] + end + + @yydefact[state.id] = state.default_reduction_rule ? state.default_reduction_rule.id + 1 : 0 + end + end + + def compute_yydefgoto + # Default GOTO (nterm transition) for each nterm. + # Index is sequence number of nterm, value is state id + # of a default nterm transition destination. + @yydefgoto = Array.new(@states.nterms.count, 0) + # Mapping from nterm to next_states + nterm_to_to_states = {} + + @states.states.each do |state| + state.nterm_transitions.each do |goto| + key = goto.next_sym + nterm_to_to_states[key] ||= [] + nterm_to_to_states[key] << [state, goto.to_state] # [from_state, to_state] + end + end + + @states.nterms.each do |nterm| + if (states = nterm_to_to_states[nterm]) + default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first + default_goto = default_state.id + not_default_gotos = [] + states.each do |from_state, to_state| + next if to_state.id == default_goto + not_default_gotos << [from_state.id, to_state.id] + end + else + default_goto = 0 + not_default_gotos = [] + end + + k = nterm_number_to_sequence_number(nterm.number) + @yydefgoto[k] = default_goto + + if not_default_gotos.count != 0 + v = nterm_number_to_vector_number(nterm.number) + + # Entry of @_actions is an array of + # + # * Nterm number as vector number + # * Array of tuple, [from, to] where from is state number and to is state number. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [v, not_default_gotos, not_default_gotos.count, not_default_gotos.last[0] - not_default_gotos.first[0] + 1] + end + end + end + + def sort_actions + # This is not same with #sort_actions + # + # @sorted_actions = @_actions.sort_by do |_, _, count, width| + # [-width, -count] + # end + + @sorted_actions = [] + + @_actions.each do |action| + if @sorted_actions.empty? + @sorted_actions << action + next + end + + j = @sorted_actions.count - 1 + _state_id, _froms_and_tos, count, width = action + + while (j >= 0) do + case + when @sorted_actions[j][3] < width + j -= 1 + when @sorted_actions[j][3] == width && @sorted_actions[j][2] < count + j -= 1 + else + break + end + end + + @sorted_actions.insert(j + 1, action) + end + end + + def debug_sorted_actions + ary = Array.new + @sorted_actions.each do |state_id, froms_and_tos, count, width| + ary[state_id] = [state_id, froms_and_tos, count, width] + end + + print sprintf("table_print:\n\n") + + print sprintf("order [\n") + vectors_count.times do |i| + print sprintf("%d, ", @sorted_actions[i] ? @sorted_actions[i][0] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("width [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][3] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("tally [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][2] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + end + + def compute_packed_table + # yypact and yypgoto + @base = Array.new(vectors_count, BaseMin) + # yytable + @table = [] + # yycheck + @check = [] + # Key is froms_and_tos, value is index position + pushed = {} + used_res = {} + lowzero = 0 + high = 0 + + @sorted_actions.each do |state_id, froms_and_tos, _, _| + if (res = pushed[froms_and_tos]) + @base[state_id] = res + next + end + + res = lowzero - froms_and_tos.first[0] + + # Find the smallest `res` such that `@table[res + from]` is empty for all `from` in `froms_and_tos` + while true do + advanced = false + + while used_res[res] + res += 1 + advanced = true + end + + froms_and_tos.each do |from, to| + while @table[res + from] + res += 1 + advanced = true + end + end + + unless advanced + # no advance means that the current `res` satisfies the condition + break + end + end + + loc = 0 + + froms_and_tos.each do |from, to| + loc = res + from + + @table[loc] = to + @check[loc] = from + end + + while (@table[lowzero]) do + lowzero += 1 + end + + high = loc if high < loc + + @base[state_id] = res + pushed[froms_and_tos] = res + used_res[res] = true + end + + @yylast = high + + # replace_ninf + @yypact_ninf = (@base.reject {|i| i == BaseMin } + [0]).min - 1 + @base.map! do |i| + case i + when BaseMin + @yypact_ninf + else + i + end + end + + @yytable_ninf = (@table.compact.reject {|i| i == ErrorActionNumber } + [0]).min - 1 + @table.map! do |i| + case i + when nil + 0 + when ErrorActionNumber + @yytable_ninf + else + i + end + end + + @check.map! do |i| + case i + when nil + -1 + else + i + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb new file mode 100644 index 0000000000..60d830d048 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -0,0 +1,426 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "set" +require "timeout" + +require_relative "counterexamples/derivation" +require_relative "counterexamples/example" +require_relative "counterexamples/node" +require_relative "counterexamples/path" +require_relative "counterexamples/state_item" +require_relative "counterexamples/triple" + +module Lrama + # See: https://www.cs.cornell.edu/andru/papers/cupex/cupex.pdf + # 4. Constructing Nonunifying Counterexamples + class Counterexamples + PathSearchTimeLimit = 10 # 10 sec + CumulativeTimeLimit = 120 # 120 sec + + # @rbs! + # @states: States + # @iterate_count: Integer + # @total_duration: Float + # @exceed_cumulative_time_limit: bool + # @state_items: Hash[[State, State::Item], StateItem] + # @triples: Hash[Integer, Triple] + # @transitions: Hash[[StateItem, Grammar::Symbol], StateItem] + # @reverse_transitions: Hash[[StateItem, Grammar::Symbol], Set[StateItem]] + # @productions: Hash[StateItem, Set[StateItem]] + # @reverse_productions: Hash[[State, Grammar::Symbol], Set[StateItem]] # Grammar::Symbol is nterm + # @state_item_shift: Integer + + attr_reader :transitions #: Hash[[StateItem, Grammar::Symbol], StateItem] + attr_reader :productions #: Hash[StateItem, Set[StateItem]] + + # @rbs (States states) -> void + def initialize(states) + @states = states + @iterate_count = 0 + @total_duration = 0 + @exceed_cumulative_time_limit = false + @triples = {} + setup_state_items + setup_transitions + setup_productions + end + + # @rbs () -> "#<Counterexamples>" + def to_s + "#<Counterexamples>" + end + alias :inspect :to_s + + # @rbs (State conflict_state) -> Array[Example] + def compute(conflict_state) + conflict_state.conflicts.flat_map do |conflict| + # Check cumulative time limit for not each path search method call but each conflict + # to avoid one of example's path to be nil. + next if @exceed_cumulative_time_limit + + case conflict.type + when :shift_reduce + # @type var conflict: State::ShiftReduceConflict + shift_reduce_example(conflict_state, conflict) + when :reduce_reduce + # @type var conflict: State::ReduceReduceConflict + reduce_reduce_examples(conflict_state, conflict) + end + rescue Timeout::Error => e + STDERR.puts "Counterexamples calculation for state #{conflict_state.id} #{e.message} with #{@iterate_count} iteration" + increment_total_duration(PathSearchTimeLimit) + nil + end.compact + end + + private + + # @rbs (State state, State::Item item) -> StateItem + def get_state_item(state, item) + @state_items[[state, item]] + end + + # For optimization, create all StateItem in advance + # and use them by fetching an instance from `@state_items`. + # Do not create new StateItem instance in the shortest path search process + # to avoid miss hash lookup. + # + # @rbs () -> void + def setup_state_items + @state_items = {} + count = 0 + + @states.states.each do |state| + state.items.each do |item| + @state_items[[state, item]] = StateItem.new(count, state, item) + count += 1 + end + end + + @state_item_shift = Math.log(count, 2).ceil + end + + # @rbs () -> void + def setup_transitions + @transitions = {} + @reverse_transitions = {} + + @states.states.each do |src_state| + trans = {} #: Hash[Grammar::Symbol, State] + + src_state.transitions.each do |transition| + trans[transition.next_sym] = transition.to_state + end + + src_state.items.each do |src_item| + next if src_item.end_of_rule? + sym = src_item.next_sym + dest_state = trans[sym] + + dest_state.kernels.each do |dest_item| + next unless (src_item.rule == dest_item.rule) && (src_item.position + 1 == dest_item.position) + src_state_item = get_state_item(src_state, src_item) + dest_state_item = get_state_item(dest_state, dest_item) + + @transitions[[src_state_item, sym]] = dest_state_item + + # @type var key: [StateItem, Grammar::Symbol] + key = [dest_state_item, sym] + @reverse_transitions[key] ||= Set.new + @reverse_transitions[key] << src_state_item + end + end + end + end + + # @rbs () -> void + def setup_productions + @productions = {} + @reverse_productions = {} + + @states.states.each do |state| + # Grammar::Symbol is LHS + h = {} #: Hash[Grammar::Symbol, Set[StateItem]] + + state.closure.each do |item| + sym = item.lhs + + h[sym] ||= Set.new + h[sym] << get_state_item(state, item) + end + + state.items.each do |item| + next if item.end_of_rule? + next if item.next_sym.term? + + sym = item.next_sym + state_item = get_state_item(state, item) + @productions[state_item] = h[sym] + + # @type var key: [State, Grammar::Symbol] + key = [state, sym] + @reverse_productions[key] ||= Set.new + @reverse_productions[key] << state_item + end + end + end + + # For optimization, use same Triple if it's already created. + # Do not create new Triple instance anywhere else + # to avoid miss hash lookup. + # + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> Triple + def get_triple(state_item, precise_lookahead_set) + key = (precise_lookahead_set << @state_item_shift) | state_item.id + @triples[key] ||= Triple.new(state_item, precise_lookahead_set) + end + + # @rbs (State conflict_state, State::ShiftReduceConflict conflict) -> Example + def shift_reduce_example(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + # @type var shift_conflict_item: ::Lrama::State::Item + shift_conflict_item = conflict_state.items.find { |item| item.next_sym == conflict_symbol } + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) + end + path1 = with_timeout("#find_shift_conflict_shortest_path:") do + find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + end + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + # @rbs (State conflict_state, State::ReduceReduceConflict conflict) -> Example + def reduce_reduce_examples(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + path1 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) + end + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + end + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + # @rbs (Array[StateItem]? reduce_state_items, State conflict_state, State::Item conflict_item) -> Array[StateItem] + def find_shift_conflict_shortest_path(reduce_state_items, conflict_state, conflict_item) + time1 = Time.now.to_f + @iterate_count = 0 + + target_state_item = get_state_item(conflict_state, conflict_item) + result = [target_state_item] + reversed_state_items = reduce_state_items.to_a.reverse + # Index for state_item + i = 0 + + while (state_item = reversed_state_items[i]) + # Index for prev_state_item + j = i + 1 + _j = j + + while (prev_state_item = reversed_state_items[j]) + if prev_state_item.type == :production + j += 1 + else + break + end + end + + if target_state_item == state_item || target_state_item.item.start_item? + result.concat( + reversed_state_items[_j..-1] #: Array[StateItem] + ) + break + end + + if target_state_item.type == :production + queue = [] #: Array[Node[StateItem]] + queue << Node.new(target_state_item, nil) + + # Find reverse production + while (sis = queue.shift) + @iterate_count += 1 + si = sis.elem + + # Reach to start state + if si.item.start_item? + a = Node.to_a(sis).reverse + a.shift + result.concat(a) + target_state_item = si + break + end + + if si.type == :production + # @type var key: [State, Grammar::Symbol] + key = [si.state, si.item.lhs] + @reverse_productions[key].each do |state_item| + queue << Node.new(state_item, sis) + end + else + # @type var key: [StateItem, Grammar::Symbol] + key = [si, si.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item&.state + a = Node.to_a(sis).reverse + a.shift + result.concat(a) + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + queue.clear + break + end + end + end + else + # Find reverse transition + # @type var key: [StateItem, Grammar::Symbol] + key = [target_state_item, target_state_item.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item&.state + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + break + end + end + end + + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "find_shift_conflict_shortest_path #{@iterate_count} iteration", duration) + end + + result.reverse + end + + # @rbs (StateItem target) -> Set[StateItem] + def reachable_state_items(target) + result = Set.new + queue = [target] + + while (state_item = queue.shift) + next if result.include?(state_item) + result << state_item + + @reverse_transitions[[state_item, state_item.item.previous_sym]]&.each do |prev_state_item| + queue << prev_state_item + end + + if state_item.item.beginning_of_rule? + @reverse_productions[[state_item.state, state_item.item.lhs]]&.each do |si| + queue << si + end + end + end + + result + end + + # @rbs (State conflict_state, State::Item conflict_reduce_item, Grammar::Symbol conflict_term) -> ::Array[StateItem]? + def shortest_path(conflict_state, conflict_reduce_item, conflict_term) + time1 = Time.now.to_f + @iterate_count = 0 + + queue = [] #: Array[[Triple, Path]] + visited = {} #: Hash[Triple, true] + start_state = @states.states.first #: Lrama::State + conflict_term_bit = Bitmap::from_integer(conflict_term.number) + raise "BUG: Start state should be just one kernel." if start_state.kernels.count != 1 + reachable = reachable_state_items(get_state_item(conflict_state, conflict_reduce_item)) + start = get_triple(get_state_item(start_state, start_state.kernels.first), Bitmap::from_integer(@states.eof_symbol.number)) + + queue << [start, Path.new(start.state_item, nil)] + + while (triple, path = queue.shift) + @iterate_count += 1 + + # Found + if (triple.state == conflict_state) && (triple.item == conflict_reduce_item) && (triple.l & conflict_term_bit != 0) + state_items = [path.state_item] + + while (path = path.parent) + state_items << path.state_item + end + + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "shortest_path #{@iterate_count} iteration", duration) + end + + return state_items.reverse + end + + # transition + next_state_item = @transitions[[triple.state_item, triple.item.next_sym]] + if next_state_item && reachable.include?(next_state_item) + # @type var t: Triple + t = get_triple(next_state_item, triple.l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end + end + + # production step + @productions[triple.state_item]&.each do |si| + next unless reachable.include?(si) + + l = follow_l(triple.item, triple.l) + # @type var t: Triple + t = get_triple(si, l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end + end + end + + return nil + end + + # @rbs (State::Item item, Bitmap::bitmap current_l) -> Bitmap::bitmap + def follow_l(item, current_l) + # 1. follow_L (A -> X1 ... Xn-1 • Xn) = L + # 2. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = {Xk+2} if Xk+2 is a terminal + # 3. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) if Xk+2 is a nonnullable nonterminal + # 4. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) + follow_L (A -> X1 ... Xk+1 • Xk+2 ... Xn) if Xk+2 is a nullable nonterminal + case + when item.number_of_rest_symbols == 1 + current_l + when item.next_next_sym.term? + item.next_next_sym.number_bitmap + when !item.next_next_sym.nullable + item.next_next_sym.first_set_bitmap + else + item.next_next_sym.first_set_bitmap | follow_l(item.new_by_next_position, current_l) + end + end + + # @rbs [T] (String message) { -> T } -> T + def with_timeout(message) + Timeout.timeout(PathSearchTimeLimit, Timeout::Error, message + " timeout of #{PathSearchTimeLimit} sec exceeded") do + yield + end + end + + # @rbs (Float|Integer duration) -> void + def increment_total_duration(duration) + @total_duration += duration + + if !@exceed_cumulative_time_limit && @total_duration > CumulativeTimeLimit + @exceed_cumulative_time_limit = true + STDERR.puts "CumulativeTimeLimit #{CumulativeTimeLimit} sec exceeded then skip following Counterexamples calculation" + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/derivation.rb b/tool/lrama/lib/lrama/counterexamples/derivation.rb new file mode 100644 index 0000000000..a2b74767a9 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/derivation.rb @@ -0,0 +1,76 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Derivation + # @rbs! + # @item: State::Item + # @left: Derivation? + + attr_reader :item #: State::Item + attr_reader :left #: Derivation? + attr_accessor :right #: Derivation? + + # @rbs (State::Item item, Derivation? left) -> void + def initialize(item, left) + @item = item + @left = left + end + + # @rbs () -> ::String + def to_s + "#<Derivation(#{item.display_name})>" + end + alias :inspect :to_s + + # @rbs () -> Array[String] + def render_strings_for_report + result = [] #: Array[String] + _render_for_report(self, 0, result, 0) + result.map(&:rstrip) + end + + # @rbs () -> String + def render_for_report + render_strings_for_report.join("\n") + end + + private + + # @rbs (Derivation derivation, Integer offset, Array[String] strings, Integer index) -> Integer + def _render_for_report(derivation, offset, strings, index) + item = derivation.item + if strings[index] + strings[index] << " " * (offset - strings[index].length) + else + strings[index] = " " * offset + end + str = strings[index] + str << "#{item.rule_id}: #{item.symbols_before_dot.map(&:display_name).join(" ")} " + + if derivation.left + len = str.length + str << "#{item.next_sym.display_name}" + length = _render_for_report(derivation.left, len, strings, index + 1) + # I want String#ljust! + str << " " * (length - str.length) if length > str.length + else + str << " • #{item.symbols_after_dot.map(&:display_name).join(" ")} " + return str.length + end + + if derivation.right&.left + left = derivation.right&.left #: Derivation + length = _render_for_report(left, str.length, strings, index + 1) + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore + str << " " * (length - str.length) if length > str.length + elsif item.next_next_sym + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore + end + + return str.length + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/example.rb b/tool/lrama/lib/lrama/counterexamples/example.rb new file mode 100644 index 0000000000..c007f45af4 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/example.rb @@ -0,0 +1,154 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Example + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @path1: ::Array[StateItem] + # @path2: ::Array[StateItem] + # @conflict: State::conflict + # @conflict_symbol: Grammar::Symbol + # @counterexamples: Counterexamples + # @derivations1: Derivation + # @derivations2: Derivation + + attr_reader :path1 #: ::Array[StateItem] + attr_reader :path2 #: ::Array[StateItem] + attr_reader :conflict #: State::conflict + attr_reader :conflict_symbol #: Grammar::Symbol + + # path1 is shift conflict when S/R conflict + # path2 is always reduce conflict + # + # @rbs (Array[StateItem]? path1, Array[StateItem]? path2, State::conflict conflict, Grammar::Symbol conflict_symbol, Counterexamples counterexamples) -> void + def initialize(path1, path2, conflict, conflict_symbol, counterexamples) + @path1 = path1 + @path2 = path2 + @conflict = conflict + @conflict_symbol = conflict_symbol + @counterexamples = counterexamples + end + + # @rbs () -> (:shift_reduce | :reduce_reduce) + def type + @conflict.type + end + + # @rbs () -> State::Item + def path1_item + @path1.last.item + end + + # @rbs () -> State::Item + def path2_item + @path2.last.item + end + + # @rbs () -> Derivation + def derivations1 + @derivations1 ||= _derivations(path1) + end + + # @rbs () -> Derivation + def derivations2 + @derivations2 ||= _derivations(path2) + end + + private + + # @rbs (Array[StateItem] state_items) -> Derivation + def _derivations(state_items) + derivation = nil #: Derivation + current = :production + last_state_item = state_items.last #: StateItem + lookahead_sym = last_state_item.item.end_of_rule? ? @conflict_symbol : nil + + state_items.reverse_each do |si| + item = si.item + + case current + when :production + case si.type + when :start + derivation = Derivation.new(item, derivation) + current = :start + when :transition + derivation = Derivation.new(item, derivation) + current = :transition + when :production + derivation = Derivation.new(item, derivation) + current = :production + else + raise "Unexpected. #{si}" + end + + if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym) + si2 = @counterexamples.transitions[[si, item.next_sym]] + derivation2 = find_derivation_for_symbol(si2, lookahead_sym) + derivation.right = derivation2 # steep:ignore + lookahead_sym = nil + end + + when :transition + case si.type + when :start + derivation = Derivation.new(item, derivation) + current = :start + when :transition + # ignore + current = :transition + when :production + # ignore + current = :production + end + else + raise "BUG: Unknown #{current}" + end + + break if current == :start + end + + derivation + end + + # @rbs (StateItem state_item, Grammar::Symbol sym) -> Derivation? + def find_derivation_for_symbol(state_item, sym) + queue = [] #: Array[Array[StateItem]] + queue << [state_item] + + while (sis = queue.shift) + si = sis.last + next_sym = si.item.next_sym + + if next_sym == sym + derivation = nil + + sis.reverse_each do |si| + derivation = Derivation.new(si.item, derivation) + end + + return derivation + end + + if next_sym.nterm? && next_sym.first_set.include?(sym) + @counterexamples.productions[si].each do |next_si| + next if next_si.item.empty_rule? + next if sis.include?(next_si) + queue << (sis + [next_si]) + end + + if next_sym.nullable + next_si = @counterexamples.transitions[[si, next_sym]] + queue << (sis + [next_si]) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/node.rb b/tool/lrama/lib/lrama/counterexamples/node.rb new file mode 100644 index 0000000000..9214a0e7f1 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/node.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + # @rbs generic E < Object -- Type of an element + class Node + attr_reader :elem #: E + attr_reader :next_node #: Node[E]? + + # @rbs [E < Object] (Node[E] node) -> Array[E] + def self.to_a(node) + a = [] # steep:ignore UnannotatedEmptyCollection + + while (node) + a << node.elem + node = node.next_node + end + + a + end + + # @rbs (E elem, Node[E]? next_node) -> void + def initialize(elem, next_node) + @elem = elem + @next_node = next_node + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/path.rb b/tool/lrama/lib/lrama/counterexamples/path.rb new file mode 100644 index 0000000000..6b1325f73b --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/path.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Path + # @rbs! + # @state_item: StateItem + # @parent: Path? + + attr_reader :state_item #: StateItem + attr_reader :parent #: Path? + + # @rbs (StateItem state_item, Path? parent) -> void + def initialize(state_item, parent) + @state_item = state_item + @parent = parent + end + + # @rbs () -> ::String + def to_s + "#<Path>" + end + alias :inspect :to_s + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/state_item.rb b/tool/lrama/lib/lrama/counterexamples/state_item.rb new file mode 100644 index 0000000000..8c2481d793 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/state_item.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class StateItem + attr_reader :id #: Integer + attr_reader :state #: State + attr_reader :item #: State::Item + + # @rbs (Integer id, State state, State::Item item) -> void + def initialize(id, state, item) + @id = id + @state = state + @item = item + end + + # @rbs () -> (:start | :transition | :production) + def type + case + when item.start_item? + :start + when item.beginning_of_rule? + :production + else + :transition + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/triple.rb b/tool/lrama/lib/lrama/counterexamples/triple.rb new file mode 100644 index 0000000000..98fe051f53 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/triple.rb @@ -0,0 +1,41 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Triple + attr_reader :precise_lookahead_set #: Bitmap::bitmap + + alias :l :precise_lookahead_set + + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> void + def initialize(state_item, precise_lookahead_set) + @state_item = state_item + @precise_lookahead_set = precise_lookahead_set + end + + # @rbs () -> State + def state + @state_item.state + end + alias :s :state + + # @rbs () -> State::Item + def item + @state_item.item + end + alias :itm :item + + # @rbs () -> StateItem + def state_item + @state_item + end + + # @rbs () -> ::String + def inspect + "#{state.inspect}. #{item.display_name}. #{l.to_s(2)}" + end + alias :to_s :inspect + end + end +end diff --git a/tool/lrama/lib/lrama/diagram.rb b/tool/lrama/lib/lrama/diagram.rb new file mode 100644 index 0000000000..985808933f --- /dev/null +++ b/tool/lrama/lib/lrama/diagram.rb @@ -0,0 +1,77 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Diagram + class << self + # @rbs (IO out, Grammar grammar, String template_name) -> void + def render(out:, grammar:, template_name: 'diagram/diagram.html') + return unless require_railroad_diagrams + new(out: out, grammar: grammar, template_name: template_name).render + end + + # @rbs () -> bool + def require_railroad_diagrams + require "railroad_diagrams" + true + rescue LoadError + warn "railroad_diagrams is not installed. Please run `bundle install`." + false + end + end + + # @rbs (IO out, Grammar grammar, String template_name) -> void + def initialize(out:, grammar:, template_name: 'diagram/diagram.html') + @grammar = grammar + @out = out + @template_name = template_name + end + + # @rbs () -> void + def render + RailroadDiagrams::TextDiagram.set_formatting(RailroadDiagrams::TextDiagram::PARTS_UNICODE) + @out << ERB.render(template_file, output: self) + end + + # @rbs () -> string + def default_style + RailroadDiagrams::Style::default_style + end + + # @rbs () -> string + def diagrams + result = +'' + @grammar.unique_rule_s_values.each do |s_value| + diagrams = + @grammar.select_rules_by_s_value(s_value).map { |r| r.to_diagrams } + add_diagram( + s_value, + RailroadDiagrams::Diagram.new( + RailroadDiagrams::Choice.new(0, *diagrams), + ), + result + ) + end + result + end + + private + + # @rbs () -> string + def template_dir + File.expand_path('../../template', __dir__) + end + + # @rbs () -> string + def template_file + File.join(template_dir, @template_name) + end + + # @rbs (String name, RailroadDiagrams::Diagram diagram, String result) -> void + def add_diagram(name, diagram, result) + result << "\n<h2 class=\"diagram-header\">#{RailroadDiagrams.escape_html(name)}</h2>" + diagram.write_svg(result.method(:<<)) + result << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/digraph.rb b/tool/lrama/lib/lrama/digraph.rb new file mode 100644 index 0000000000..52865f52dd --- /dev/null +++ b/tool/lrama/lib/lrama/digraph.rb @@ -0,0 +1,104 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Digraph Algorithm of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # + # Digraph is an algorithm for graph data structure. + # The algorithm efficiently traverses SCC (Strongly Connected Component) of graph + # and merges nodes attributes within the same SCC. + # + # `compute_read_sets` and `compute_follow_sets` have the same structure. + # Graph of gotos and attributes of gotos are given then compute propagated attributes for each node. + # + # In the case of `compute_read_sets`: + # + # * Set of gotos is nodes of graph + # * `reads_relation` is edges of graph + # * `direct_read_sets` is nodes attributes + # + # In the case of `compute_follow_sets`: + # + # * Set of gotos is nodes of graph + # * `includes_relation` is edges of graph + # * `read_sets` is nodes attributes + # + # + # @rbs generic X < Object -- Type of a node + # @rbs generic Y < _Or -- Type of attribute sets assigned to a node which should support merge operation (#| method) + class Digraph + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # interface _Or + # def |: (self) -> self + # end + # @sets: Array[X] + # @relation: Hash[X, Array[X]] + # @base_function: Hash[X, Y] + # @stack: Array[X] + # @h: Hash[X, (Integer|Float)?] + # @result: Hash[X, Y] + + # @rbs sets: Array[X] -- Nodes of graph + # @rbs relation: Hash[X, Array[X]] -- Edges of graph + # @rbs base_function: Hash[X, Y] -- Attributes of nodes + # @rbs return: void + def initialize(sets, relation, base_function) + + # X in the paper + @sets = sets + + # R in the paper + @relation = relation + + # F' in the paper + @base_function = base_function + + # S in the paper + @stack = [] + + # N in the paper + @h = Hash.new(0) + + # F in the paper + @result = {} + end + + # @rbs () -> Hash[X, Y] + def compute + @sets.each do |x| + next if @h[x] != 0 + traverse(x) + end + + return @result + end + + private + + # @rbs (X x) -> void + def traverse(x) + @stack.push(x) + d = @stack.count + @h[x] = d + @result[x] = @base_function[x] # F x = F' x + + @relation[x]&.each do |y| + traverse(y) if @h[y] == 0 + @h[x] = [@h[x], @h[y]].min + @result[x] |= @result[y] # F x = F x + F y + end + + if @h[x] == d + while (z = @stack.pop) do + @h[z] = Float::INFINITY + break if z == x + @result[z] = @result[x] # F (Top of S) = F x + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/erb.rb b/tool/lrama/lib/lrama/erb.rb new file mode 100644 index 0000000000..8f8be54811 --- /dev/null +++ b/tool/lrama/lib/lrama/erb.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "erb" + +module Lrama + class ERB + # @rbs (String file, **untyped kwargs) -> String + def self.render(file, **kwargs) + new(file).render(**kwargs) + end + + # @rbs (String file) -> void + def initialize(file) + input = File.read(file) + if ::ERB.instance_method(:initialize).parameters.last.first == :key + @erb = ::ERB.new(input, trim_mode: '-') + else + @erb = ::ERB.new(input, nil, '-') # steep:ignore UnexpectedPositionalArgument + end + @erb.filename = file + end + + # @rbs (**untyped kwargs) -> String + def render(**kwargs) + @erb.result_with_hash(kwargs) + end + end +end diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb new file mode 100644 index 0000000000..95a80bb01c --- /dev/null +++ b/tool/lrama/lib/lrama/grammar.rb @@ -0,0 +1,603 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "grammar/auxiliary" +require_relative "grammar/binding" +require_relative "grammar/code" +require_relative "grammar/counter" +require_relative "grammar/destructor" +require_relative "grammar/error_token" +require_relative "grammar/inline" +require_relative "grammar/parameterized" +require_relative "grammar/percent_code" +require_relative "grammar/precedence" +require_relative "grammar/printer" +require_relative "grammar/reference" +require_relative "grammar/rule" +require_relative "grammar/rule_builder" +require_relative "grammar/symbol" +require_relative "grammar/symbols" +require_relative "grammar/type" +require_relative "grammar/union" +require_relative "lexer" + +module Lrama + # Grammar is the result of parsing an input grammar file + class Grammar + # @rbs! + # + # interface _DelegatedMethods + # def rules: () -> Array[Rule] + # def accept_symbol: () -> Grammar::Symbol + # def eof_symbol: () -> Grammar::Symbol + # def undef_symbol: () -> Grammar::Symbol + # def precedences: () -> Array[Precedence] + # + # # delegate to @symbols_resolver + # def symbols: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def ielr_defined?: () -> bool + # end + # + # include Symbols::Resolver::_DelegatedMethods + # + # @rule_counter: Counter + # @percent_codes: Array[PercentCode] + # @printers: Array[Printer] + # @destructors: Array[Destructor] + # @error_tokens: Array[ErrorToken] + # @symbols_resolver: Symbols::Resolver + # @types: Array[Type] + # @rule_builders: Array[RuleBuilder] + # @rules: Array[Rule] + # @sym_to_rules: Hash[Integer, Array[Rule]] + # @parameterized_resolver: Parameterized::Resolver + # @empty_symbol: Grammar::Symbol + # @eof_symbol: Grammar::Symbol + # @error_symbol: Grammar::Symbol + # @undef_symbol: Grammar::Symbol + # @accept_symbol: Grammar::Symbol + # @aux: Auxiliary + # @no_stdlib: bool + # @locations: bool + # @define: Hash[String, String] + # @required: bool + # @union: Union + # @precedences: Array[Precedence] + # @start_nterm: Lrama::Lexer::Token::Base? + + extend Forwardable + + attr_reader :percent_codes #: Array[PercentCode] + attr_reader :eof_symbol #: Grammar::Symbol + attr_reader :error_symbol #: Grammar::Symbol + attr_reader :undef_symbol #: Grammar::Symbol + attr_reader :accept_symbol #: Grammar::Symbol + attr_reader :aux #: Auxiliary + attr_reader :parameterized_resolver #: Parameterized::Resolver + attr_reader :precedences #: Array[Precedence] + attr_accessor :union #: Union + attr_accessor :expect #: Integer + attr_accessor :printers #: Array[Printer] + attr_accessor :error_tokens #: Array[ErrorToken] + attr_accessor :lex_param #: String + attr_accessor :parse_param #: String + attr_accessor :initial_action #: Grammar::Code::InitialActionCode + attr_accessor :after_shift #: Lexer::Token::Base + attr_accessor :before_reduce #: Lexer::Token::Base + attr_accessor :after_reduce #: Lexer::Token::Base + attr_accessor :after_shift_error_token #: Lexer::Token::Base + attr_accessor :after_pop_stack #: Lexer::Token::Base + attr_accessor :symbols_resolver #: Symbols::Resolver + attr_accessor :types #: Array[Type] + attr_accessor :rules #: Array[Rule] + attr_accessor :rule_builders #: Array[RuleBuilder] + attr_accessor :sym_to_rules #: Hash[Integer, Array[Rule]] + attr_accessor :no_stdlib #: bool + attr_accessor :locations #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :required #: bool + + def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value, + :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, + :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type, + :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number! + + # @rbs (Counter rule_counter, bool locations, Hash[String, String] define) -> void + def initialize(rule_counter, locations, define = {}) + @rule_counter = rule_counter + + # Code defined by "%code" + @percent_codes = [] + @printers = [] + @destructors = [] + @error_tokens = [] + @symbols_resolver = Grammar::Symbols::Resolver.new + @types = [] + @rule_builders = [] + @rules = [] + @sym_to_rules = {} + @parameterized_resolver = Parameterized::Resolver.new + @empty_symbol = nil + @eof_symbol = nil + @error_symbol = nil + @undef_symbol = nil + @accept_symbol = nil + @aux = Auxiliary.new + @no_stdlib = false + @locations = locations + @define = define + @required = false + @precedences = [] + @start_nterm = nil + + append_special_symbols + end + + # @rbs (Counter rule_counter, Counter midrule_action_counter) -> RuleBuilder + def create_rule_builder(rule_counter, midrule_action_counter) + RuleBuilder.new(rule_counter, midrule_action_counter, @parameterized_resolver) + end + + # @rbs (id: Lexer::Token::Base, code: Lexer::Token::UserCode) -> Array[PercentCode] + def add_percent_code(id:, code:) + @percent_codes << PercentCode.new(id.s_value, code.s_value) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Destructor] + def add_destructor(ident_or_tags:, token_code:, lineno:) + @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Printer] + def add_printer(ident_or_tags:, token_code:, lineno:) + @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[ErrorToken] + def add_error_token(ident_or_tags:, token_code:, lineno:) + @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> Array[Type] + def add_type(id:, tag:) + @types << Type.new(id: id, tag: tag) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_nonassoc(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :nonassoc, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_left(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :left, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_right(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :right, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_precedence(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :precedence, precedence: precedence, lineno: lineno)) + end + + # @rbs (Lrama::Lexer::Token::Base id) -> Lrama::Lexer::Token::Base + def set_start_nterm(id) + # When multiple `%start` directives are defined, Bison does not generate an error, + # whereas Lrama does generate an error. + # Related Bison's specification are + # refs: https://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html + if @start_nterm.nil? + @start_nterm = id + else + start = @start_nterm #: Lrama::Lexer::Token::Base + raise "Start non-terminal is already set to #{start.s_value} (line: #{start.first_line}). Cannot set to #{id.s_value} (line: #{id.first_line})." + end + end + + # @rbs (Grammar::Symbol sym, Precedence precedence) -> (Precedence | bot) + def set_precedence(sym, precedence) + @precedences << precedence + sym.precedence = precedence + end + + # @rbs (Grammar::Code::NoReferenceCode code, Integer lineno) -> Union + def set_union(code, lineno) + @union = Union.new(code: code, lineno: lineno) + end + + # @rbs (RuleBuilder builder) -> Array[RuleBuilder] + def add_rule_builder(builder) + @rule_builders << builder + end + + # @rbs (Parameterized::Rule rule) -> Array[Parameterized::Rule] + def add_parameterized_rule(rule) + @parameterized_resolver.add_rule(rule) + end + + # @rbs () -> Array[Parameterized::Rule] + def parameterized_rules + @parameterized_resolver.rules + end + + # @rbs (Array[Parameterized::Rule] rules) -> Array[Parameterized::Rule] + def prepend_parameterized_rules(rules) + @parameterized_resolver.rules = rules + @parameterized_resolver.rules + end + + # @rbs (Integer prologue_first_lineno) -> Integer + def prologue_first_lineno=(prologue_first_lineno) + @aux.prologue_first_lineno = prologue_first_lineno + end + + # @rbs (String prologue) -> String + def prologue=(prologue) + @aux.prologue = prologue + end + + # @rbs (Integer epilogue_first_lineno) -> Integer + def epilogue_first_lineno=(epilogue_first_lineno) + @aux.epilogue_first_lineno = epilogue_first_lineno + end + + # @rbs (String epilogue) -> String + def epilogue=(epilogue) + @aux.epilogue = epilogue + end + + # @rbs () -> void + def prepare + resolve_inline_rules + normalize_rules + collect_symbols + set_lhs_and_rhs + fill_default_precedence + fill_symbols + fill_sym_to_rules + sort_precedence + compute_nullable + compute_first_set + set_locations + end + + # TODO: More validation methods + # + # * Validation for no_declared_type_reference + # + # @rbs () -> void + def validate! + @symbols_resolver.validate! + validate_no_precedence_for_nterm! + validate_rule_lhs_is_nterm! + validate_duplicated_precedence! + end + + # @rbs (Grammar::Symbol sym) -> Array[Rule] + def find_rules_by_symbol!(sym) + find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found") + end + + # @rbs (Grammar::Symbol sym) -> Array[Rule]? + def find_rules_by_symbol(sym) + @sym_to_rules[sym.number] + end + + # @rbs (String s_value) -> Array[Rule] + def select_rules_by_s_value(s_value) + @rules.select {|rule| rule.lhs.id.s_value == s_value } + end + + # @rbs () -> Array[String] + def unique_rule_s_values + @rules.map {|rule| rule.lhs.id.s_value }.uniq + end + + # @rbs () -> bool + def ielr_defined? + @define.key?('lr.type') && @define['lr.type'] == 'ielr' + end + + private + + # @rbs () -> void + def sort_precedence + @precedences.sort_by! do |prec| + prec.symbol.number + end + @precedences.freeze + end + + # @rbs () -> Array[Grammar::Symbol] + def compute_nullable + @rules.each do |rule| + case + when rule.empty_rule? + rule.nullable = true + when rule.rhs.any?(&:term) + rule.nullable = false + else + # noop + end + end + + while true do + rs = @rules.select {|e| e.nullable.nil? } + nts = nterms.select {|e| e.nullable.nil? } + rule_count_1 = rs.count + nterm_count_1 = nts.count + + rs.each do |rule| + if rule.rhs.all?(&:nullable) + rule.nullable = true + end + end + + nts.each do |nterm| + find_rules_by_symbol!(nterm).each do |rule| + if rule.nullable + nterm.nullable = true + end + end + end + + rule_count_2 = @rules.count {|e| e.nullable.nil? } + nterm_count_2 = nterms.count {|e| e.nullable.nil? } + + if (rule_count_1 == rule_count_2) && (nterm_count_1 == nterm_count_2) + break + end + end + + rules.select {|r| r.nullable.nil? }.each do |rule| + rule.nullable = false + end + + nterms.select {|e| e.nullable.nil? }.each do |nterm| + nterm.nullable = false + end + end + + # @rbs () -> Array[Grammar::Symbol] + def compute_first_set + terms.each do |term| + term.first_set = Set.new([term]).freeze + term.first_set_bitmap = Lrama::Bitmap.from_array([term.number]) + end + + nterms.each do |nterm| + nterm.first_set = Set.new([]).freeze + nterm.first_set_bitmap = Lrama::Bitmap.from_array([]) + end + + while true do + changed = false + + @rules.each do |rule| + rule.rhs.each do |r| + if rule.lhs.first_set_bitmap | r.first_set_bitmap != rule.lhs.first_set_bitmap + changed = true + rule.lhs.first_set_bitmap = rule.lhs.first_set_bitmap | r.first_set_bitmap + end + + break unless r.nullable + end + end + + break unless changed + end + + nterms.each do |nterm| + nterm.first_set = Lrama::Bitmap.to_array(nterm.first_set_bitmap).map do |number| + find_symbol_by_number!(number) + end.to_set + end + end + + # @rbs () -> Array[RuleBuilder] + def setup_rules + @rule_builders.each do |builder| + builder.setup_rules + end + end + + # @rbs () -> Grammar::Symbol + def append_special_symbols + # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated + # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) + # term.number = -2 + # @empty_symbol = term + + # YYEOF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYEOF"), alias_name: "\"end of file\"", token_id: 0) + term.number = 0 + term.eof_symbol = true + @eof_symbol = term + + # YYerror + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYerror"), alias_name: "error") + term.number = 1 + term.error_symbol = true + @error_symbol = term + + # YYUNDEF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYUNDEF"), alias_name: "\"invalid token\"") + term.number = 2 + term.undef_symbol = true + @undef_symbol = term + + # $accept + term = add_nterm(id: Lrama::Lexer::Token::Ident.new(s_value: "$accept")) + term.accept_symbol = true + @accept_symbol = term + end + + # @rbs () -> void + def resolve_inline_rules + while @rule_builders.any?(&:has_inline_rules?) do + @rule_builders = @rule_builders.flat_map do |builder| + if builder.has_inline_rules? + Inline::Resolver.new(builder).resolve + else + builder + end + end + end + end + + # @rbs () -> void + def normalize_rules + add_accept_rule + setup_rules + @rule_builders.each do |builder| + builder.rules.each do |rule| + add_nterm(id: rule._lhs, tag: rule.lhs_tag) + @rules << rule + end + end + + nterms.freeze + @rules.sort_by!(&:id).freeze + end + + # Add $accept rule to the top of rules + def add_accept_rule + if @start_nterm + start = @start_nterm #: Lrama::Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [start, @eof_symbol.id], token_code: nil, lineno: start.line) + else + rule_builder = @rule_builders.first #: RuleBuilder + lineno = rule_builder ? rule_builder.line : 0 + lhs = rule_builder.lhs #: Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + end + end + + # Collect symbols from rules + # + # @rbs () -> void + def collect_symbols + @rules.flat_map(&:_rhs).each do |s| + case s + when Lrama::Lexer::Token::Char + add_term(id: s) + when Lrama::Lexer::Token::Base + # skip + else + raise "Unknown class: #{s}" + end + end + + terms.freeze + end + + # @rbs () -> void + def set_lhs_and_rhs + @rules.each do |rule| + rule.lhs = token_to_symbol(rule._lhs) if rule._lhs + + rule.rhs = rule._rhs.map do |t| + token_to_symbol(t) + end + end + end + + # Rule inherits precedence from the last term in RHS. + # + # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html + # + # @rbs () -> void + def fill_default_precedence + @rules.each do |rule| + # Explicitly specified precedence has the highest priority + next if rule.precedence_sym + + precedence_sym = nil + rule.rhs.each do |sym| + precedence_sym = sym if sym.term? + end + + rule.precedence_sym = precedence_sym + end + end + + # @rbs () -> Array[Grammar::Symbol] + def fill_symbols + fill_symbol_number + fill_nterm_type(@types) + fill_printer(@printers) + fill_destructor(@destructors) + fill_error_token(@error_tokens) + sort_by_number! + end + + # @rbs () -> Array[Rule] + def fill_sym_to_rules + @rules.each do |rule| + key = rule.lhs.number + @sym_to_rules[key] ||= [] + @sym_to_rules[key] << rule + end + end + + # @rbs () -> void + def validate_no_precedence_for_nterm! + errors = [] #: Array[String] + + nterms.each do |nterm| + next if nterm.precedence.nil? + + errors << "[BUG] Precedence #{nterm.name} (line: #{nterm.precedence.lineno}) is defined for nonterminal symbol (line: #{nterm.id.first_line}). Precedence can be defined for only terminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void + def validate_rule_lhs_is_nterm! + errors = [] #: Array[String] + + rules.each do |rule| + next if rule.lhs.nterm? + + errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is terminal symbol. It should be nonterminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # # @rbs () -> void + def validate_duplicated_precedence! + errors = [] #: Array[String] + seen = {} #: Hash[String, Precedence] + + precedences.each do |prec| + s_value = prec.s_value + if first = seen[s_value] + errors << "%#{prec.type} redeclaration for #{s_value} (line: #{prec.lineno}) previous declaration was %#{first.type} (line: #{first.lineno})" + else + seen[s_value] = prec + end + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void + def set_locations + @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? } + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/auxiliary.rb b/tool/lrama/lib/lrama/grammar/auxiliary.rb new file mode 100644 index 0000000000..76cfb74d4d --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/auxiliary.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # Grammar file information not used by States but by Output + class Auxiliary + attr_accessor :prologue_first_lineno #: Integer? + attr_accessor :prologue #: String? + attr_accessor :epilogue_first_lineno #: Integer? + attr_accessor :epilogue #: String? + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/binding.rb b/tool/lrama/lib/lrama/grammar/binding.rb new file mode 100644 index 0000000000..5940d153a9 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/binding.rb @@ -0,0 +1,79 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Binding + # @rbs @actual_args: Array[Lexer::Token::Base] + # @rbs @param_to_arg: Hash[String, Lexer::Token::Base] + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> void + def initialize(params, actual_args) + @actual_args = actual_args + @param_to_arg = build_param_to_arg(params, @actual_args) + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def resolve_symbol(sym) + return create_instantiate_rule(sym) if sym.is_a?(Lexer::Token::InstantiateRule) + find_arg_for_param(sym) + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def concatenated_args_str(token) + "#{token.rule_name}_#{format_args(token)}" + end + + private + + # @rbs (Lexer::Token::InstantiateRule sym) -> Lexer::Token::InstantiateRule + def create_instantiate_rule(sym) + Lrama::Lexer::Token::InstantiateRule.new( + s_value: sym.s_value, + location: sym.location, + args: resolve_args(sym.args), + lhs_tag: sym.lhs_tag + ) + end + + # @rbs (Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def resolve_args(args) + args.map { |arg| resolve_symbol(arg) } + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def find_arg_for_param(sym) + if (arg = @param_to_arg[sym.s_value]&.dup) + arg.alias_name = sym.alias_name + arg + else + sym + end + end + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> Hash[String, Lexer::Token::Base?] + def build_param_to_arg(params, actual_args) + params.zip(actual_args).map do |param, arg| + [param.s_value, arg] + end.to_h + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def format_args(token) + token_to_args_s_values(token).join('_') + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Array[String] + def token_to_args_s_values(token) + token.args.flat_map do |arg| + resolved = resolve_symbol(arg) + if resolved.is_a?(Lexer::Token::InstantiateRule) + [resolved.s_value] + resolved.args.map(&:s_value) + else + [resolved.s_value] + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb new file mode 100644 index 0000000000..f1b860eeba --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -0,0 +1,68 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "code/destructor_code" +require_relative "code/initial_action_code" +require_relative "code/no_reference_code" +require_relative "code/printer_code" +require_relative "code/rule_action" + +module Lrama + class Grammar + class Code + # @rbs! + # + # # delegated + # def s_value: -> String + # def line: -> Integer + # def column: -> Integer + # def references: -> Array[Lrama::Grammar::Reference] + + extend Forwardable + + def_delegators "token_code", :s_value, :line, :column, :references + + attr_reader :type #: ::Symbol + attr_reader :token_code #: Lexer::Token::UserCode + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode) -> void + def initialize(type:, token_code:) + @type = type + @token_code = token_code + end + + # @rbs (Code other) -> bool + def ==(other) + self.class == other.class && + self.type == other.type && + self.token_code == other.token_code + end + + # $$, $n, @$, @n are translated to C code + # + # @rbs () -> String + def translated_code + t_code = s_value.dup + + references.reverse_each do |ref| + first_column = ref.first_column + last_column = ref.last_column + + str = reference_to_c(ref) + + t_code[first_column...last_column] = str + end + + return t_code + end + + private + + # @rbs (Lrama::Grammar::Reference ref) -> bot + def reference_to_c(ref) + raise NotImplementedError.new("#reference_to_c is not implemented") + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb new file mode 100644 index 0000000000..d71b62e513 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class DestructorCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb new file mode 100644 index 0000000000..cb36041524 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class InitialActionCode < Code + private + + # * ($$) yylval + # * (@$) yylloc + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + "yylval" + when ref.type == :at && ref.name == "$" # @$ + "yylloc" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in initial_action." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in initial_action." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in initial_action." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in initial_action." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb new file mode 100644 index 0000000000..1d39919979 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class NoReferenceCode < Code + private + + # * ($$) error + # * (@$) error + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> bot + def reference_to_c(ref) + case + when ref.type == :dollar # $$, $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @$, @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:$, $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/printer_code.rb b/tool/lrama/lib/lrama/grammar/code/printer_code.rb new file mode 100644 index 0000000000..c6e25d5235 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/printer_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class PrinterCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/rule_action.rb b/tool/lrama/lib/lrama/grammar/code/rule_action.rb new file mode 100644 index 0000000000..e71e93e5a5 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/rule_action.rb @@ -0,0 +1,109 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class RuleAction < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @rule: Rule + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, rule: Rule) -> void + def initialize(type:, token_code:, rule:) + super(type: type, token_code: token_code) + @rule = rule + end + + private + + # * ($$) yyval + # * (@$) yyloc + # * ($:$) error + # * ($1) yyvsp[i] + # * (@1) yylsp[i] + # * ($:1) i - 1 + # + # + # Consider a rule like + # + # class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # + # For the semantic action of original rule: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 $2 $3 $4 $5 + # "Index for yyvsp" -4 -3 -2 -1 0 + # "$:n" $:1 $:2 $:3 $:4 $:5 + # "index of $:n" -5 -4 -3 -2 -1 + # + # + # For the first midrule action: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 + # "Index for yyvsp" 0 + # "$:n" $:1 + # + # @rbs (Reference ref) -> String + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + tag = ref.ex_tag || lhs.tag + raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag + "(yyval.#{tag.member})" + when ref.type == :at && ref.name == "$" # @$ + "(yyloc)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:$ is not supported" + when ref.type == :dollar # $n + i = -position_in_rhs + ref.index + tag = ref.ex_tag || rhs[ref.index - 1].tag + raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag + "(yyvsp[#{i}].#{tag.member})" + when ref.type == :at # @n + i = -position_in_rhs + ref.index + "(yylsp[#{i}])" + when ref.type == :index # $:n + i = -position_in_rhs + ref.index + "(#{i} - 1)" + else + raise "Unexpected. #{self}, #{ref}" + end + end + + # @rbs () -> Integer + def position_in_rhs + # If rule is not derived rule, User Code is only action at + # the end of rule RHS. In such case, the action is located on + # `@rule.rhs.count`. + @rule.position_in_original_rule_rhs || @rule.rhs.count + end + + # If this is midrule action, RHS is an RHS of the original rule. + # + # @rbs () -> Array[Grammar::Symbol] + def rhs + (@rule.original_rule || @rule).rhs + end + + # Unlike `rhs`, LHS is always an LHS of the rule. + # + # @rbs () -> Grammar::Symbol + def lhs + @rule.lhs + end + + # @rbs (Reference ref) -> bot + def raise_tag_not_found_error(ref) + raise "Tag is not specified for '$#{ref.value}' in '#{@rule.display_name}'" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/counter.rb b/tool/lrama/lib/lrama/grammar/counter.rb new file mode 100644 index 0000000000..ced934309d --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/counter.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Counter + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @number: Integer + + # @rbs (Integer number) -> void + def initialize(number) + @number = number + end + + # @rbs () -> Integer + def increment + n = @number + @number += 1 + n + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/destructor.rb b/tool/lrama/lib/lrama/grammar/destructor.rb new file mode 100644 index 0000000000..0ce8611e77 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/destructor.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Destructor + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident|Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::DestructorCode.new(type: :destructor, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/error_token.rb b/tool/lrama/lib/lrama/grammar/error_token.rb new file mode 100644 index 0000000000..9d9ed54ae2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/error_token.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class ErrorToken + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident | Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::PrinterCode.new(type: :error_token, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/inline.rb b/tool/lrama/lib/lrama/grammar/inline.rb new file mode 100644 index 0000000000..c02ab6002b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative 'inline/resolver' diff --git a/tool/lrama/lib/lrama/grammar/inline/resolver.rb b/tool/lrama/lib/lrama/grammar/inline/resolver.rb new file mode 100644 index 0000000000..aca689ccfb --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline/resolver.rb @@ -0,0 +1,80 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Inline + class Resolver + # @rbs (Lrama::Grammar::RuleBuilder rule_builder) -> void + def initialize(rule_builder) + @rule_builder = rule_builder + end + + # @rbs () -> Array[Lrama::Grammar::RuleBuilder] + def resolve + resolved_builders = [] #: Array[Lrama::Grammar::RuleBuilder] + @rule_builder.rhs.each_with_index do |token, i| + if (rule = @rule_builder.parameterized_resolver.find_inline(token)) + rule.rhs.each do |rhs| + builder = build_rule(rhs, token, i, rule) + resolved_builders << builder + end + break + end + end + resolved_builders + end + + private + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Lrama::Lexer::Token token, Integer index, Lrama::Grammar::Parameterized::Rule rule) -> Lrama::Grammar::RuleBuilder + def build_rule(rhs, token, index, rule) + builder = RuleBuilder.new( + @rule_builder.rule_counter, + @rule_builder.midrule_action_counter, + @rule_builder.parameterized_resolver, + lhs_tag: @rule_builder.lhs_tag + ) + resolve_rhs(builder, rhs, index, token, rule) + builder.lhs = @rule_builder.lhs + builder.line = @rule_builder.line + builder.precedence_sym = @rule_builder.precedence_sym + builder.user_code = replace_user_code(rhs, index) + builder + end + + # @rbs (Lrama::Grammar::RuleBuilder builder, Lrama::Grammar::Parameterized::Rhs rhs, Integer index, Lrama::Lexer::Token token, Lrama::Grammar::Parameterized::Rule rule) -> void + def resolve_rhs(builder, rhs, index, token, rule) + @rule_builder.rhs.each_with_index do |tok, i| + if i == index + rhs.symbols.each do |sym| + if token.is_a?(Lexer::Token::InstantiateRule) + bindings = Binding.new(rule.parameters, token.args) + builder.add_rhs(bindings.resolve_symbol(sym)) + else + builder.add_rhs(sym) + end + end + else + builder.add_rhs(tok) + end + end + end + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Integer index) -> Lrama::Lexer::Token::UserCode + def replace_user_code(rhs, index) + user_code = @rule_builder.user_code + return user_code if rhs.user_code.nil? || user_code.nil? + + code = user_code.s_value.gsub(/\$#{index + 1}/, rhs.user_code.s_value) + user_code.references.each do |ref| + next if ref.index.nil? || ref.index <= index # nil は $$ の場合 + code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (rhs.symbols.count - 1)}") + code = code.gsub(/@#{ref.index}/, "@#{ref.index + (rhs.symbols.count - 1)}") + end + Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized.rb b/tool/lrama/lib/lrama/grammar/parameterized.rb new file mode 100644 index 0000000000..48db3433f3 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'parameterized/resolver' +require_relative 'parameterized/rhs' +require_relative 'parameterized/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb new file mode 100644 index 0000000000..558f308190 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Resolver + attr_accessor :rules #: Array[Rule] + attr_accessor :created_lhs_list #: Array[Lexer::Token::Base] + + # @rbs () -> void + def initialize + @rules = [] + @created_lhs_list = [] + end + + # @rbs (Rule rule) -> Array[Rule] + def add_rule(rule) + @rules << rule + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Rule? + def find_rule(token) + select_rules(@rules, token).last + end + + # @rbs (Lexer::Token::Base token) -> Rule? + def find_inline(token) + @rules.reverse.find { |rule| rule.name == token.s_value && rule.inline? } + end + + # @rbs (String lhs_s_value) -> Lexer::Token::Base? + def created_lhs(lhs_s_value) + @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } + end + + # @rbs () -> Array[Rule] + def redefined_rules + @rules.select { |rule| @rules.count { |r| r.name == rule.name && r.required_parameters_count == rule.required_parameters_count } > 1 } + end + + private + + # @rbs (Array[Rule] rules, Lexer::Token::InstantiateRule token) -> Array[Rule] + def select_rules(rules, token) + rules = reject_inline_rules(rules) + rules = select_rules_by_name(rules, token.rule_name) + rules = rules.select { |rule| rule.required_parameters_count == token.args_count } + if rules.empty? + raise "Invalid number of arguments. `#{token.rule_name}`" + else + rules + end + end + + # @rbs (Array[Rule] rules) -> Array[Rule] + def reject_inline_rules(rules) + rules.reject(&:inline?) + end + + # @rbs (Array[Rule] rules, String rule_name) -> Array[Rule] + def select_rules_by_name(rules, rule_name) + rules = rules.select { |rule| rule.name == rule_name } + if rules.empty? + raise "Parameterized rule does not exist. `#{rule_name}`" + else + rules + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb new file mode 100644 index 0000000000..663de49100 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rhs + attr_accessor :symbols #: Array[Lexer::Token::Base] + attr_accessor :user_code #: Lexer::Token::UserCode? + attr_accessor :precedence_sym #: Grammar::Symbol? + + # @rbs () -> void + def initialize + @symbols = [] + @user_code = nil + @precedence_sym = nil + end + + # @rbs (Grammar::Binding bindings) -> Lexer::Token::UserCode? + def resolve_user_code(bindings) + return unless user_code + + resolved = Lexer::Token::UserCode.new(s_value: user_code.s_value, location: user_code.location) + var_to_arg = {} #: Hash[String, String] + symbols.each do |sym| + resolved_sym = bindings.resolve_symbol(sym) + if resolved_sym != sym + var_to_arg[sym.s_value] = resolved_sym.s_value + end + end + + var_to_arg.each do |var, arg| + resolved.references.each do |ref| + if ref.name == var + ref.name = arg + end + end + end + + return resolved + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rule.rb b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb new file mode 100644 index 0000000000..7048be3cff --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb @@ -0,0 +1,36 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rule + attr_reader :name #: String + attr_reader :parameters #: Array[Lexer::Token::Base] + attr_reader :rhs #: Array[Rhs] + attr_reader :required_parameters_count #: Integer + attr_reader :tag #: Lexer::Token::Tag? + + # @rbs (String name, Array[Lexer::Token::Base] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void + def initialize(name, parameters, rhs, tag: nil, is_inline: false) + @name = name + @parameters = parameters + @rhs = rhs + @tag = tag + @is_inline = is_inline + @required_parameters_count = parameters.count + end + + # @rbs () -> String + def to_s + "#{@name}(#{@parameters.map(&:s_value).join(', ')})" + end + + # @rbs () -> bool + def inline? + @is_inline + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/percent_code.rb b/tool/lrama/lib/lrama/grammar/percent_code.rb new file mode 100644 index 0000000000..9afb903056 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/percent_code.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class PercentCode + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @name: String + # @code: String + + attr_reader :name #: String + attr_reader :code #: String + + # @rbs (String name, String code) -> void + def initialize(name, code) + @name = name + @code = code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/precedence.rb b/tool/lrama/lib/lrama/grammar/precedence.rb new file mode 100644 index 0000000000..b4c6403372 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/precedence.rb @@ -0,0 +1,55 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Precedence < Struct.new(:type, :symbol, :precedence, :s_value, :lineno, keyword_init: true) + include Comparable + # @rbs! + # type type_enum = :left | :right | :nonassoc | :precedence + # + # attr_accessor type: type_enum + # attr_accessor symbol: Grammar::Symbol + # attr_accessor precedence: Integer + # attr_accessor s_value: String + # attr_accessor lineno: Integer + # + # def initialize: (?type: type_enum, ?symbol: Grammar::Symbol, ?precedence: Integer, ?s_value: ::String, ?lineno: Integer) -> void + + attr_reader :used_by_lalr #: Array[State::ResolvedConflict] + attr_reader :used_by_ielr #: Array[State::ResolvedConflict] + + # @rbs (Precedence other) -> Integer + def <=>(other) + self.precedence <=> other.precedence + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_lalr(resolved_conflict) + @used_by_lalr ||= [] #: Array[State::ResolvedConflict] + @used_by_lalr << resolved_conflict + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_ielr(resolved_conflict) + @used_by_ielr ||= [] #: Array[State::ResolvedConflict] + @used_by_ielr << resolved_conflict + end + + # @rbs () -> bool + def used_by? + used_by_lalr? || used_by_ielr? + end + + # @rbs () -> bool + def used_by_lalr? + !@used_by_lalr.nil? && !@used_by_lalr.empty? + end + + # @rbs () -> bool + def used_by_ielr? + !@used_by_ielr.nil? && !@used_by_ielr.empty? + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/printer.rb b/tool/lrama/lib/lrama/grammar/printer.rb new file mode 100644 index 0000000000..490fe701db --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/printer.rb @@ -0,0 +1,20 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Printer < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + # @rbs! + # attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag] + # attr_accessor token_code: Lexer::Token::UserCode + # attr_accessor lineno: Integer + # + # def initialize: (?ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], ?token_code: Lexer::Token::UserCode, ?lineno: Integer) -> void + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::PrinterCode.new(type: :printer, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/reference.rb b/tool/lrama/lib/lrama/grammar/reference.rb new file mode 100644 index 0000000000..7e3badfecc --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/reference.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # type: :dollar or :at + # name: String (e.g. $$, $foo, $expr.right) + # number: Integer (e.g. $1) + # index: Integer + # ex_tag: "$<tag>1" (Optional) + class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + # @rbs! + # attr_accessor type: ::Symbol + # attr_accessor name: String + # attr_accessor number: Integer + # attr_accessor index: Integer + # attr_accessor ex_tag: Lexer::Token::Base? + # attr_accessor first_column: Integer + # attr_accessor last_column: Integer + # + # def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token::Base?, first_column: Integer, last_column: Integer) -> void + + # @rbs () -> (String|Integer) + def value + name || number + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule.rb b/tool/lrama/lib/lrama/grammar/rule.rb new file mode 100644 index 0000000000..d00d6a8883 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule.rb @@ -0,0 +1,135 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # _rhs holds original RHS element. Use rhs to refer to Symbol. + class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true) + # @rbs! + # + # interface _DelegatedMethods + # def lhs: -> Grammar::Symbol + # def rhs: -> Array[Grammar::Symbol] + # end + # + # attr_accessor id: Integer + # attr_accessor _lhs: Lexer::Token::Base + # attr_accessor lhs: Grammar::Symbol + # attr_accessor lhs_tag: Lexer::Token::Tag? + # attr_accessor _rhs: Array[Lexer::Token::Base] + # attr_accessor rhs: Array[Grammar::Symbol] + # attr_accessor token_code: Lexer::Token::UserCode? + # attr_accessor position_in_original_rule_rhs: Integer + # attr_accessor nullable: bool + # attr_accessor precedence_sym: Grammar::Symbol? + # attr_accessor lineno: Integer? + # + # def initialize: ( + # ?id: Integer, ?_lhs: Lexer::Token::Base?, ?lhs: Lexer::Token::Base, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token::Base], ?rhs: Array[Grammar::Symbol], + # ?token_code: Lexer::Token::UserCode?, ?position_in_original_rule_rhs: Integer?, ?nullable: bool, + # ?precedence_sym: Grammar::Symbol?, ?lineno: Integer? + # ) -> void + + attr_accessor :original_rule #: Rule + + # @rbs (Rule other) -> bool + def ==(other) + self.class == other.class && + self.lhs == other.lhs && + self.lhs_tag == other.lhs_tag && + self.rhs == other.rhs && + self.token_code == other.token_code && + self.position_in_original_rule_rhs == other.position_in_original_rule_rhs && + self.nullable == other.nullable && + self.precedence_sym == other.precedence_sym && + self.lineno == other.lineno + end + + # @rbs () -> String + def display_name + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map {|r| r.id.s_value }.join(" ") + "#{l} -> #{r}" + end + + # @rbs () -> String + def display_name_without_action + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map do |r| + r.id.s_value if r.first_set.any? + end.compact.join(" ") + + "#{l} -> #{r}" + end + + # @rbs () -> (RailroadDiagrams::Skip | RailroadDiagrams::Sequence) + def to_diagrams + if rhs.empty? + RailroadDiagrams::Skip.new + else + RailroadDiagrams::Sequence.new(*rhs_to_diagram) + end + end + + # Used by #user_actions + # + # @rbs () -> String + def as_comment + l = lhs.id.s_value + r = empty_rule? ? "%empty" : rhs.map(&:display_name).join(" ") + + "#{l}: #{r}" + end + + # @rbs () -> String + def with_actions + "#{display_name} {#{token_code&.s_value}}" + end + + # opt_nl: ε <-- empty_rule + # | '\n' <-- not empty_rule + # + # @rbs () -> bool + def empty_rule? + rhs.empty? + end + + # @rbs () -> Precedence? + def precedence + precedence_sym&.precedence + end + + # @rbs () -> bool + def initial_rule? + id == 0 + end + + # @rbs () -> String? + def translated_code + return nil unless token_code + + Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self).translated_code + end + + # @rbs () -> bool + def contains_at_reference? + return false unless token_code + + token_code.references.any? {|r| r.type == :at } + end + + private + + # @rbs () -> Array[(RailroadDiagrams::Terminal | RailroadDiagrams::NonTerminal)] + def rhs_to_diagram + rhs.map do |r| + if r.term + RailroadDiagrams::Terminal.new(r.id.s_value) + else + RailroadDiagrams::NonTerminal.new(r.id.s_value) + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb new file mode 100644 index 0000000000..34fdca6c86 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -0,0 +1,270 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class RuleBuilder + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @position_in_original_rule_rhs: Integer? + # @skip_preprocess_references: bool + # @rules: Array[Rule] + # @rule_builders_for_parameterized: Array[RuleBuilder] + # @rule_builders_for_derived_rules: Array[RuleBuilder] + # @parameterized_rules: Array[Rule] + # @midrule_action_rules: Array[Rule] + # @replaced_rhs: Array[Lexer::Token::Base]? + + attr_accessor :lhs #: Lexer::Token::Base? + attr_accessor :line #: Integer? + attr_reader :rule_counter #: Counter + attr_reader :midrule_action_counter #: Counter + attr_reader :parameterized_resolver #: Grammar::Parameterized::Resolver + attr_reader :lhs_tag #: Lexer::Token::Tag? + attr_reader :rhs #: Array[Lexer::Token::Base] + attr_reader :user_code #: Lexer::Token::UserCode? + attr_reader :precedence_sym #: Grammar::Symbol? + + # @rbs (Counter rule_counter, Counter midrule_action_counter, Grammar::Parameterized::Resolver parameterized_resolver, ?Integer position_in_original_rule_rhs, ?lhs_tag: Lexer::Token::Tag?, ?skip_preprocess_references: bool) -> void + def initialize(rule_counter, midrule_action_counter, parameterized_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) + @rule_counter = rule_counter + @midrule_action_counter = midrule_action_counter + @parameterized_resolver = parameterized_resolver + @position_in_original_rule_rhs = position_in_original_rule_rhs + @skip_preprocess_references = skip_preprocess_references + + @lhs = nil + @lhs_tag = lhs_tag + @rhs = [] + @user_code = nil + @precedence_sym = nil + @line = nil + @rules = [] + @rule_builders_for_parameterized = [] + @rule_builders_for_derived_rules = [] + @parameterized_rules = [] + @midrule_action_rules = [] + end + + # @rbs (Lexer::Token::Base rhs) -> void + def add_rhs(rhs) + @line ||= rhs.line + + flush_user_code + + @rhs << rhs + end + + # @rbs (Lexer::Token::UserCode? user_code) -> void + def user_code=(user_code) + @line ||= user_code&.line + + flush_user_code + + @user_code = user_code + end + + # @rbs (Grammar::Symbol? precedence_sym) -> void + def precedence_sym=(precedence_sym) + flush_user_code + + @precedence_sym = precedence_sym + end + + # @rbs () -> void + def complete_input + freeze_rhs + end + + # @rbs () -> void + def setup_rules + preprocess_references unless @skip_preprocess_references + process_rhs + resolve_inline_rules + build_rules + end + + # @rbs () -> Array[Grammar::Rule] + def rules + @parameterized_rules + @midrule_action_rules + @rules + end + + # @rbs () -> bool + def has_inline_rules? + rhs.any? { |token| @parameterized_resolver.find_inline(token) } + end + + private + + # @rbs () -> void + def freeze_rhs + @rhs.freeze + end + + # @rbs () -> void + def preprocess_references + numberize_references + end + + # @rbs () -> void + def build_rules + tokens = @replaced_rhs #: Array[Lexer::Token::Base] + return if tokens.any? { |t| @parameterized_resolver.find_inline(t) } + + rule = Rule.new( + id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code, + position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line + ) + @rules = [rule] + @parameterized_rules = @rule_builders_for_parameterized.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules.each do |r| + r.original_rule = rule + end + end + + # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on. + # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`. + # + # @rbs () -> void + def process_rhs + return if @replaced_rhs + + replaced_rhs = [] #: Array[Lexer::Token::Base] + + rhs.each_with_index do |token, i| + case token + when Lrama::Lexer::Token::Char + replaced_rhs << token + when Lrama::Lexer::Token::Ident + replaced_rhs << token + when Lrama::Lexer::Token::InstantiateRule + parameterized_rule = @parameterized_resolver.find_rule(token) + raise "Unexpected token. #{token}" unless parameterized_rule + + bindings = Binding.new(parameterized_rule.parameters, token.args) + lhs_s_value = bindings.concatenated_args_str(token) + if (created_lhs = @parameterized_resolver.created_lhs(lhs_s_value)) + replaced_rhs << created_lhs + else + lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) + replaced_rhs << lhs_token + @parameterized_resolver.created_lhs_list << lhs_token + parameterized_rule.rhs.each do |r| + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, lhs_tag: token.lhs_tag || parameterized_rule.tag) + rule_builder.lhs = lhs_token + r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } + rule_builder.line = line + rule_builder.precedence_sym = r.precedence_sym + rule_builder.user_code = r.resolve_user_code(bindings) + rule_builder.complete_input + rule_builder.setup_rules + @rule_builders_for_parameterized << rule_builder + end + end + when Lrama::Lexer::Token::UserCode + prefix = token.referred ? "@" : "$@" + tag = token.tag || lhs_tag + new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) + replaced_rhs << new_token + + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, i, lhs_tag: tag, skip_preprocess_references: true) + rule_builder.lhs = new_token + rule_builder.user_code = token + rule_builder.complete_input + rule_builder.setup_rules + + @rule_builders_for_derived_rules << rule_builder + when Lrama::Lexer::Token::Empty + # Noop + else + raise "Unexpected token. #{token}" + end + end + + @replaced_rhs = replaced_rhs + end + + # @rbs () -> void + def resolve_inline_rules + while @rule_builders_for_parameterized.any?(&:has_inline_rules?) do + @rule_builders_for_parameterized = @rule_builders_for_parameterized.flat_map do |rule_builder| + if rule_builder.has_inline_rules? + inlined_builders = Inline::Resolver.new(rule_builder).resolve + inlined_builders.each { |builder| builder.setup_rules } + inlined_builders + else + rule_builder + end + end + end + end + + # @rbs () -> void + def numberize_references + # Bison n'th component is 1-origin + (rhs + [user_code]).compact.each.with_index(1) do |token, i| + next unless token.is_a?(Lrama::Lexer::Token::UserCode) + + token.references.each do |ref| + ref_name = ref.name + + if ref_name + if ref_name == '$' + ref.name = '$' + else + candidates = ([lhs] + rhs).each_with_index.select do |token, _i| + # @type var token: Lexer::Token::Base + token.referred_by?(ref_name) + end + + if candidates.size >= 2 + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is duplicated.") + end + + unless (referring_symbol = candidates.first) + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is not found.") + end + + if referring_symbol[1] == 0 # Refers to LHS + ref.name = '$' + else + ref.number = referring_symbol[1] + end + end + end + + if ref.number + ref.index = ref.number + end + + # TODO: Need to check index of @ too? + next if ref.type == :at + + if ref.index + # TODO: Prohibit $0 even so Bison allows it? + # See: https://www.gnu.org/software/bison/manual/html_node/Actions.html + token.invalid_ref(ref, "Can not refer following component. #{ref.index} >= #{i}.") if ref.index >= i + rhs[ref.index - 1].referred = true + end + end + end + end + + # @rbs () -> void + def flush_user_code + if (c = @user_code) + @rhs << c + @user_code = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/stdlib.y b/tool/lrama/lib/lrama/grammar/stdlib.y new file mode 100644 index 0000000000..dd397c9e08 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/stdlib.y @@ -0,0 +1,142 @@ +/********************************************************************** + + stdlib.y + + This is lrama's standard library. It provides a number of + parameterized rule definitions, such as options and lists, + that should be useful in a number of situations. + +**********************************************************************/ + +%% + +// ------------------------------------------------------------------- +// Options + +/* + * program: option(X) + * + * => + * + * program: option_X + * option_X: %empty + * option_X: X + */ +%rule option(X) + : /* empty */ + | X + ; + + +/* + * program: ioption(X) + * + * => + * + * program: %empty + * program: X + */ +%rule %inline ioption(X) + : /* empty */ + | X + ; + +// ------------------------------------------------------------------- +// Sequences + +/* + * program: preceded(opening, X) + * + * => + * + * program: preceded_opening_X + * preceded_opening_X: opening X + */ +%rule preceded(opening, X) + : opening X { $$ = $2; } + ; + +/* + * program: terminated(X, closing) + * + * => + * + * program: terminated_X_closing + * terminated_X_closing: X closing + */ +%rule terminated(X, closing) + : X closing { $$ = $1; } + ; + +/* + * program: delimited(opening, X, closing) + * + * => + * + * program: delimited_opening_X_closing + * delimited_opening_X_closing: opening X closing + */ +%rule delimited(opening, X, closing) + : opening X closing { $$ = $2; } + ; + +// ------------------------------------------------------------------- +// Lists + +/* + * program: list(X) + * + * => + * + * program: list_X + * list_X: %empty + * list_X: list_X X + */ +%rule list(X) + : /* empty */ + | list(X) X + ; + +/* + * program: nonempty_list(X) + * + * => + * + * program: nonempty_list_X + * nonempty_list_X: X + * nonempty_list_X: nonempty_list_X X + */ +%rule nonempty_list(X) + : X + | nonempty_list(X) X + ; + +/* + * program: separated_nonempty_list(separator, X) + * + * => + * + * program: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separated_nonempty_list_separator_X separator X + */ +%rule separated_nonempty_list(separator, X) + : X + | separated_nonempty_list(separator, X) separator X + ; + +/* + * program: separated_list(separator, X) + * + * => + * + * program: separated_list_separator_X + * separated_list_separator_X: option_separated_nonempty_list_separator_X + * option_separated_nonempty_list_separator_X: %empty + * option_separated_nonempty_list_separator_X: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separator separated_nonempty_list_separator_X X + */ +%rule separated_list(separator, X) + : option(separated_nonempty_list(separator, X)) + ; diff --git a/tool/lrama/lib/lrama/grammar/symbol.rb b/tool/lrama/lib/lrama/grammar/symbol.rb new file mode 100644 index 0000000000..07aee0c0a2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbol.rb @@ -0,0 +1,149 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# Symbol is both of nterm and term +# `number` is both for nterm and term +# `token_id` is tokentype for term, internal sequence number for nterm +# + +module Lrama + class Grammar + class Symbol + attr_accessor :id #: Lexer::Token::Base + attr_accessor :alias_name #: String? + attr_reader :number #: Integer + attr_accessor :number_bitmap #: Bitmap::bitmap + attr_accessor :tag #: Lexer::Token::Tag? + attr_accessor :token_id #: Integer + attr_accessor :nullable #: bool + attr_accessor :precedence #: Precedence? + attr_accessor :printer #: Printer? + attr_accessor :destructor #: Destructor? + attr_accessor :error_token #: ErrorToken + attr_accessor :first_set #: Set[Grammar::Symbol] + attr_accessor :first_set_bitmap #: Bitmap::bitmap + attr_reader :term #: bool + attr_writer :eof_symbol #: bool + attr_writer :error_symbol #: bool + attr_writer :undef_symbol #: bool + attr_writer :accept_symbol #: bool + + # @rbs (id: Lexer::Token::Base, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token::Tag?, + # ?token_id: Integer?, ?nullable: bool?, ?precedence: Precedence?, ?printer: Printer?) -> void + def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil) + @id = id + @alias_name = alias_name + @number = number + @tag = tag + @term = term + @token_id = token_id + @nullable = nullable + @precedence = precedence + @printer = printer + @destructor = destructor + end + + # @rbs (Integer) -> void + def number=(number) + @number = number + @number_bitmap = Bitmap::from_integer(number) + end + + # @rbs () -> bool + def term? + term + end + + # @rbs () -> bool + def nterm? + !term + end + + # @rbs () -> bool + def eof_symbol? + !!@eof_symbol + end + + # @rbs () -> bool + def error_symbol? + !!@error_symbol + end + + # @rbs () -> bool + def undef_symbol? + !!@undef_symbol + end + + # @rbs () -> bool + def accept_symbol? + !!@accept_symbol + end + + # @rbs () -> bool + def midrule? + return false if term? + + name.include?("$") || name.include?("@") + end + + # @rbs () -> String + def name + id.s_value + end + + # @rbs () -> String + def display_name + alias_name || name + end + + # name for yysymbol_kind_t + # + # See: b4_symbol_kind_base + # @type var name: String + # @rbs () -> String + def enum_name + case + when accept_symbol? + res = "YYACCEPT" + when eof_symbol? + res = "YYEOF" + when term? && id.is_a?(Lrama::Lexer::Token::Char) + res = number.to_s + display_name + when term? && id.is_a?(Lrama::Lexer::Token::Ident) + res = name + when midrule? + res = number.to_s + name + when nterm? + res = name + else + raise "Unexpected #{self}" + end + + "YYSYMBOL_" + res.gsub(/\W+/, "_") + end + + # comment for yysymbol_kind_t + # + # @rbs () -> String? + def comment + case + when accept_symbol? + # YYSYMBOL_YYACCEPT + name + when eof_symbol? + # YYEOF + alias_name + when (term? && 0 < token_id && token_id < 128) + # YYSYMBOL_3_backslash_, YYSYMBOL_14_ + display_name + when midrule? + # YYSYMBOL_21_1 + name + else + # YYSYMBOL_keyword_class, YYSYMBOL_strings_1 + display_name + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/symbols.rb b/tool/lrama/lib/lrama/grammar/symbols.rb new file mode 100644 index 0000000000..337241d1b2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative "symbols/resolver" diff --git a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb new file mode 100644 index 0000000000..085a835d28 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb @@ -0,0 +1,362 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Symbols + class Resolver + # @rbs! + # + # interface _DelegatedMethods + # def symbols: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def add_nterm: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + # def add_term: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + # def find_symbol_by_number!: (Integer number) -> Grammar::Symbol + # def find_symbol_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol + # def token_to_symbol: (Lexer::Token::Base token) -> Grammar::Symbol + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def fill_nterm_type: (Array[Grammar::Type] types) -> void + # def fill_symbol_number: () -> void + # def fill_printer: (Array[Grammar::Printer] printers) -> void + # def fill_destructor: (Array[Destructor] destructors) -> (Destructor | bot) + # def fill_error_token: (Array[Grammar::ErrorToken] error_tokens) -> void + # def sort_by_number!: () -> Array[Grammar::Symbol] + # end + # + # @symbols: Array[Grammar::Symbol]? + # @number: Integer + # @used_numbers: Hash[Integer, bool] + + attr_reader :terms #: Array[Grammar::Symbol] + attr_reader :nterms #: Array[Grammar::Symbol] + + # @rbs () -> void + def initialize + @terms = [] + @nterms = [] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols + @symbols ||= (@terms + @nterms) + end + + # @rbs () -> Array[Grammar::Symbol] + def sort_by_number! + symbols.sort_by!(&:number) + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) + if token_id && (sym = find_symbol_by_token_id(token_id)) + if replace + sym.id = id + sym.alias_name = alias_name + sym.tag = tag + end + + return sym + end + + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + term = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: true, token_id: token_id, nullable: false + ) + @terms << term + term + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + def add_nterm(id:, alias_name: nil, tag: nil) + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + nterm = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: false, token_id: nil, nullable: nil, + ) + @nterms << nterm + nterm + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_term_by_s_value(s_value) + terms.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_symbol_by_s_value(s_value) + symbols.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol + def find_symbol_by_s_value!(s_value) + find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`") + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol? + def find_symbol_by_id(id) + symbols.find do |s| + s.id == id || s.alias_name == id.s_value + end + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_symbol_by_id!(id) + find_symbol_by_id(id) || (raise "Symbol not found. #{id}") + end + + # @rbs (Integer token_id) -> Grammar::Symbol? + def find_symbol_by_token_id(token_id) + symbols.find {|s| s.token_id == token_id } + end + + # @rbs (Integer number) -> Grammar::Symbol + def find_symbol_by_number!(number) + sym = symbols[number] + + raise "Symbol not found. number: `#{number}`" unless sym + raise "[BUG] Symbol number mismatch. #{number}, #{sym}" if sym.number != number + + sym + end + + # @rbs () -> void + def fill_symbol_number + # YYEMPTY = -2 + # YYEOF = 0 + # YYerror = 1 + # YYUNDEF = 2 + @number = 3 + fill_terms_number + fill_nterms_number + end + + # @rbs (Array[Grammar::Type] types) -> void + def fill_nterm_type(types) + types.each do |type| + nterm = find_nterm_by_id!(type.id) + nterm.tag = type.tag + end + end + + # @rbs (Array[Grammar::Printer] printers) -> void + def fill_printer(printers) + symbols.each do |sym| + printers.each do |printer| + printer.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.printer = printer if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.printer = printer if sym.tag == ident_or_tag + else + raise "Unknown token type. #{printer}" + end + end + end + end + end + + # @rbs (Array[Destructor] destructors) -> (Array[Grammar::Symbol] | bot) + def fill_destructor(destructors) + symbols.each do |sym| + destructors.each do |destructor| + destructor.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.destructor = destructor if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.destructor = destructor if sym.tag == ident_or_tag + else + raise "Unknown token type. #{destructor}" + end + end + end + end + end + + # @rbs (Array[Grammar::ErrorToken] error_tokens) -> void + def fill_error_token(error_tokens) + symbols.each do |sym| + error_tokens.each do |token| + token.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.error_token = token if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.error_token = token if sym.tag == ident_or_tag + else + raise "Unknown token type. #{token}" + end + end + end + end + end + + # @rbs (Lexer::Token::Base token) -> Grammar::Symbol + def token_to_symbol(token) + case token + when Lrama::Lexer::Token::Base + find_symbol_by_id!(token) + else + raise "Unknown class: #{token}" + end + end + + # @rbs () -> void + def validate! + validate_number_uniqueness! + validate_alias_name_uniqueness! + validate_symbols! + end + + private + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_nterm_by_id!(id) + @nterms.find do |s| + s.id == id + end || (raise "Symbol not found. #{id}") + end + + # @rbs () -> void + def fill_terms_number + # Character literal in grammar file has + # token id corresponding to ASCII code by default, + # so start token_id from 256. + token_id = 256 + + @terms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + # If id is Token::Char, it uses ASCII code + if sym.token_id.nil? + if sym.id.is_a?(Lrama::Lexer::Token::Char) + # Ignore ' on the both sides + case sym.id.s_value[1..-2] + when "\\b" + sym.token_id = 8 + when "\\f" + sym.token_id = 12 + when "\\n" + sym.token_id = 10 + when "\\r" + sym.token_id = 13 + when "\\t" + sym.token_id = 9 + when "\\v" + sym.token_id = 11 + when "\"" + sym.token_id = 34 + when "'" + sym.token_id = 39 + when "\\\\" + sym.token_id = 92 + when /\A\\(\d+)\z/ + unless (id = Integer($1, 8)).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + when /\A(.)\z/ + unless (id = $1&.bytes&.first).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + else + raise "Unknown Char s_value #{sym}" + end + else + sym.token_id = token_id + token_id += 1 + end + end + end + end + + # @rbs () -> void + def fill_nterms_number + token_id = 0 + + @nterms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + if sym.token_id.nil? + sym.token_id = token_id + token_id += 1 + end + end + end + + # @rbs () -> Hash[Integer, bool] + def used_numbers + return @used_numbers if defined?(@used_numbers) + + @used_numbers = {} + symbols.map(&:number).each do |n| + @used_numbers[n] = true + end + @used_numbers + end + + # @rbs () -> void + def validate_number_uniqueness! + invalid = symbols.group_by(&:number).select do |number, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol number is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_alias_name_uniqueness! + invalid = symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol alias name is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_symbols! + symbols.each { |sym| sym.id.validate } + errors = symbols.map { |sym| sym.id.errors }.flatten.compact + return if errors.empty? + + raise errors.join("\n") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/type.rb b/tool/lrama/lib/lrama/grammar/type.rb new file mode 100644 index 0000000000..c631769447 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/type.rb @@ -0,0 +1,32 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Type + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @id: Lexer::Token::Base + # @tag: Lexer::Token::Tag + + attr_reader :id #: Lexer::Token::Base + attr_reader :tag #: Lexer::Token::Tag + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> void + def initialize(id:, tag:) + @id = id + @tag = tag + end + + # @rbs (Grammar::Type other) -> bool + def ==(other) + self.class == other.class && + self.id == other.id && + self.tag == other.tag + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/union.rb b/tool/lrama/lib/lrama/grammar/union.rb new file mode 100644 index 0000000000..774cc66fc6 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/union.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Union + attr_reader :code #: Grammar::Code::NoReferenceCode + attr_reader :lineno #: Integer + + # @rbs (code: Grammar::Code::NoReferenceCode, lineno: Integer) -> void + def initialize(code:, lineno:) + @code = code + @lineno = lineno + end + + # @rbs () -> String + def braces_less_code + # Braces is already removed by lexer + code.s_value + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb new file mode 100644 index 0000000000..ce98b505a7 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer.rb @@ -0,0 +1,219 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "strscan" + +require_relative "lexer/grammar_file" +require_relative "lexer/location" +require_relative "lexer/token" + +module Lrama + class Lexer + # @rbs! + # + # type token = lexer_token | c_token + # + # type lexer_token = [String, Token::Token] | + # [::Symbol, Token::Tag] | + # [::Symbol, Token::Char] | + # [::Symbol, Token::Str] | + # [::Symbol, Token::Int] | + # [::Symbol, Token::Ident] + # + # type c_token = [:C_DECLARATION, Token::UserCode] + + attr_reader :head_line #: Integer + attr_reader :head_column #: Integer + attr_reader :line #: Integer + attr_accessor :status #: :initial | :c_declaration + attr_accessor :end_symbol #: String? + + SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze #: Array[String] + PERCENT_TOKENS = %w( + %union + %token + %type + %nterm + %left + %right + %nonassoc + %expect + %define + %require + %printer + %destructor + %lex-param + %parse-param + %initial-action + %precedence + %prec + %error-token + %before-reduce + %after-reduce + %after-shift-error-token + %after-shift + %after-pop-stack + %empty + %code + %rule + %no-stdlib + %inline + %locations + %categories + %start + ).freeze #: Array[String] + + # @rbs (GrammarFile grammar_file) -> void + def initialize(grammar_file) + @grammar_file = grammar_file + @scanner = StringScanner.new(grammar_file.text) + @head_column = @head = @scanner.pos + @head_line = @line = 1 + @status = :initial + @end_symbol = nil + end + + # @rbs () -> token? + def next_token + case @status + when :initial + lex_token + when :c_declaration + lex_c_code + end + end + + # @rbs () -> Integer + def column + @scanner.pos - @head + end + + # @rbs () -> Location + def location + Location.new( + grammar_file: @grammar_file, + first_line: @head_line, first_column: @head_column, + last_line: line, last_column: column + ) + end + + # @rbs () -> lexer_token? + def lex_token + until @scanner.eos? do + case + when @scanner.scan(/\n/) + newline + when @scanner.scan(/\s+/) + @scanner.matched.count("\n").times { newline } + when @scanner.scan(/\/\*/) + lex_comment + when @scanner.scan(/\/\/.*(?<newline>\n)?/) + newline if @scanner[:newline] + else + break + end + end + + reset_first_position + + case + when @scanner.eos? + return + when @scanner.scan(/#{SYMBOLS.join('|')}/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/[\?\+\*]/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/<\w+>/) + return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'.'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/".*?"/) + return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] + when @scanner.scan(/\d+/) + return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)] + when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) + token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location) + type = + if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/) + :IDENT_COLON + else + :IDENTIFIER + end + return [type, token] + else + raise ParseError, location.generate_error_message("Unexpected token") # steep:ignore UnknownConstant + end + end + + # @rbs () -> c_token + def lex_c_code + nested = 0 + code = +'' + reset_first_position + + until @scanner.eos? do + case + when @scanner.scan(/{/) + code << @scanner.matched + nested += 1 + when @scanner.scan(/}/) + if nested == 0 && @end_symbol == '}' + @scanner.unscan + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + else + code << @scanner.matched + nested -= 1 + end + when @scanner.check(/#{@end_symbol}/) + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + when @scanner.scan(/\n/) + code << @scanner.matched + newline + when @scanner.scan(/".*?"/) + code << %Q(#{@scanner.matched}) + @line += @scanner.matched.count("\n") + when @scanner.scan(/'.*?'/) + code << %Q(#{@scanner.matched}) + when @scanner.scan(/[^\"'\{\}\n]+/) + code << @scanner.matched + when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) # steep:ignore + code << @scanner.matched + else + code << @scanner.getch + end + end + raise ParseError, location.generate_error_message("Unexpected code: #{code}") # steep:ignore UnknownConstant + end + + private + + # @rbs () -> void + def lex_comment + until @scanner.eos? do + case + when @scanner.scan_until(/[\s\S]*?\*\//) + @scanner.matched.count("\n").times { newline } + return + when @scanner.scan_until(/\n/) + newline + end + end + end + + # @rbs () -> void + def reset_first_position + @head_line = line + @head_column = column + end + + # @rbs () -> void + def newline + @line += 1 + @head = @scanner.pos + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/grammar_file.rb b/tool/lrama/lib/lrama/lexer/grammar_file.rb new file mode 100644 index 0000000000..37e82ff18d --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/grammar_file.rb @@ -0,0 +1,40 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + class GrammarFile + class Text < String + # @rbs () -> String + def inspect + length <= 50 ? super : "#{self[0..47]}...".inspect + end + end + + attr_reader :path #: String + attr_reader :text #: String + + # @rbs (String path, String text) -> void + def initialize(path, text) + @path = path + @text = Text.new(text).freeze + end + + # @rbs () -> String + def inspect + "<#{self.class}: @path=#{path}, @text=#{text.inspect}>" + end + + # @rbs (GrammarFile other) -> bool + def ==(other) + self.class == other.class && + self.path == other.path + end + + # @rbs () -> Array[String] + def lines + @lines ||= text.split("\n") + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/location.rb b/tool/lrama/lib/lrama/lexer/location.rb new file mode 100644 index 0000000000..4465576d53 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/location.rb @@ -0,0 +1,132 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + class Location + attr_reader :grammar_file #: GrammarFile + attr_reader :first_line #: Integer + attr_reader :first_column #: Integer + attr_reader :last_line #: Integer + attr_reader :last_column #: Integer + + # @rbs (grammar_file: GrammarFile, first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void + def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:) + @grammar_file = grammar_file + @first_line = first_line + @first_column = first_column + @last_line = last_line + @last_column = last_column + end + + # @rbs (Location other) -> bool + def ==(other) + self.class == other.class && + self.grammar_file == other.grammar_file && + self.first_line == other.first_line && + self.first_column == other.first_column && + self.last_line == other.last_line && + self.last_column == other.last_column + end + + # @rbs (Integer left, Integer right) -> Location + def partial_location(left, right) + offset = -first_column + new_first_line = -1 + new_first_column = -1 + new_last_line = -1 + new_last_column = -1 + + _text.each.with_index do |line, index| + new_offset = offset + line.length + 1 + + if offset <= left && left <= new_offset + new_first_line = first_line + index + new_first_column = left - offset + end + + if offset <= right && right <= new_offset + new_last_line = first_line + index + new_last_column = right - offset + end + + offset = new_offset + end + + Location.new( + grammar_file: grammar_file, + first_line: new_first_line, first_column: new_first_column, + last_line: new_last_line, last_column: new_last_column + ) + end + + # @rbs () -> String + def to_s + "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})" + end + + # @rbs (String error_message) -> String + def generate_error_message(error_message) + <<~ERROR.chomp + #{path}:#{first_line}:#{first_column}: #{error_message} + #{error_with_carets} + ERROR + end + + # @rbs () -> String + def error_with_carets + <<~TEXT + #{formatted_first_lineno} | #{text} + #{line_number_padding} | #{carets_line} + TEXT + end + + private + + # @rbs () -> String + def path + grammar_file.path + end + + # @rbs () -> String + def carets_line + leading_whitespace + highlight_marker + end + + # @rbs () -> String + def leading_whitespace + (text[0...first_column] or raise "Invalid first_column: #{first_column}") + .gsub(/[^\t]/, ' ') + end + + # @rbs () -> String + def highlight_marker + length = last_column - first_column + '^' + '~' * [0, length - 1].max + end + + # @rbs () -> String + def formatted_first_lineno + first_line.to_s.rjust(4) + end + + # @rbs () -> String + def line_number_padding + ' ' * formatted_first_lineno.length + end + + # @rbs () -> String + def text + @text ||= _text.join("\n") + end + + # @rbs () -> Array[String] + def _text + @_text ||=begin + range = (first_line - 1)...last_line + grammar_file.lines[range] or raise "#{range} is invalid" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb new file mode 100644 index 0000000000..37f77aa069 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -0,0 +1,20 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'token/base' +require_relative 'token/char' +require_relative 'token/empty' +require_relative 'token/ident' +require_relative 'token/instantiate_rule' +require_relative 'token/int' +require_relative 'token/str' +require_relative 'token/tag' +require_relative 'token/token' +require_relative 'token/user_code' + +module Lrama + class Lexer + module Token + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/base.rb b/tool/lrama/lib/lrama/lexer/token/base.rb new file mode 100644 index 0000000000..3df93bbc73 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/base.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Base + attr_reader :s_value #: String + attr_reader :location #: Location + attr_accessor :alias_name #: String + attr_accessor :referred #: bool + attr_reader :errors #: Array[String] + + # @rbs (s_value: String, ?alias_name: String, ?location: Location) -> void + def initialize(s_value:, alias_name: nil, location: nil) + s_value.freeze + @s_value = s_value + @alias_name = alias_name + @location = location + @errors = [] + end + + # @rbs () -> String + def to_s + "value: `#{s_value}`, location: #{location}" + end + + # @rbs (String string) -> bool + def referred_by?(string) + [self.s_value, self.alias_name].compact.include?(string) + end + + # @rbs (Lexer::Token::Base other) -> bool + def ==(other) + self.class == other.class && self.s_value == other.s_value + end + + # @rbs () -> Integer + def first_line + location.first_line + end + alias :line :first_line + + # @rbs () -> Integer + def first_column + location.first_column + end + alias :column :first_column + + # @rbs () -> Integer + def last_line + location.last_line + end + + # @rbs () -> Integer + def last_column + location.last_column + end + + # @rbs (Lrama::Grammar::Reference ref, String message) -> bot + def invalid_ref(ref, message) + location = self.location.partial_location(ref.first_column, ref.last_column) + raise location.generate_error_message(message) + end + + # @rbs () -> bool + def validate + true + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/char.rb b/tool/lrama/lib/lrama/lexer/token/char.rb new file mode 100644 index 0000000000..f4ef7c9fbc --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/char.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Char < Base + # @rbs () -> void + def validate + validate_ascii_code_range + end + + private + + # @rbs () -> void + def validate_ascii_code_range + unless s_value.ascii_only? + errors << "Invalid character: `#{s_value}`. Only ASCII characters are allowed." + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/empty.rb b/tool/lrama/lib/lrama/lexer/token/empty.rb new file mode 100644 index 0000000000..375e256493 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/empty.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Empty < Base + def initialize(location: nil) + super(s_value: '%empty', location: location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/ident.rb b/tool/lrama/lib/lrama/lexer/token/ident.rb new file mode 100644 index 0000000000..4880be9073 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/ident.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Ident < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb new file mode 100644 index 0000000000..7051ba75a4 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class InstantiateRule < Base + attr_reader :args #: Array[Lexer::Token::Base] + attr_reader :lhs_tag #: Lexer::Token::Tag? + + # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token::Base], ?lhs_tag: Lexer::Token::Tag?) -> void + def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil) + super s_value: s_value, alias_name: alias_name, location: location + @args = args + @lhs_tag = lhs_tag + end + + # @rbs () -> String + def rule_name + s_value + end + + # @rbs () -> Integer + def args_count + args.count + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/int.rb b/tool/lrama/lib/lrama/lexer/token/int.rb new file mode 100644 index 0000000000..7daf48d4d3 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/int.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Int < Base + # @rbs! + # def initialize: (s_value: Integer, ?alias_name: String, ?location: Location) -> void + # def s_value: () -> Integer + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/str.rb b/tool/lrama/lib/lrama/lexer/token/str.rb new file mode 100644 index 0000000000..cf9de6cf0f --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/str.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Str < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/tag.rb b/tool/lrama/lib/lrama/lexer/token/tag.rb new file mode 100644 index 0000000000..68c6268219 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/tag.rb @@ -0,0 +1,16 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Tag < Base + # @rbs () -> String + def member + # Omit "<>" + s_value[1..-2] or raise "Unexpected Tag format (#{s_value})" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/token.rb b/tool/lrama/lib/lrama/lexer/token/token.rb new file mode 100644 index 0000000000..935797efc6 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/token.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Token < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb new file mode 100644 index 0000000000..166f04954a --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -0,0 +1,109 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "strscan" + +module Lrama + class Lexer + module Token + class UserCode < Base + attr_accessor :tag #: Lexer::Token::Tag + + # @rbs () -> Array[Lrama::Grammar::Reference] + def references + @references ||= _references + end + + private + + # @rbs () -> Array[Lrama::Grammar::Reference] + def _references + scanner = StringScanner.new(s_value) + references = [] #: Array[Grammar::Reference] + + until scanner.eos? do + case + when reference = scan_reference(scanner) + references << reference + when scanner.scan(/\/\*/) + scanner.scan_until(/\*\//) + else + scanner.getch + end + end + + references + end + + # @rbs (StringScanner scanner) -> Lrama::Grammar::Reference? + def scan_reference(scanner) + start = scanner.pos + if scanner.scan(/ + # $ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + \$(<[a-zA-Z0-9_]+>)?(?: + (\$) # $$, $<long>$ + | (\d+) # $1, $2, $<long>1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $foo, $expr, $<long>program (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + ) + | + # @ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + @(?: + (\$) # @$ + | (\d+) # @1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # @foo, @expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # @[expr.right], @[expr-right] (named reference with brackets) + ) + | + # $: references + \$: + (?: + (\$) # $:$ + | (\d+) # $:1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $:foo, $:expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $:[expr.right], $:[expr-right] (named reference with brackets) + ) + /x) + case + # $ references + when scanner[2] # $$, $<long>$ + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[3] # $1, $2, $<long>1 + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[3]), index: Integer(scanner[3]), ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[4] # $foo, $expr, $<long>program (named reference without brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[4], ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[5] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[5], ex_tag: tag, first_column: start, last_column: scanner.pos) + + # @ references + when scanner[6] # @$ + return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) + when scanner[7] # @1 + return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[7]), index: Integer(scanner[7]), first_column: start, last_column: scanner.pos) + when scanner[8] # @foo, @expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[8], first_column: start, last_column: scanner.pos) + when scanner[9] # @[expr.right], @[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[9], first_column: start, last_column: scanner.pos) + + # $: references + when scanner[10] # $:$ + return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) + when scanner[11] # $:1 + return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[11]), index: Integer(scanner[11]), first_column: start, last_column: scanner.pos) + when scanner[12] # $:foo, $:expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[12], first_column: start, last_column: scanner.pos) + when scanner[13] # $:[expr.right], $:[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[13], first_column: start, last_column: scanner.pos) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/logger.rb b/tool/lrama/lib/lrama/logger.rb new file mode 100644 index 0000000000..291eea5296 --- /dev/null +++ b/tool/lrama/lib/lrama/logger.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Logger + # @rbs (IO out) -> void + def initialize(out = STDERR) + @out = out + end + + # @rbs () -> void + def line_break + @out << "\n" + end + + # @rbs (String message) -> void + def trace(message) + @out << message << "\n" + end + + # @rbs (String message) -> void + def warn(message) + @out << 'warning: ' << message << "\n" + end + + # @rbs (String message) -> void + def error(message) + @out << 'error: ' << message << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/option_parser.rb b/tool/lrama/lib/lrama/option_parser.rb new file mode 100644 index 0000000000..5a15d59c7b --- /dev/null +++ b/tool/lrama/lib/lrama/option_parser.rb @@ -0,0 +1,223 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require 'optparse' + +module Lrama + # Handle option parsing for the command line interface. + class OptionParser + # @rbs! + # @options: Lrama::Options + # @trace: Array[String] + # @report: Array[String] + # @profile: Array[String] + + # @rbs (Array[String]) -> Lrama::Options + def self.parse(argv) + new.parse(argv) + end + + # @rbs () -> void + def initialize + @options = Options.new + @trace = [] + @report = [] + @profile = [] + end + + # @rbs (Array[String]) -> Lrama::Options + def parse(argv) + parse_by_option_parser(argv) + + @options.trace_opts = validate_trace(@trace) + @options.report_opts = validate_report(@report) + @options.profile_opts = validate_profile(@profile) + @options.grammar_file = argv.shift + + unless @options.grammar_file + abort "File should be specified\n" + end + + if @options.grammar_file == '-' + @options.grammar_file = argv.shift or abort "File name for STDIN should be specified\n" + else + @options.y = File.open(@options.grammar_file, 'r') + end + + if !@report.empty? && @options.report_file.nil? && @options.grammar_file + @options.report_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".output" + end + + if !@options.header_file && @options.header + case + when @options.outfile + @options.header_file = File.dirname(@options.outfile) + "/" + File.basename(@options.outfile, ".*") + ".h" + when @options.grammar_file + @options.header_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".h" + end + end + + @options + end + + private + + # @rbs (Array[String]) -> void + def parse_by_option_parser(argv) + ::OptionParser.new do |o| + o.banner = <<~BANNER + Lrama is LALR (1) parser generator written by Ruby. + + Usage: lrama [options] FILE + BANNER + o.separator '' + o.separator 'STDIN mode:' + o.separator 'lrama [options] - FILE read grammar from STDIN' + o.separator '' + o.separator 'Tuning the Parser:' + o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v } + o.on('-t', '--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.separator " same as '-Dparse.trace'" + o.on('--locations', 'enable location support') {|v| @options.locations = true } + o.on('-D', '--define=NAME[=VALUE]', Array, "similar to '%define NAME VALUE'") do |v| + @options.define = v.each_with_object({}) do |item, hash| # steep:ignore UnannotatedEmptyCollection + key, value = item.split('=', 2) + hash[key] = value + end + end + o.separator '' + o.separator 'Output:' + o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v } + o.on('-d', 'also produce a header file') { @options.header = true } + o.on('-r', '--report=REPORTS', Array, 'also produce details on the automaton') {|v| @report = v } + o.on_tail '' + o.on_tail 'REPORTS is a list of comma-separated words that can include:' + o.on_tail ' states describe the states' + o.on_tail ' itemsets complete the core item sets with their closure' + o.on_tail ' lookaheads explicitly associate lookahead tokens to items' + o.on_tail ' solved describe shift/reduce conflicts solving' + o.on_tail ' counterexamples, cex generate conflict counterexamples' + o.on_tail ' rules list unused rules' + o.on_tail ' terms list unused terminals' + o.on_tail ' verbose report detailed internal state and analysis results' + o.on_tail ' all include all the above reports' + o.on_tail ' none disable all reports' + o.on('--report-file=FILE', 'also produce details on the automaton output to a file named FILE') {|v| @options.report_file = v } + o.on('-o', '--output=FILE', 'leave output to FILE') {|v| @options.outfile = v } + o.on('--trace=TRACES', Array, 'also output trace logs at runtime') {|v| @trace = v } + o.on_tail '' + o.on_tail 'TRACES is a list of comma-separated words that can include:' + o.on_tail ' automaton display states' + o.on_tail ' closure display states' + o.on_tail ' rules display grammar rules' + o.on_tail ' only-explicit-rules display only explicit grammar rules' + o.on_tail ' actions display grammar rules with actions' + o.on_tail ' time display generation time' + o.on_tail ' all include all the above traces' + o.on_tail ' none disable all traces' + o.on('--diagram=[FILE]', 'generate a diagram of the rules') do |v| + @options.diagram = true + @options.diagram_file = v if v + end + o.on('--profile=PROFILES', Array, 'profiles parser generation parts') {|v| @profile = v } + o.on_tail '' + o.on_tail 'PROFILES is a list of comma-separated words that can include:' + o.on_tail ' call-stack use sampling call-stack profiler (stackprof gem)' + o.on_tail ' memory use memory profiler (memory_profiler gem)' + o.on('-v', '--verbose', "same as '--report=state'") {|_v| @report << 'states' } + o.separator '' + o.separator 'Diagnostics:' + o.on('-W', '--warnings', 'report the warnings') {|v| @options.warnings = true } + o.separator '' + o.separator 'Error Recovery:' + o.on('-e', 'enable error recovery') {|v| @options.error_recovery = true } + o.separator '' + o.separator 'Other options:' + o.on('-V', '--version', "output version information and exit") {|v| puts "lrama #{Lrama::VERSION}"; exit 0 } + o.on('-h', '--help', "display this help and exit") {|v| puts o; exit 0 } + o.on_tail + o.parse!(argv) + end + end + + ALIASED_REPORTS = { cex: :counterexamples }.freeze #: Hash[Symbol, Symbol] + VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze #: Array[Symbol] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_report(report) + h = { grammar: true } + return h if report.empty? + return {} if report == ['none'] + if report == ['all'] + VALID_REPORTS.each { |r| h[r] = true } + return h + end + + report.each do |r| + aliased = aliased_report_option(r) + if VALID_REPORTS.include?(aliased) + h[aliased] = true + else + raise "Invalid report option \"#{r}\"." + end + end + + return h + end + + # @rbs (String) -> Symbol + def aliased_report_option(opt) + (ALIASED_REPORTS[opt.to_sym] || opt).to_sym + end + + VALID_TRACES = %w[ + locations scan parse automaton bitsets closure + grammar rules only-explicit-rules actions resource + sets muscles tools m4-early m4 skeleton time ielr cex + ].freeze #: Array[String] + NOT_SUPPORTED_TRACES = %w[ + locations scan parse bitsets grammar resource + sets muscles tools m4-early m4 skeleton ielr cex + ].freeze #: Array[String] + SUPPORTED_TRACES = VALID_TRACES - NOT_SUPPORTED_TRACES #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_trace(trace) + h = {} #: Hash[Symbol, bool] + return h if trace.empty? || trace == ['none'] + all_traces = SUPPORTED_TRACES - %w[only-explicit-rules] + if trace == ['all'] + all_traces.each { |t| h[t.gsub(/-/, '_').to_sym] = true } + return h + end + + trace.each do |t| + if SUPPORTED_TRACES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid trace option \"#{t}\".\nValid options are [#{SUPPORTED_TRACES.join(", ")}]." + end + end + + return h + end + + VALID_PROFILES = %w[call-stack memory].freeze #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_profile(profile) + h = {} #: Hash[Symbol, bool] + return h if profile.empty? + + profile.each do |t| + if VALID_PROFILES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid profile option \"#{t}\".\nValid options are [#{VALID_PROFILES.join(", ")}]." + end + end + + return h + end + end +end diff --git a/tool/lrama/lib/lrama/options.rb b/tool/lrama/lib/lrama/options.rb new file mode 100644 index 0000000000..87aec62448 --- /dev/null +++ b/tool/lrama/lib/lrama/options.rb @@ -0,0 +1,46 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Command line options. + class Options + attr_accessor :skeleton #: String + attr_accessor :locations #: bool + attr_accessor :header #: bool + attr_accessor :header_file #: String? + attr_accessor :report_file #: String? + attr_accessor :outfile #: String + attr_accessor :error_recovery #: bool + attr_accessor :grammar_file #: String + attr_accessor :trace_opts #: Hash[Symbol, bool]? + attr_accessor :report_opts #: Hash[Symbol, bool]? + attr_accessor :warnings #: bool + attr_accessor :y #: IO + attr_accessor :debug #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :diagram #: bool + attr_accessor :diagram_file #: String + attr_accessor :profile_opts #: Hash[Symbol, bool]? + + # @rbs () -> void + def initialize + @skeleton = "bison/yacc.c" + @locations = false + @define = {} + @header = false + @header_file = nil + @report_file = nil + @outfile = "y.tab.c" + @error_recovery = false + @grammar_file = '' + @trace_opts = nil + @report_opts = nil + @warnings = false + @y = STDIN + @debug = false + @diagram = false + @diagram_file = "diagram.html" + @profile_opts = nil + end + end +end diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb new file mode 100644 index 0000000000..d527be8bd4 --- /dev/null +++ b/tool/lrama/lib/lrama/output.rb @@ -0,0 +1,452 @@ +# frozen_string_literal: true + +require "forwardable" +require_relative "tracer/duration" + +module Lrama + class Output + extend Forwardable + include Tracer::Duration + + attr_reader :grammar_file_path, :context, :grammar, :error_recovery, :include_header + + def_delegators "@context", :yyfinal, :yylast, :yyntokens, :yynnts, :yynrules, :yynstates, + :yymaxutok, :yypact_ninf, :yytable_ninf + + def_delegators "@grammar", :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + + def initialize( + out:, output_file_path:, template_name:, grammar_file_path:, + context:, grammar:, header_out: nil, header_file_path: nil, error_recovery: false + ) + @out = out + @output_file_path = output_file_path + @template_name = template_name + @grammar_file_path = grammar_file_path + @header_out = header_out + @header_file_path = header_file_path + @context = context + @grammar = grammar + @error_recovery = error_recovery + @include_header = header_file_path ? header_file_path.sub("./", "") : nil + end + + if ERB.instance_method(:initialize).parameters.last.first == :key + def self.erb(input) + ERB.new(input, trim_mode: '-') + end + else + def self.erb(input) + ERB.new(input, nil, '-') + end + end + + def render_partial(file) + ERB.render(partial_file(file), context: @context, output: self) + end + + def render + report_duration(:render) do + tmp = eval_template(template_file, @output_file_path) + @out << tmp + + if @header_file_path + tmp = eval_template(header_template_file, @header_file_path) + + if @header_out + @header_out << tmp + else + File.write(@header_file_path, tmp) + end + end + end + end + + # A part of b4_token_enums + def token_enums + @context.yytokentype.map do |s_value, token_id, display_name| + s = sprintf("%s = %d%s", s_value, token_id, token_id == yymaxutok ? "" : ",") + + if display_name + sprintf(" %-30s /* %s */\n", s, display_name) + else + sprintf(" %s\n", s) + end + end.join + end + + # b4_symbol_enum + def symbol_enum + last_sym_number = @context.yysymbol_kind_t.last[1] + @context.yysymbol_kind_t.map do |s_value, sym_number, display_name| + s = sprintf("%s = %d%s", s_value, sym_number, (sym_number == last_sym_number) ? "" : ",") + + if display_name + sprintf(" %-40s /* %s */\n", s, display_name) + else + sprintf(" %s\n", s) + end + end.join + end + + def yytranslate + int_array_to_string(@context.yytranslate) + end + + def yytranslate_inverted + int_array_to_string(@context.yytranslate_inverted) + end + + def yyrline + int_array_to_string(@context.yyrline) + end + + def yytname + string_array_to_string(@context.yytname) + " YY_NULLPTR" + end + + # b4_int_type_for + def int_type_for(ary) + min = ary.min + max = ary.max + + case + when (-127 <= min && min <= 127) && (-127 <= max && max <= 127) + "yytype_int8" + when (0 <= min && min <= 255) && (0 <= max && max <= 255) + "yytype_uint8" + when (-32767 <= min && min <= 32767) && (-32767 <= max && max <= 32767) + "yytype_int16" + when (0 <= min && min <= 65535) && (0 <= max && max <= 65535) + "yytype_uint16" + else + "int" + end + end + + def symbol_actions_for_printer + @grammar.symbols.map do |sym| + next unless sym.printer + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.printer.lineno} "#{@grammar_file_path}" + {#{sym.printer.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + def symbol_actions_for_destructor + @grammar.symbols.map do |sym| + next unless sym.destructor + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.destructor.lineno} "#{@grammar_file_path}" + {#{sym.destructor.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + # b4_user_initial_action + def user_initial_action(comment = "") + return "" unless @grammar.initial_action + + <<-STR + #{comment} +#line #{@grammar.initial_action.line} "#{@grammar_file_path}" + {#{@grammar.initial_action.translated_code}} + STR + end + + def after_shift_function(comment = "") + return "" unless @grammar.after_shift + + <<-STR + #{comment} +#line #{@grammar.after_shift.line} "#{@grammar_file_path}" + {#{@grammar.after_shift.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def before_reduce_function(comment = "") + return "" unless @grammar.before_reduce + + <<-STR + #{comment} +#line #{@grammar.before_reduce.line} "#{@grammar_file_path}" + {#{@grammar.before_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_reduce_function(comment = "") + return "" unless @grammar.after_reduce + + <<-STR + #{comment} +#line #{@grammar.after_reduce.line} "#{@grammar_file_path}" + {#{@grammar.after_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_shift_error_token_function(comment = "") + return "" unless @grammar.after_shift_error_token + + <<-STR + #{comment} +#line #{@grammar.after_shift_error_token.line} "#{@grammar_file_path}" + {#{@grammar.after_shift_error_token.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def after_pop_stack_function(len, comment = "") + return "" unless @grammar.after_pop_stack + + <<-STR + #{comment} +#line #{@grammar.after_pop_stack.line} "#{@grammar_file_path}" + {#{@grammar.after_pop_stack.s_value}(#{len}#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def symbol_actions_for_error_token + @grammar.symbols.map do |sym| + next unless sym.error_token + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.error_token.lineno} "#{@grammar_file_path}" + {#{sym.error_token.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + # b4_user_actions + def user_actions + action = @context.states.rules.map do |rule| + next unless rule.token_code + + code = rule.token_code + spaces = " " * (code.column - 1) + + <<-STR + case #{rule.id + 1}: /* #{rule.as_comment} */ +#line #{code.line} "#{@grammar_file_path}" +#{spaces}{#{rule.translated_code}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + + action + <<-STR + +#line [@oline@] [@ofile@] + STR + end + + def omit_blanks(param) + param.strip + end + + # b4_parse_param + def parse_param + if @grammar.parse_param + omit_blanks(@grammar.parse_param) + else + "" + end + end + + def lex_param + if @grammar.lex_param + omit_blanks(@grammar.lex_param) + else + "" + end + end + + # b4_user_formals + def user_formals + if @grammar.parse_param + ", #{parse_param}" + else + "" + end + end + + # b4_user_args + def user_args + if @grammar.parse_param + ", #{parse_param_name}" + else + "" + end + end + + def extract_param_name(param) + param[/\b([a-zA-Z0-9_]+)(?=\s*\z)/] + end + + def parse_param_name + if @grammar.parse_param + extract_param_name(parse_param) + else + "" + end + end + + def lex_param_name + if @grammar.lex_param + extract_param_name(lex_param) + else + "" + end + end + + # b4_parse_param_use + def parse_param_use(val, loc) + str = <<-STR.dup + YY_USE (#{val}); + YY_USE (#{loc}); + STR + + if @grammar.parse_param + str << " YY_USE (#{parse_param_name});" + end + + str + end + + # b4_yylex_formals + def yylex_formals + ary = ["&yylval"] + ary << "&yylloc" if @grammar.locations + + if @grammar.lex_param + ary << lex_param_name + end + + "(#{ary.join(', ')})" + end + + # b4_table_value_equals + def table_value_equals(table, value, literal, symbol) + if literal < table.min || table.max < literal + "0" + else + "((#{value}) == #{symbol})" + end + end + + # b4_yyerror_args + def yyerror_args + ary = ["&yylloc"] + + if @grammar.parse_param + ary << parse_param_name + end + + "#{ary.join(', ')}" + end + + def template_basename + File.basename(template_file) + end + + def aux + @grammar.aux + end + + def int_array_to_string(ary) + last = ary.count - 1 + + ary.each_with_index.each_slice(10).map do |slice| + " " + slice.map { |e, i| sprintf("%6d%s", e, (i == last) ? "" : ",") }.join + end.join("\n") + end + + def spec_mapped_header_file + @header_file_path + end + + def b4_cpp_guard__b4_spec_mapped_header_file + if @header_file_path + "YY_YY_" + @header_file_path.gsub(/[^a-zA-Z_0-9]+/, "_").upcase + "_INCLUDED" + else + "" + end + end + + # b4_percent_code_get + def percent_code(name) + @grammar.percent_codes.select do |percent_code| + percent_code.name == name + end.map do |percent_code| + percent_code.code + end.join + end + + private + + def eval_template(file, path) + tmp = ERB.render(file, context: @context, output: self) + replace_special_variables(tmp, path) + end + + def template_file + File.join(template_dir, @template_name) + end + + def header_template_file + File.join(template_dir, "bison/yacc.h") + end + + def partial_file(file) + File.join(template_dir, file) + end + + def template_dir + File.expand_path('../../template', __dir__) + end + + def string_array_to_string(ary) + result = "" + tmp = " " + + ary.each do |s| + replaced = s.gsub('\\', '\\\\\\\\').gsub('"', '\\"') + if (tmp + replaced + " \"\",").length > 75 + result = "#{result}#{tmp}\n" + tmp = " \"#{replaced}\"," + else + tmp = "#{tmp} \"#{replaced}\"," + end + end + + result + tmp + end + + def replace_special_variables(str, ofile) + str.each_line.with_index(1).map do |line, i| + line.gsub!("[@oline@]", (i + 1).to_s) + line.gsub!("[@ofile@]", "\"#{ofile}\"") + line + end.join + end + end +end diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb new file mode 100644 index 0000000000..20c3ad347f --- /dev/null +++ b/tool/lrama/lib/lrama/parser.rb @@ -0,0 +1,2275 @@ +# frozen_string_literal: true +# +# DO NOT MODIFY!!!! +# This file is automatically generated by Racc 1.8.1 +# from Racc grammar file "parser.y". +# + +###### racc/parser.rb begin +unless $".find {|p| p.end_with?('/racc/parser.rb')} +$".push "#{__dir__}/racc/parser.rb" +self.class.module_eval(<<'...end racc/parser.rb/module_eval...', 'racc/parser.rb', 1) +#-- +# Copyright (c) 1999-2006 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the same terms of ruby. +# +# As a special exception, when this code is copied by Racc +# into a Racc output file, you may use that output file +# without restriction. +#++ + +unless $".find {|p| p.end_with?('/racc/info.rb')} +$".push "#{__dir__}/racc/info.rb" + +module Racc + VERSION = '1.8.1' + Version = VERSION + Copyright = 'Copyright (c) 1999-2006 Minero Aoki' +end + +end + + +module Racc + class ParseError < StandardError; end +end +unless defined?(::ParseError) + ParseError = Racc::ParseError # :nodoc: +end + +# Racc is an LALR(1) parser generator. +# It is written in Ruby itself, and generates Ruby programs. +# +# == Command-line Reference +# +# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] +# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] +# [-v] [--verbose] +# [-O<var>filename</var>] [--log-file=<var>filename</var>] +# [-g] [--debug] +# [-E] [--embedded] +# [-l] [--no-line-convert] +# [-c] [--line-convert-all] +# [-a] [--no-omit-actions] +# [-C] [--check-only] +# [-S] [--output-status] +# [--version] [--copyright] [--help] <var>grammarfile</var> +# +# [+grammarfile+] +# Racc grammar file. Any extension is permitted. +# [-o+outfile+, --output-file=+outfile+] +# A filename for output. default is <+filename+>.tab.rb +# [-O+filename+, --log-file=+filename+] +# Place logging output in file +filename+. +# Default log file name is <+filename+>.output. +# [-e+rubypath+, --executable=+rubypath+] +# output executable file(mode 755). where +path+ is the Ruby interpreter. +# [-v, --verbose] +# verbose mode. create +filename+.output file, like yacc's y.output file. +# [-g, --debug] +# add debug code to parser class. To display debugging information, +# use this '-g' option and set @yydebug true in parser class. +# [-E, --embedded] +# Output parser which doesn't need runtime files (racc/parser.rb). +# [-F, --frozen] +# Output parser which declares frozen_string_literals: true +# [-C, --check-only] +# Check syntax of racc grammar file and quit. +# [-S, --output-status] +# Print messages time to time while compiling. +# [-l, --no-line-convert] +# turns off line number converting. +# [-c, --line-convert-all] +# Convert line number of actions, inner, header and footer. +# [-a, --no-omit-actions] +# Call all actions, even if an action is empty. +# [--version] +# print Racc version and quit. +# [--copyright] +# Print copyright and quit. +# [--help] +# Print usage and quit. +# +# == Generating Parser Using Racc +# +# To compile Racc grammar file, simply type: +# +# $ racc parse.y +# +# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. +# +# == Writing A Racc Grammar File +# +# If you want your own parser, you have to write a grammar file. +# A grammar file contains the name of your parser class, grammar for the parser, +# user code, and anything else. +# When writing a grammar file, yacc's knowledge is helpful. +# If you have not used yacc before, Racc is not too difficult. +# +# Here's an example Racc grammar file. +# +# class Calcparser +# rule +# target: exp { print val[0] } +# +# exp: exp '+' exp +# | exp '*' exp +# | '(' exp ')' +# | NUMBER +# end +# +# Racc grammar files resemble yacc files. +# But (of course), this is Ruby code. +# yacc's $$ is the 'result', $0, $1... is +# an array called 'val', and $-1, $-2... is an array called '_values'. +# +# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for +# more information on grammar files. +# +# == Parser +# +# Then you must prepare the parse entry method. There are two types of +# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse +# +# Racc::Parser#do_parse is simple. +# +# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). +# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. +# EOF is [false, false]. +# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. +# If you want to change this, see the grammar reference. +# +# Racc::Parser#yyparse is little complicated, but useful. +# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. +# +# For example, <code>yyparse(obj, :scan)</code> causes +# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. +# +# == Debugging +# +# When debugging, "-v" or/and the "-g" option is helpful. +# +# "-v" creates verbose log file (.output). +# "-g" creates a "Verbose Parser". +# Verbose Parser prints the internal status when parsing. +# But it's _not_ automatic. +# You must use -g option and set +@yydebug+ to +true+ in order to get output. +# -g option only creates the verbose parser. +# +# === Racc reported syntax error. +# +# Isn't there too many "end"? +# grammar of racc file is changed in v0.10. +# +# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. +# +# === Racc reported "XXXX conflicts". +# +# Try "racc -v xxxx.y". +# It causes producing racc's internal log file, xxxx.output. +# +# === Generated parsers does not work correctly +# +# Try "racc -g xxxx.y". +# This command let racc generate "debugging parser". +# Then set @yydebug=true in your parser. +# It produces a working log of your parser. +# +# == Re-distributing Racc runtime +# +# A parser, which is created by Racc, requires the Racc runtime module; +# racc/parser.rb. +# +# Ruby 1.8.x comes with Racc runtime module, +# you need NOT distribute Racc runtime files. +# +# If you want to include the Racc runtime module with your parser. +# This can be done by using '-E' option: +# +# $ racc -E -omyparser.rb myparser.y +# +# This command creates myparser.rb which `includes' Racc runtime. +# Only you must do is to distribute your parser file (myparser.rb). +# +# Note: parser.rb is ruby license, but your parser is not. +# Your own parser is completely yours. +module Racc + + unless defined?(Racc_No_Extensions) + Racc_No_Extensions = false # :nodoc: + end + + class Parser + + Racc_Runtime_Version = ::Racc::VERSION + Racc_Runtime_Core_Version_R = ::Racc::VERSION + + begin + if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' + require 'jruby' + require 'racc/cparse-jruby.jar' + com.headius.racc.Cparse.new.load(JRuby.runtime, false) + else + require 'racc/cparse' + end + + unless new.respond_to?(:_racc_do_parse_c, true) + raise LoadError, 'old cparse.so' + end + if Racc_No_Extensions + raise LoadError, 'selecting ruby version of racc runtime core' + end + + Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: + Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: + Racc_Runtime_Type = 'c' # :nodoc: + rescue LoadError + Racc_Main_Parsing_Routine = :_racc_do_parse_rb + Racc_YY_Parse_Method = :_racc_yyparse_rb + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R + Racc_Runtime_Type = 'ruby' + end + + def Parser.racc_runtime_type # :nodoc: + Racc_Runtime_Type + end + + def _racc_setup + @yydebug = false unless self.class::Racc_debug_parser + @yydebug = false unless defined?(@yydebug) + if @yydebug + @racc_debug_out = $stderr unless defined?(@racc_debug_out) + @racc_debug_out ||= $stderr + end + arg = self.class::Racc_arg + arg[13] = true if arg.size < 14 + arg + end + + def _racc_init_sysvars + @racc_state = [0] + @racc_tstack = [] + @racc_vstack = [] + + @racc_t = nil + @racc_val = nil + + @racc_read_next = true + + @racc_user_yyerror = false + @racc_error_status = 0 + end + + # The entry point of the parser. This method is used with #next_token. + # If Racc wants to get token (and its value), calls next_token. + # + # Example: + # def parse + # @q = [[1,1], + # [2,2], + # [3,3], + # [false, '$']] + # do_parse + # end + # + # def next_token + # @q.shift + # end + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def do_parse + #{Racc_Main_Parsing_Routine}(_racc_setup(), false) + end + RUBY + + # The method to fetch next token. + # If you use #do_parse method, you must implement #next_token. + # + # The format of return value is [TOKEN_SYMBOL, VALUE]. + # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT + # for 'IDENT'. ";" (String) for ';'. + # + # The final symbol (End of file) must be false. + def next_token + raise NotImplementedError, "#{self.class}\#next_token is not defined" + end + + def _racc_do_parse_rb(arg, in_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + tok = act = i = nil + + catch(:racc_end_parse) { + while true + if i = action_pointer[@racc_state[-1]] + if @racc_read_next + if @racc_t != 0 # not EOF + tok, @racc_val = next_token() + unless tok # EOF + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + racc_read_token(@racc_t, tok, @racc_val) if @yydebug + @racc_read_next = false + end + end + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + else + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + } + end + + # Another entry point for the parser. + # If you use this method, you must implement RECEIVER#METHOD_ID method. + # + # RECEIVER#METHOD_ID is a method to get next token. + # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def yyparse(recv, mid) + #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) + end + RUBY + + def _racc_yyparse_rb(recv, mid, arg, c_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + + catch(:racc_end_parse) { + until i = action_pointer[@racc_state[-1]] + while act = _racc_evalact(action_default[@racc_state[-1]], arg) + ; + end + end + recv.__send__(mid) do |tok, val| + unless tok + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + @racc_val = val + @racc_read_next = false + + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + + while !(i = action_pointer[@racc_state[-1]]) || + ! @racc_read_next || + @racc_t == 0 # $ + unless i and i += @racc_t and + i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + end + } + end + + ### + ### common + ### + + def _racc_evalact(act, arg) + action_table, action_check, _, action_pointer, + _, _, _, _, + _, _, _, shift_n, + reduce_n, * = arg + nerr = 0 # tmp + + if act > 0 and act < shift_n + # + # shift + # + if @racc_error_status > 0 + @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF + end + @racc_vstack.push @racc_val + @racc_state.push act + @racc_read_next = true + if @yydebug + @racc_tstack.push @racc_t + racc_shift @racc_t, @racc_tstack, @racc_vstack + end + + elsif act < 0 and act > -reduce_n + # + # reduce + # + code = catch(:racc_jump) { + @racc_state.push _racc_do_reduce(arg, act) + false + } + if code + case code + when 1 # yyerror + @racc_user_yyerror = true # user_yyerror + return -reduce_n + when 2 # yyaccept + return shift_n + else + raise '[Racc Bug] unknown jump code' + end + end + + elsif act == shift_n + # + # accept + # + racc_accept if @yydebug + throw :racc_end_parse, @racc_vstack[0] + + elsif act == -reduce_n + # + # error + # + case @racc_error_status + when 0 + unless arg[21] # user_yyerror + nerr += 1 + on_error @racc_t, @racc_val, @racc_vstack + end + when 3 + if @racc_t == 0 # is $ + # We're at EOF, and another error occurred immediately after + # attempting auto-recovery + throw :racc_end_parse, nil + end + @racc_read_next = true + end + @racc_user_yyerror = false + @racc_error_status = 3 + while true + if i = action_pointer[@racc_state[-1]] + i += 1 # error token + if i >= 0 and + (act = action_table[i]) and + action_check[i] == @racc_state[-1] + break + end + end + throw :racc_end_parse, nil if @racc_state.size <= 1 + @racc_state.pop + @racc_vstack.pop + if @yydebug + @racc_tstack.pop + racc_e_pop @racc_state, @racc_tstack, @racc_vstack + end + end + return act + + else + raise "[Racc Bug] unknown action #{act.inspect}" + end + + racc_next_state(@racc_state[-1], @racc_state) if @yydebug + + nil + end + + def _racc_do_reduce(arg, act) + _, _, _, _, + goto_table, goto_check, goto_default, goto_pointer, + nt_base, reduce_table, _, _, + _, use_result, * = arg + + state = @racc_state + vstack = @racc_vstack + tstack = @racc_tstack + + i = act * -3 + len = reduce_table[i] + reduce_to = reduce_table[i+1] + method_id = reduce_table[i+2] + void_array = [] + + tmp_t = tstack[-len, len] if @yydebug + tmp_v = vstack[-len, len] + tstack[-len, len] = void_array if @yydebug + vstack[-len, len] = void_array + state[-len, len] = void_array + + # tstack must be updated AFTER method call + if use_result + vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) + else + vstack.push __send__(method_id, tmp_v, vstack) + end + tstack.push reduce_to + + racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug + + k1 = reduce_to - nt_base + if i = goto_pointer[k1] + i += state[-1] + if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 + return curstate + end + end + goto_default[k1] + end + + # This method is called when a parse error is found. + # + # ERROR_TOKEN_ID is an internal ID of token which caused error. + # You can get string representation of this ID by calling + # #token_to_str. + # + # ERROR_VALUE is a value of error token. + # + # value_stack is a stack of symbol values. + # DO NOT MODIFY this object. + # + # This method raises ParseError by default. + # + # If this method returns, parsers enter "error recovering mode". + def on_error(t, val, vstack) + raise ParseError, sprintf("parse error on value %s (%s)", + val.inspect, token_to_str(t) || '?') + end + + # Enter error recovering mode. + # This method does not call #on_error. + def yyerror + throw :racc_jump, 1 + end + + # Exit parser. + # Return value is +Symbol_Value_Stack[0]+. + def yyaccept + throw :racc_jump, 2 + end + + # Leave error recovering mode. + def yyerrok + @racc_error_status = 0 + end + + # For debugging output + def racc_read_token(t, tok, val) + @racc_debug_out.print 'read ' + @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' + @racc_debug_out.puts val.inspect + @racc_debug_out.puts + end + + def racc_shift(tok, tstack, vstack) + @racc_debug_out.puts "shift #{racc_token2str tok}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_reduce(toks, sim, tstack, vstack) + out = @racc_debug_out + out.print 'reduce ' + if toks.empty? + out.print ' <none>' + else + toks.each {|t| out.print ' ', racc_token2str(t) } + end + out.puts " --> #{racc_token2str(sim)}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_accept + @racc_debug_out.puts 'accept' + @racc_debug_out.puts + end + + def racc_e_pop(state, tstack, vstack) + @racc_debug_out.puts 'error recovering mode: pop token' + racc_print_states state + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_next_state(curstate, state) + @racc_debug_out.puts "goto #{curstate}" + racc_print_states state + @racc_debug_out.puts + end + + def racc_print_stacks(t, v) + out = @racc_debug_out + out.print ' [' + t.each_index do |i| + out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' + end + out.puts ' ]' + end + + def racc_print_states(s) + out = @racc_debug_out + out.print ' [' + s.each {|st| out.print ' ', st } + out.puts ' ]' + end + + def racc_token2str(tok) + self.class::Racc_token_to_s_table[tok] or + raise "[Racc Bug] can't convert token #{tok} to string" + end + + # Convert internal ID of token symbol to the string. + def token_to_str(t) + self.class::Racc_token_to_s_table[t] + end + + end + +end + +...end racc/parser.rb/module_eval... +end +###### racc/parser.rb end +module Lrama + class Parser < Racc::Parser + +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 504) + +include Lrama::Tracer::Duration + +def initialize(text, path, debug = false, locations = false, define = {}) + @path = path + @grammar_file = Lrama::Lexer::GrammarFile.new(path, text) + @yydebug = debug || define.key?('parse.trace') + @rule_counter = Lrama::Grammar::Counter.new(0) + @midrule_action_counter = Lrama::Grammar::Counter.new(1) + @locations = locations + @define = define +end + +def parse + message = "parse '#{File.basename(@path)}'" + report_duration(message) do + @lexer = Lrama::Lexer.new(@grammar_file) + @grammar = Lrama::Grammar.new(@rule_counter, @locations, @define) + @precedence_number = 0 + reset_precs + do_parse + @grammar + end +end + +def next_token + @lexer.next_token +end + +def on_error(error_token_id, error_value, value_stack) + case error_value + when Lrama::Lexer::Token::Int + location = error_value.location + value = "#{error_value.s_value}" + when Lrama::Lexer::Token::Token + location = error_value.location + value = "\"#{error_value.s_value}\"" + when Lrama::Lexer::Token::Base + location = error_value.location + value = "'#{error_value.s_value}'" + else + location = @lexer.location + value = error_value.inspect + end + + error_message = "parse error on value #{value} (#{token_to_str(error_token_id) || '?'})" + + raise_parse_error(error_message, location) +end + +def on_action_error(error_message, error_value) + if error_value.is_a?(Lrama::Lexer::Token::Base) + location = error_value.location + else + location = @lexer.location + end + + raise_parse_error(error_message, location) +end + +private + +def reset_precs + @opening_prec_seen = false + @trailing_prec_seen = false + @code_after_prec = false +end + +def prec_seen? + @opening_prec_seen || @trailing_prec_seen +end + +def begin_c_declaration(end_symbol) + @lexer.status = :c_declaration + @lexer.end_symbol = end_symbol +end + +def end_c_declaration + @lexer.status = :initial + @lexer.end_symbol = nil +end + +def raise_parse_error(error_message, location) + raise ParseError, location.generate_error_message(error_message) +end +...end parser.y/module_eval... +##### State transition tables begin ### + +racc_action_table = [ + 98, 98, 99, 99, 87, 53, 53, 52, 178, 110, + 110, 97, 53, 53, 184, 178, 110, 110, 53, 181, + 184, 162, 110, 6, 163, 181, 181, 53, 53, 52, + 52, 181, 79, 79, 53, 53, 52, 52, 43, 79, + 79, 53, 4, 52, 5, 110, 88, 94, 182, 125, + 126, 163, 100, 100, 180, 193, 194, 195, 137, 185, + 188, 180, 4, 44, 5, 185, 188, 94, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 46, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 47, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 47, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 50, + 57, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 57, + 60, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 53, 53, 52, + 52, 110, 105, 53, 53, 52, 52, 110, 105, 53, + 53, 52, 52, 110, 105, 53, 53, 52, 52, 110, + 105, 53, 53, 52, 52, 110, 110, 53, 53, 52, + 209, 110, 110, 53, 53, 209, 52, 110, 110, 53, + 53, 209, 52, 110, 193, 194, 195, 137, 216, 222, + 229, 217, 217, 217, 53, 53, 52, 52, 193, 194, + 195, 57, 57, 57, 57, 66, 67, 68, 69, 70, + 72, 72, 72, 86, 89, 47, 57, 57, 113, 117, + 117, 79, 123, 124, 131, 47, 133, 137, 139, 143, + 149, 150, 151, 152, 133, 155, 156, 157, 110, 166, + 149, 169, 172, 173, 72, 175, 176, 183, 189, 166, + 196, 137, 200, 202, 137, 166, 211, 166, 137, 72, + 176, 218, 176, 72, 72, 227, 137, 72 ] + +racc_action_check = [ + 51, 97, 51, 97, 41, 75, 165, 75, 165, 75, + 165, 51, 171, 190, 171, 190, 171, 190, 201, 165, + 201, 148, 201, 1, 148, 171, 190, 36, 37, 36, + 37, 201, 36, 37, 38, 39, 38, 39, 5, 38, + 39, 117, 0, 117, 0, 117, 41, 46, 168, 88, + 88, 168, 51, 97, 165, 177, 177, 177, 177, 171, + 171, 190, 2, 6, 2, 201, 201, 90, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 9, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 10, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 11, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 3, 3, 12, + 14, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 15, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 8, 8, 16, + 17, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 18, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 63, 13, 63, + 13, 63, 63, 64, 73, 64, 73, 64, 64, 65, + 78, 65, 78, 65, 65, 106, 79, 106, 79, 106, + 106, 118, 180, 118, 180, 118, 180, 188, 196, 188, + 196, 188, 196, 202, 217, 202, 217, 202, 217, 218, + 113, 218, 113, 218, 186, 186, 186, 186, 208, 213, + 226, 208, 213, 226, 114, 123, 114, 123, 210, 210, + 210, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 40, 42, 47, 55, 60, 71, 74, + 76, 80, 81, 87, 91, 92, 93, 94, 102, 116, + 124, 125, 126, 127, 133, 136, 137, 138, 144, 150, + 151, 153, 156, 158, 162, 163, 164, 170, 174, 176, + 178, 179, 182, 184, 187, 189, 199, 200, 204, 205, + 207, 209, 212, 214, 216, 221, 222, 228 ] + +racc_action_pointer = [ + 32, 23, 52, 93, nil, 31, 63, nil, 123, 68, + 74, 84, 103, 165, 94, 111, 123, 135, 141, nil, + nil, nil, nil, nil, 215, 216, 217, 218, 230, 231, + 232, 233, 234, 232, 233, 234, 24, 25, 31, 32, + 238, -1, 242, nil, nil, nil, 43, 232, nil, nil, + nil, -5, nil, nil, nil, 230, nil, nil, nil, nil, + 231, nil, nil, 164, 170, 176, nil, nil, nil, nil, + nil, 240, nil, 171, 241, 2, 242, nil, 177, 183, + 243, 244, nil, nil, nil, nil, nil, 209, 45, nil, + 63, 245, 242, 243, 202, nil, nil, -4, nil, nil, + nil, nil, 256, nil, nil, nil, 182, nil, nil, nil, + nil, nil, nil, 207, 221, nil, 253, 38, 188, nil, + nil, nil, nil, 222, 255, 215, 218, 252, nil, nil, + nil, nil, nil, 251, nil, nil, 219, 261, 250, nil, + nil, nil, nil, nil, 261, nil, nil, nil, -24, nil, + 219, 265, nil, 269, nil, nil, 216, nil, 256, nil, + nil, nil, 266, 270, 227, 3, nil, nil, 3, nil, + 228, 9, nil, nil, 232, nil, 229, 3, 236, 226, + 189, nil, 236, nil, 239, nil, 162, 229, 194, 235, + 10, nil, nil, nil, nil, nil, 195, nil, nil, 284, + 237, 15, 200, nil, 233, 281, nil, 241, 173, 247, + 176, nil, 243, 174, 285, nil, 286, 201, 206, nil, + nil, 278, 241, nil, nil, nil, 175, nil, 289, nil, + nil ] + +racc_action_default = [ + -1, -136, -1, -3, -10, -136, -136, -2, -3, -136, + -14, -14, -136, -136, -136, -136, -136, -136, -136, -28, + -29, -34, -35, -36, -136, -136, -136, -136, -136, -136, + -136, -136, -136, -54, -54, -54, -136, -136, -136, -136, + -136, -136, -136, -13, 231, -4, -136, -14, -16, -17, + -20, -131, -100, -101, -130, -18, -23, -89, -24, -25, + -136, -27, -37, -136, -136, -136, -41, -42, -43, -44, + -45, -46, -55, -136, -47, -136, -48, -49, -92, -136, + -95, -97, -98, -50, -51, -52, -53, -136, -136, -11, + -5, -7, -14, -136, -72, -15, -21, -131, -132, -133, + -134, -19, -136, -26, -30, -31, -32, -38, -87, -88, + -135, -39, -40, -136, -56, -58, -60, -136, -83, -85, + -93, -94, -96, -136, -136, -136, -136, -136, -6, -8, + -9, -128, -104, -102, -105, -73, -136, -136, -136, -90, + -33, -59, -57, -61, -80, -86, -84, -99, -136, -66, + -70, -136, -12, -136, -103, -109, -136, -22, -136, -62, + -81, -82, -54, -136, -64, -68, -71, -74, -136, -129, + -106, -107, -127, -91, -136, -67, -70, -72, -100, -72, + -136, -124, -136, -109, -100, -110, -72, -72, -136, -70, + -69, -75, -76, -116, -117, -118, -136, -78, -79, -136, + -70, -108, -136, -111, -72, -54, -115, -63, -136, -100, + -119, -125, -65, -136, -54, -114, -54, -136, -136, -120, + -121, -136, -72, -112, -77, -122, -136, -126, -54, -123, + -113 ] + +racc_goto_table = [ + 73, 118, 136, 54, 48, 49, 164, 96, 91, 120, + 121, 93, 187, 148, 107, 111, 112, 119, 134, 171, + 56, 58, 59, 3, 61, 7, 78, 78, 78, 78, + 62, 63, 64, 65, 115, 74, 76, 192, 1, 129, + 168, 95, 187, 118, 118, 207, 204, 201, 77, 83, + 84, 85, 128, 138, 147, 93, 212, 140, 154, 145, + 146, 101, 130, 116, 42, 127, 103, 208, 78, 78, + 219, 9, 51, 213, 141, 142, 45, 71, 159, 144, + 190, 160, 161, 102, 158, 191, 132, 197, 122, 226, + 170, 177, 220, 199, 203, 205, 221, 186, 153, nil, + nil, nil, nil, 116, 116, nil, 198, nil, nil, nil, + nil, nil, 214, 78, 206, nil, 177, nil, nil, nil, + nil, nil, 210, nil, nil, nil, nil, 186, 210, 174, + 228, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 225, 210, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 215, nil, nil, nil, nil, nil, nil, nil, + nil, 223, nil, 224, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 230 ] + +racc_goto_check = [ + 29, 22, 42, 31, 14, 14, 35, 16, 8, 48, + 48, 13, 40, 34, 24, 24, 24, 45, 52, 54, + 18, 18, 18, 6, 17, 6, 31, 31, 31, 31, + 17, 17, 17, 17, 30, 26, 26, 38, 1, 5, + 34, 14, 40, 22, 22, 35, 38, 54, 27, 27, + 27, 27, 8, 16, 48, 13, 35, 24, 52, 45, + 45, 18, 9, 31, 10, 11, 17, 39, 31, 31, + 38, 7, 15, 39, 30, 30, 7, 25, 32, 33, + 36, 43, 44, 46, 47, 42, 14, 42, 50, 39, + 53, 22, 55, 56, 42, 42, 57, 22, 58, nil, + nil, nil, nil, 31, 31, nil, 22, nil, nil, nil, + nil, nil, 42, 31, 22, nil, 22, nil, nil, nil, + nil, nil, 22, nil, nil, nil, nil, 22, 22, 29, + 42, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 22, 22, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 29, nil, nil, nil, nil, nil, nil, nil, + nil, 29, nil, 29, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 29 ] + +racc_goto_pointer = [ + nil, 38, nil, nil, nil, -52, 23, 68, -38, -29, + 60, -24, nil, -35, -6, 59, -44, 6, 6, nil, + nil, nil, -74, nil, -49, 44, 1, 12, nil, -33, + -39, -10, -66, -37, -111, -144, -96, nil, -140, -129, + -159, nil, -92, -63, -62, -58, 26, -55, -69, nil, + 8, nil, -75, -65, -136, -118, -88, -115, -33 ] + +racc_goto_default = [ + nil, nil, 2, 8, 90, nil, nil, nil, nil, nil, + nil, nil, 10, 11, nil, nil, nil, 55, nil, 21, + 22, 23, 104, 106, nil, nil, nil, nil, 114, 75, + nil, 108, nil, nil, nil, nil, 165, 135, nil, nil, + 179, 167, nil, 109, nil, nil, nil, nil, 81, 80, + 82, 92, nil, nil, nil, nil, nil, nil, nil ] + +racc_reduce_table = [ + 0, 0, :racc_error, + 0, 64, :_reduce_1, + 2, 64, :_reduce_2, + 0, 65, :_reduce_3, + 2, 65, :_reduce_4, + 1, 66, :_reduce_5, + 2, 66, :_reduce_6, + 0, 67, :_reduce_none, + 1, 67, :_reduce_none, + 5, 59, :_reduce_none, + 0, 68, :_reduce_10, + 0, 69, :_reduce_11, + 5, 60, :_reduce_12, + 2, 60, :_reduce_13, + 0, 72, :_reduce_14, + 2, 72, :_reduce_15, + 2, 61, :_reduce_none, + 2, 61, :_reduce_none, + 1, 76, :_reduce_18, + 2, 76, :_reduce_19, + 2, 70, :_reduce_20, + 3, 70, :_reduce_21, + 5, 70, :_reduce_22, + 2, 70, :_reduce_none, + 2, 70, :_reduce_24, + 2, 70, :_reduce_25, + 3, 70, :_reduce_26, + 2, 70, :_reduce_27, + 1, 70, :_reduce_28, + 1, 70, :_reduce_29, + 1, 81, :_reduce_30, + 1, 81, :_reduce_31, + 1, 82, :_reduce_32, + 2, 82, :_reduce_33, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 2, 71, :_reduce_37, + 3, 71, :_reduce_38, + 3, 71, :_reduce_39, + 3, 71, :_reduce_40, + 2, 71, :_reduce_41, + 2, 71, :_reduce_42, + 2, 71, :_reduce_43, + 2, 71, :_reduce_44, + 2, 71, :_reduce_45, + 2, 77, :_reduce_none, + 2, 77, :_reduce_47, + 2, 77, :_reduce_48, + 2, 77, :_reduce_49, + 2, 77, :_reduce_50, + 2, 77, :_reduce_51, + 2, 77, :_reduce_52, + 2, 77, :_reduce_53, + 0, 87, :_reduce_none, + 1, 87, :_reduce_none, + 1, 88, :_reduce_56, + 2, 88, :_reduce_57, + 2, 83, :_reduce_58, + 3, 83, :_reduce_59, + 0, 91, :_reduce_none, + 1, 91, :_reduce_none, + 3, 86, :_reduce_62, + 8, 78, :_reduce_63, + 5, 79, :_reduce_64, + 8, 79, :_reduce_65, + 1, 92, :_reduce_66, + 3, 92, :_reduce_67, + 1, 93, :_reduce_68, + 3, 93, :_reduce_69, + 0, 99, :_reduce_none, + 1, 99, :_reduce_none, + 0, 100, :_reduce_none, + 1, 100, :_reduce_none, + 1, 94, :_reduce_74, + 3, 94, :_reduce_75, + 3, 94, :_reduce_76, + 6, 94, :_reduce_77, + 3, 94, :_reduce_78, + 3, 94, :_reduce_79, + 0, 102, :_reduce_none, + 1, 102, :_reduce_none, + 1, 90, :_reduce_82, + 1, 103, :_reduce_83, + 2, 103, :_reduce_84, + 2, 84, :_reduce_85, + 3, 84, :_reduce_86, + 1, 80, :_reduce_none, + 1, 80, :_reduce_none, + 0, 104, :_reduce_89, + 0, 105, :_reduce_90, + 5, 75, :_reduce_91, + 1, 106, :_reduce_92, + 2, 106, :_reduce_93, + 2, 107, :_reduce_94, + 1, 108, :_reduce_95, + 2, 108, :_reduce_96, + 1, 85, :_reduce_97, + 1, 85, :_reduce_98, + 3, 85, :_reduce_99, + 1, 89, :_reduce_none, + 1, 89, :_reduce_none, + 1, 110, :_reduce_102, + 2, 110, :_reduce_103, + 2, 62, :_reduce_none, + 2, 62, :_reduce_none, + 4, 109, :_reduce_106, + 1, 111, :_reduce_107, + 3, 111, :_reduce_108, + 0, 112, :_reduce_109, + 2, 112, :_reduce_110, + 3, 112, :_reduce_111, + 5, 112, :_reduce_112, + 7, 112, :_reduce_113, + 4, 112, :_reduce_114, + 3, 112, :_reduce_115, + 1, 96, :_reduce_116, + 1, 96, :_reduce_117, + 1, 96, :_reduce_118, + 0, 113, :_reduce_none, + 1, 113, :_reduce_none, + 2, 97, :_reduce_121, + 3, 97, :_reduce_122, + 4, 97, :_reduce_123, + 0, 114, :_reduce_124, + 0, 115, :_reduce_125, + 5, 98, :_reduce_126, + 3, 95, :_reduce_127, + 0, 116, :_reduce_128, + 3, 63, :_reduce_129, + 1, 73, :_reduce_none, + 0, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 101, :_reduce_135 ] + +racc_reduce_n = 136 + +racc_shift_n = 231 + +racc_token_table = { + false => 0, + :error => 1, + :C_DECLARATION => 2, + :CHARACTER => 3, + :IDENT_COLON => 4, + :IDENTIFIER => 5, + :INTEGER => 6, + :STRING => 7, + :TAG => 8, + "%%" => 9, + "%{" => 10, + "%}" => 11, + "%require" => 12, + ";" => 13, + "%expect" => 14, + "%define" => 15, + "{" => 16, + "}" => 17, + "%param" => 18, + "%lex-param" => 19, + "%parse-param" => 20, + "%code" => 21, + "%initial-action" => 22, + "%no-stdlib" => 23, + "%locations" => 24, + "%union" => 25, + "%destructor" => 26, + "%printer" => 27, + "%error-token" => 28, + "%after-shift" => 29, + "%before-reduce" => 30, + "%after-reduce" => 31, + "%after-shift-error-token" => 32, + "%after-pop-stack" => 33, + "-temp-group" => 34, + "%token" => 35, + "%type" => 36, + "%nterm" => 37, + "%left" => 38, + "%right" => 39, + "%precedence" => 40, + "%nonassoc" => 41, + "%start" => 42, + "%rule" => 43, + "(" => 44, + ")" => 45, + ":" => 46, + "%inline" => 47, + "," => 48, + "|" => 49, + "%empty" => 50, + "%prec" => 51, + "?" => 52, + "+" => 53, + "*" => 54, + "[" => 55, + "]" => 56, + "{...}" => 57 } + +racc_nt_base = 58 + +racc_use_result_var = true + +Racc_arg = [ + racc_action_table, + racc_action_check, + racc_action_default, + racc_action_pointer, + racc_goto_table, + racc_goto_check, + racc_goto_default, + racc_goto_pointer, + racc_nt_base, + racc_reduce_table, + racc_token_table, + racc_shift_n, + racc_reduce_n, + racc_use_result_var ] +Ractor.make_shareable(Racc_arg) if defined?(Ractor) + +Racc_token_to_s_table = [ + "$end", + "error", + "C_DECLARATION", + "CHARACTER", + "IDENT_COLON", + "IDENTIFIER", + "INTEGER", + "STRING", + "TAG", + "\"%%\"", + "\"%{\"", + "\"%}\"", + "\"%require\"", + "\";\"", + "\"%expect\"", + "\"%define\"", + "\"{\"", + "\"}\"", + "\"%param\"", + "\"%lex-param\"", + "\"%parse-param\"", + "\"%code\"", + "\"%initial-action\"", + "\"%no-stdlib\"", + "\"%locations\"", + "\"%union\"", + "\"%destructor\"", + "\"%printer\"", + "\"%error-token\"", + "\"%after-shift\"", + "\"%before-reduce\"", + "\"%after-reduce\"", + "\"%after-shift-error-token\"", + "\"%after-pop-stack\"", + "\"-temp-group\"", + "\"%token\"", + "\"%type\"", + "\"%nterm\"", + "\"%left\"", + "\"%right\"", + "\"%precedence\"", + "\"%nonassoc\"", + "\"%start\"", + "\"%rule\"", + "\"(\"", + "\")\"", + "\":\"", + "\"%inline\"", + "\",\"", + "\"|\"", + "\"%empty\"", + "\"%prec\"", + "\"?\"", + "\"+\"", + "\"*\"", + "\"[\"", + "\"]\"", + "\"{...}\"", + "$start", + "input", + "prologue_declaration", + "bison_declaration", + "rules_or_grammar_declaration", + "epilogue_declaration", + "\"-many@prologue_declaration\"", + "\"-many@bison_declaration\"", + "\"-many1@rules_or_grammar_declaration\"", + "\"-option@epilogue_declaration\"", + "@1", + "@2", + "parser_option", + "grammar_declaration", + "\"-many@;\"", + "variable", + "value", + "param", + "\"-many1@param\"", + "symbol_declaration", + "rule_declaration", + "inline_declaration", + "symbol", + "\"-group@symbol|TAG\"", + "\"-many1@-group@symbol|TAG\"", + "token_declarations", + "symbol_declarations", + "token_declarations_for_precedence", + "token_declaration", + "\"-option@TAG\"", + "\"-many1@token_declaration\"", + "id", + "alias", + "\"-option@INTEGER\"", + "rule_args", + "rule_rhs_list", + "rule_rhs", + "named_ref", + "parameterized_suffix", + "parameterized_args", + "action", + "\"-option@%empty\"", + "\"-option@named_ref\"", + "string_as_id", + "\"-option@string_as_id\"", + "\"-many1@symbol\"", + "@3", + "@4", + "\"-many1@id\"", + "\"-group@TAG-\\\"-many1@id\\\"\"", + "\"-many1@-group@TAG-\\\"-many1@id\\\"\"", + "rules", + "\"-many1@;\"", + "rhs_list", + "rhs", + "\"-option@parameterized_suffix\"", + "@5", + "@6", + "@7" ] +Ractor.make_shareable(Racc_token_to_s_table) if defined?(Ractor) + +Racc_debug_parser = true + +##### State transition tables end ##### + +# reduce 0 omitted + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_1(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_2(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_3(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_4(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_5(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_6(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 7 omitted + +# reduce 8 omitted + +# reduce 9 omitted + +module_eval(<<'.,.,', 'parser.y', 13) + def _reduce_10(val, _values, result) + begin_c_declaration("%}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 17) + def _reduce_11(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 21) + def _reduce_12(val, _values, result) + @grammar.prologue_first_lineno = val[0].first_line + @grammar.prologue = val[2].s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 26) + def _reduce_13(val, _values, result) + @grammar.required = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_14(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_15(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 16 omitted + +# reduce 17 omitted + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_18(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_19(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 36) + def _reduce_20(val, _values, result) + @grammar.expect = val[1].s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 40) + def _reduce_21(val, _values, result) + @grammar.define[val[1].s_value] = val[2]&.s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 44) + def _reduce_22(val, _values, result) + @grammar.define[val[1].s_value] = val[3]&.s_value + + result + end +.,., + +# reduce 23 omitted + +module_eval(<<'.,.,', 'parser.y', 49) + def _reduce_24(val, _values, result) + val[1].each {|token| + @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 55) + def _reduce_25(val, _values, result) + val[1].each {|token| + @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 61) + def _reduce_26(val, _values, result) + @grammar.add_percent_code(id: val[1], code: val[2]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 65) + def _reduce_27(val, _values, result) + @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 69) + def _reduce_28(val, _values, result) + @grammar.no_stdlib = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 73) + def _reduce_29(val, _values, result) + @grammar.locations = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_30(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_31(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_32(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_33(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 34 omitted + +# reduce 35 omitted + +# reduce 36 omitted + +module_eval(<<'.,.,', 'parser.y', 82) + def _reduce_37(val, _values, result) + @grammar.set_union( + Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), + val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 89) + def _reduce_38(val, _values, result) + @grammar.add_destructor( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 97) + def _reduce_39(val, _values, result) + @grammar.add_printer( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 105) + def _reduce_40(val, _values, result) + @grammar.add_error_token( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 113) + def _reduce_41(val, _values, result) + @grammar.after_shift = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 117) + def _reduce_42(val, _values, result) + @grammar.before_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 121) + def _reduce_43(val, _values, result) + @grammar.after_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 125) + def _reduce_44(val, _values, result) + @grammar.after_shift_error_token = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 129) + def _reduce_45(val, _values, result) + @grammar.after_pop_stack = val[1] + + result + end +.,., + +# reduce 46 omitted + +module_eval(<<'.,.,', 'parser.y', 136) + def _reduce_47(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + @grammar.add_type(id: id, tag: hash[:tag]) + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 144) + def _reduce_48(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + if @grammar.find_term_by_s_value(id.s_value) + on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) + else + @grammar.add_type(id: id, tag: hash[:tag]) + end + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 156) + def _reduce_49(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_left(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 166) + def _reduce_50(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_right(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 176) + def _reduce_51(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_precedence(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 186) + def _reduce_52(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_nonassoc(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 196) + def _reduce_53(val, _values, result) + @grammar.set_start_nterm(val[1]) + + result + end +.,., + +# reduce 54 omitted + +# reduce 55 omitted + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_56(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_57(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 202) + def _reduce_58(val, _values, result) + val[1].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[0], replace: true) + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 208) + def _reduce_59(val, _values, result) + val[2].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[1], replace: true) + } + + result + end +.,., + +# reduce 60 omitted + +# reduce 61 omitted + +module_eval(<<'.,.,', 'parser.y', 213) + def _reduce_62(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 218) + def _reduce_63(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 225) + def _reduce_64(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, [], val[4], is_inline: true) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 230) + def _reduce_65(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 235) + def _reduce_66(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 236) + def _reduce_67(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 241) + def _reduce_68(val, _values, result) + builder = val[0] + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 246) + def _reduce_69(val, _values, result) + builder = val[2] + result = val[0].append(builder) + + result + end +.,., + +# reduce 70 omitted + +# reduce 71 omitted + +# reduce 72 omitted + +# reduce 73 omitted + +module_eval(<<'.,.,', 'parser.y', 253) + def _reduce_74(val, _values, result) + reset_precs + result = Grammar::Parameterized::Rhs.new + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 258) + def _reduce_75(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.symbols << token + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 267) + def _reduce_76(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 274) + def _reduce_77(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 281) + def _reduce_78(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 289) + def _reduce_79(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +# reduce 80 omitted + +# reduce 81 omitted + +module_eval(<<'.,.,', 'parser.y', 301) + def _reduce_82(val, _values, result) + result = val[0].s_value if val[0] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_83(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_84(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 306) + def _reduce_85(val, _values, result) + result = if val[0] + [{tag: val[0], tokens: val[1]}] + else + [{tag: nil, tokens: val[1]}] + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 312) + def _reduce_86(val, _values, result) + result = val[0].append({tag: val[1], tokens: val[2]}) + result + end +.,., + +# reduce 87 omitted + +# reduce 88 omitted + +module_eval(<<'.,.,', 'parser.y', 321) + def _reduce_89(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 325) + def _reduce_90(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 329) + def _reduce_91(val, _values, result) + result = val[2] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_92(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_93(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_94(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_95(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_96(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 333) + def _reduce_97(val, _values, result) + result = [{tag: nil, tokens: val[0]}] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 334) + def _reduce_98(val, _values, result) + result = val[0].map {|tag, ids| {tag: tag, tokens: ids} } + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 335) + def _reduce_99(val, _values, result) + result = [{tag: nil, tokens: val[0]}, {tag: val[1], tokens: val[2]}] + result + end +.,., + +# reduce 100 omitted + +# reduce 101 omitted + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_102(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_103(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 104 omitted + +# reduce 105 omitted + +module_eval(<<'.,.,', 'parser.y', 348) + def _reduce_106(val, _values, result) + lhs = val[0] + lhs.alias_name = val[1] + val[3].each do |builder| + builder.lhs = lhs + builder.complete_input + @grammar.add_rule_builder(builder) + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 360) + def _reduce_107(val, _values, result) + if val[0].rhs.count > 1 + empties = val[0].rhs.select { |sym| sym.is_a?(Lrama::Lexer::Token::Empty) } + empties.each do |empty| + on_action_error("%empty on non-empty rule", empty) + end + end + builder = val[0] + if !builder.line + builder.line = @lexer.line - 1 + end + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 374) + def _reduce_108(val, _values, result) + builder = val[2] + if !builder.line + builder.line = @lexer.line - 1 + end + result = val[0].append(builder) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 384) + def _reduce_109(val, _values, result) + reset_precs + result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 389) + def _reduce_110(val, _values, result) + builder = val[0] + builder.add_rhs(Lrama::Lexer::Token::Empty.new(location: @lexer.location)) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 395) + def _reduce_111(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.add_rhs(token) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 404) + def _reduce_112(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 413) + def _reduce_113(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 422) + def _reduce_114(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + user_code.tag = val[3] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 431) + def _reduce_115(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 444) + def _reduce_116(val, _values, result) + result = "option" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 445) + def _reduce_117(val, _values, result) + result = "nonempty_list" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 446) + def _reduce_118(val, _values, result) + result = "list" + result + end +.,., + +# reduce 119 omitted + +# reduce 120 omitted + +module_eval(<<'.,.,', 'parser.y', 451) + def _reduce_121(val, _values, result) + result = if val[1] + [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] + else + [val[0]] + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 457) + def _reduce_122(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 458) + def _reduce_123(val, _values, result) + result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 463) + def _reduce_124(val, _values, result) + if prec_seen? + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 471) + def _reduce_125(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 475) + def _reduce_126(val, _values, result) + result = val[2] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 478) + def _reduce_127(val, _values, result) + result = val[1].s_value + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 483) + def _reduce_128(val, _values, result) + begin_c_declaration('\Z') + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 487) + def _reduce_129(val, _values, result) + end_c_declaration + @grammar.epilogue_first_lineno = val[0].first_line + 1 + @grammar.epilogue = val[2].s_value + + result + end +.,., + +# reduce 130 omitted + +# reduce 131 omitted + +# reduce 132 omitted + +# reduce 133 omitted + +# reduce 134 omitted + +module_eval(<<'.,.,', 'parser.y', 499) + def _reduce_135(val, _values, result) + result = Lrama::Lexer::Token::Ident.new(s_value: val[0].s_value) + result + end +.,., + +def _reduce_none(val, _values, result) + val[0] +end + + end # class Parser +end # module Lrama diff --git a/tool/lrama/lib/lrama/reporter.rb b/tool/lrama/lib/lrama/reporter.rb new file mode 100644 index 0000000000..ed25cc7f8f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'reporter/conflicts' +require_relative 'reporter/grammar' +require_relative 'reporter/precedences' +require_relative 'reporter/profile' +require_relative 'reporter/rules' +require_relative 'reporter/states' +require_relative 'reporter/terms' + +module Lrama + class Reporter + include Lrama::Tracer::Duration + + # @rbs (**bool options) -> void + def initialize(**options) + @options = options + @rules = Rules.new(**options) + @terms = Terms.new(**options) + @conflicts = Conflicts.new + @precedences = Precedences.new + @grammar = Grammar.new(**options) + @states = States.new(**options) + end + + # @rbs (File io, Lrama::States states) -> void + def report(io, states) + report_duration(:report) do + report_duration(:report_rules) { @rules.report(io, states) } + report_duration(:report_terms) { @terms.report(io, states) } + report_duration(:report_conflicts) { @conflicts.report(io, states) } + report_duration(:report_precedences) { @precedences.report(io, states) } + report_duration(:report_grammar) { @grammar.report(io, states) } + report_duration(:report_states) { @states.report(io, states, ielr: states.ielr_defined?) } + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/conflicts.rb b/tool/lrama/lib/lrama/reporter/conflicts.rb new file mode 100644 index 0000000000..f4d8c604c9 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/conflicts.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Conflicts + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_conflicts(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_conflicts(io, states) + has_conflict = false + + states.states.each do |state| + messages = format_conflict_messages(state.conflicts) + + unless messages.empty? + has_conflict = true + io << "State #{state.id} conflicts: #{messages.join(', ')}\n" + end + end + + io << "\n\n" if has_conflict + end + + # @rbs (Array[(Lrama::State::ShiftReduceConflict | Lrama::State::ReduceReduceConflict)] conflicts) -> Array[String] + def format_conflict_messages(conflicts) + conflict_types = { + shift_reduce: "shift/reduce", + reduce_reduce: "reduce/reduce" + } + + conflict_types.keys.map do |type| + type_conflicts = conflicts.select { |c| c.type == type } + "#{type_conflicts.count} #{conflict_types[type]}" unless type_conflicts.empty? + end.compact + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/grammar.rb b/tool/lrama/lib/lrama/reporter/grammar.rb new file mode 100644 index 0000000000..dc3f3f6bfd --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/grammar.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Grammar + # @rbs (?grammar: bool, **bool _) -> void + def initialize(grammar: false, **_) + @grammar = grammar + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @grammar + + io << "Grammar\n" + last_lhs = nil + + states.rules.each do |rule| + if rule.empty_rule? + r = "ε" + else + r = rule.rhs.map(&:display_name).join(" ") + end + + if rule.lhs == last_lhs + io << sprintf("%5d %s| %s", rule.id, " " * rule.lhs.display_name.length, r) << "\n" + else + io << "\n" + io << sprintf("%5d %s: %s", rule.id, rule.lhs.display_name, r) << "\n" + end + + last_lhs = rule.lhs + end + io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/precedences.rb b/tool/lrama/lib/lrama/reporter/precedences.rb new file mode 100644 index 0000000000..73c0888700 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/precedences.rb @@ -0,0 +1,54 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Precedences + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_precedences(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_precedences(io, states) + used_precedences = states.precedences.select(&:used_by?) + + return if used_precedences.empty? + + io << "Precedences\n\n" + + used_precedences.each do |precedence| + io << " precedence on #{precedence.symbol.display_name} is used to resolve conflict on\n" + + if precedence.used_by_lalr? + io << " LALR\n" + + precedence.used_by_lalr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + + if precedence.used_by_ielr? + io << " IELR\n" + + precedence.used_by_ielr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile.rb b/tool/lrama/lib/lrama/reporter/profile.rb new file mode 100644 index 0000000000..b569b94d4f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'profile/call_stack' +require_relative 'profile/memory' diff --git a/tool/lrama/lib/lrama/reporter/profile/call_stack.rb b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb new file mode 100644 index 0000000000..8a4d44b61c --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module CallStack + # See "Call-stack Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_stackprof + ex = nil #: Exception? + path = 'tmp/stackprof-cpu-myapp.dump' + + StackProf.run(mode: :cpu, raw: true, out: path) do + yield + rescue Exception => e + ex = e + end + + STDERR.puts("Call-stack Profiling result is generated on #{path}") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_stackprof + require "stackprof" + true + rescue LoadError + warn "stackprof is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile/memory.rb b/tool/lrama/lib/lrama/reporter/profile/memory.rb new file mode 100644 index 0000000000..a019581fdf --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/memory.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module Memory + # See "Memory Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_memory_profiler + ex = nil #: Exception? + + report = MemoryProfiler.report do # steep:ignore UnknownConstant + yield + rescue Exception => e + ex = e + end + + report.pretty_print(to_file: "tmp/memory_profiler.txt") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_memory_profiler + require "memory_profiler" + true + rescue LoadError + warn "memory_profiler is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/rules.rb b/tool/lrama/lib/lrama/reporter/rules.rb new file mode 100644 index 0000000000..3e8bf19a0a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/rules.rb @@ -0,0 +1,43 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Rules + # @rbs (?rules: bool, **bool _) -> void + def initialize(rules: false, **_) + @rules = rules + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @rules + + used_rules = states.rules.flat_map(&:rhs) + + unless used_rules.empty? + io << "Rule Usage Frequency\n\n" + frequency_counts = used_rules.each_with_object(Hash.new(0)) { |rule, counts| counts[rule] += 1 } + + frequency_counts + .select { |rule,| !rule.midrule? } + .sort_by { |rule, count| [-count, rule.name] } + .each_with_index { |(rule, count), i| io << sprintf("%5d %s (%d times)", i, rule.name, count) << "\n" } + io << "\n\n" + end + + unused_rules = states.rules.map(&:lhs).select do |rule| + !used_rules.include?(rule) && rule.token_id != 0 + end + + unless unused_rules.empty? + io << "#{unused_rules.count} Unused Rules\n\n" + unused_rules.each_with_index do |rule, index| + io << sprintf("%5d %s", index, rule.display_name) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/states.rb b/tool/lrama/lib/lrama/reporter/states.rb new file mode 100644 index 0000000000..d152d0511a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/states.rb @@ -0,0 +1,387 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class States + # @rbs (?itemsets: bool, ?lookaheads: bool, ?solved: bool, ?counterexamples: bool, ?verbose: bool, **bool _) -> void + def initialize(itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false, **_) + @itemsets = itemsets + @lookaheads = lookaheads + @solved = solved + @counterexamples = counterexamples + @verbose = verbose + end + + # @rbs (IO io, Lrama::States states, ielr: bool) -> void + def report(io, states, ielr: false) + cex = Counterexamples.new(states) if @counterexamples + + states.compute_la_sources_for_conflicted_states + report_split_states(io, states.states) if ielr + + states.states.each do |state| + report_state_header(io, state) + report_items(io, state) + report_conflicts(io, state) + report_shifts(io, state) + report_nonassoc_errors(io, state) + report_reduces(io, state) + report_nterm_transitions(io, state) + report_conflict_resolutions(io, state) if @solved + report_counterexamples(io, state, cex) if @counterexamples && state.has_conflicts? # @type var cex: Lrama::Counterexamples + report_verbose_info(io, state, states) if @verbose + # End of Report State + io << "\n" + end + end + + private + + # @rbs (IO io, Array[Lrama::State] states) -> void + def report_split_states(io, states) + ss = states.select(&:split_state?) + + return if ss.empty? + + io << "Split States\n\n" + + ss.each do |state| + io << " State #{state.id} is split from state #{state.lalr_isocore.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_state_header(io, state) + io << "State #{state.id}\n\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_items(io, state) + last_lhs = nil + list = @itemsets ? state.items : state.kernels + + list.sort_by {|i| [i.rule_id, i.position] }.each do |item| + r = item.empty_rule? ? "ε •" : item.rhs.map(&:display_name).insert(item.position, "•").join(" ") + + l = if item.lhs == last_lhs + " " * item.lhs.id.s_value.length + "|" + else + item.lhs.id.s_value + ":" + end + + la = "" + if @lookaheads && item.end_of_rule? + reduce = state.find_reduce_by_item!(item) + look_ahead = reduce.selected_look_ahead + unless look_ahead.empty? + la = " [#{look_ahead.compact.map(&:display_name).join(", ")}]" + end + end + + last_lhs = item.lhs + io << sprintf("%5i %s %s%s", item.rule_id, l, r, la) << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflicts(io, state) + return if state.conflicts.empty? + + state.conflicts.each do |conflict| + syms = conflict.symbols.map { |sym| sym.display_name } + io << " Conflict on #{syms.join(", ")}. " + + case conflict.type + when :shift_reduce + # @type var conflict: Lrama::State::ShiftReduceConflict + io << "shift/reduce(#{conflict.reduce.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + when :reduce_reduce + # @type var conflict: Lrama::State::ReduceReduceConflict + io << "reduce(#{conflict.reduce1.item.rule.lhs.display_name})/reduce(#{conflict.reduce2.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce1.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + + conflict.reduce2.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + else + raise "Unknown conflict type #{conflict.type}" + end + + io << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_shifts(io, state) + shifts = state.term_transitions.reject(&:not_selected) + + return if shifts.empty? + + next_syms = shifts.map(&:next_sym) + max_len = next_syms.map(&:display_name).map(&:length).max + shifts.each do |shift| + io << " #{shift.next_sym.display_name.ljust(max_len)} shift, and go to state #{shift.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nonassoc_errors(io, state) + error_symbols = state.resolved_conflicts.select { |resolved| resolved.which == :error }.map { |error| error.symbol.display_name } + + return if error_symbols.empty? + + max_len = error_symbols.map(&:length).max + error_symbols.each do |name| + io << " #{name.ljust(max_len)} error (nonassociative)\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_reduces(io, state) + reduce_pairs = [] #: Array[[Lrama::Grammar::Symbol, Lrama::State::Action::Reduce]] + + state.non_default_reduces.each do |reduce| + reduce.look_ahead&.each do |term| + reduce_pairs << [term, reduce] + end + end + + return if reduce_pairs.empty? && !state.default_reduction_rule + + max_len = [ + reduce_pairs.map(&:first).map(&:display_name).map(&:length).max || 0, + state.default_reduction_rule ? "$default".length : 0 + ].max + + reduce_pairs.sort_by { |term, _| term.number }.each do |term, reduce| + rule = reduce.item.rule + io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" + end + + if (r = state.default_reduction_rule) + s = "$default".ljust(max_len) + + if r.initial_rule? + io << " #{s} accept\n" + else + io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nterm_transitions(io, state) + return if state.nterm_transitions.empty? + + goto_transitions = state.nterm_transitions.sort_by do |goto| + goto.next_sym.number + end + + max_len = goto_transitions.map(&:next_sym).map do |nterm| + nterm.id.s_value.length + end.max + goto_transitions.each do |goto| + io << " #{goto.next_sym.id.s_value.ljust(max_len)} go to state #{goto.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflict_resolutions(io, state) + return if state.resolved_conflicts.empty? + + state.resolved_conflicts.each do |resolved| + io << " #{resolved.report_message}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::Counterexamples cex) -> void + def report_counterexamples(io, state, cex) + examples = cex.compute(state) + + examples.each do |example| + is_shift_reduce = example.type == :shift_reduce + label0 = is_shift_reduce ? "shift/reduce" : "reduce/reduce" + label1 = is_shift_reduce ? "Shift derivation" : "First Reduce derivation" + label2 = is_shift_reduce ? "Reduce derivation" : "Second Reduce derivation" + + io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" + io << " #{label1}\n" + + example.derivations1.render_strings_for_report.each do |str| + io << " #{str}\n" + end + + io << " #{label2}\n" + + example.derivations2.render_strings_for_report.each do |str| + io << " #{str}\n" + end + end + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_verbose_info(io, state, states) + report_direct_read_sets(io, state, states) + report_reads_relation(io, state, states) + report_read_sets(io, state, states) + report_includes_relation(io, state, states) + report_lookback_relation(io, state, states) + report_follow_sets(io, state, states) + report_look_ahead_sets(io, state, states) + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_direct_read_sets(io, state, states) + io << " [Direct Read sets]\n" + direct_read_sets = states.direct_read_sets + + state.nterm_transitions.each do |goto| + terms = direct_read_sets[goto] + next unless terms && !terms.empty? + + str = terms.map { |sym| sym.id.s_value }.join(", ") + io << " read #{goto.next_sym.id.s_value} shift #{str}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_reads_relation(io, state, states) + io << " [Reads Relation]\n" + + state.nterm_transitions.each do |goto| + goto2 = states.reads_relation[goto] + next unless goto2 + + goto2.each do |goto2| + io << " (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_read_sets(io, state, states) + io << " [Read sets]\n" + read_sets = states.read_sets + + state.nterm_transitions.each do |goto| + terms = read_sets[goto] + next unless terms && !terms.empty? + + terms.each do |sym| + io << " #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_includes_relation(io, state, states) + io << " [Includes Relation]\n" + + state.nterm_transitions.each do |goto| + gotos = states.includes_relation[goto] + next unless gotos + + gotos.each do |goto2| + io << " (State #{state.id}, #{goto.next_sym.id.s_value}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_lookback_relation(io, state, states) + io << " [Lookback Relation]\n" + + states.rules.each do |rule| + gotos = states.lookback_relation.dig(state.id, rule.id) + next unless gotos + + gotos.each do |goto2| + io << " (Rule: #{rule.display_name}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_follow_sets(io, state, states) + io << " [Follow sets]\n" + follow_sets = states.follow_sets + + state.nterm_transitions.each do |goto| + terms = follow_sets[goto] + next unless terms + + terms.each do |sym| + io << " #{goto.next_sym.id.s_value} -> #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_look_ahead_sets(io, state, states) + io << " [Look-Ahead Sets]\n" + look_ahead_rules = [] #: Array[[Lrama::Grammar::Rule, Array[Lrama::Grammar::Symbol]]] + + states.rules.each do |rule| + syms = states.la.dig(state.id, rule.id) + next unless syms + + look_ahead_rules << [rule, syms] + end + + return if look_ahead_rules.empty? + + max_len = look_ahead_rules.flat_map { |_, syms| syms.map { |s| s.id.s_value.length } }.max + + look_ahead_rules.each do |rule, syms| + syms.each do |sym| + io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/terms.rb b/tool/lrama/lib/lrama/reporter/terms.rb new file mode 100644 index 0000000000..f72d8b1a1a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/terms.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Terms + # @rbs (?terms: bool, **bool _) -> void + def initialize(terms: false, **_) + @terms = terms + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @terms + + look_aheads = states.states.each do |state| + state.reduces.flat_map do |reduce| + reduce.look_ahead unless reduce.look_ahead.nil? + end + end + + next_terms = states.states.flat_map do |state| + state.term_transitions.map {|shift| shift.next_sym } + end + + unused_symbols = states.terms.reject do |term| + (look_aheads + next_terms).include?(term) + end + + io << states.terms.count << " Terms\n\n" + + io << states.nterms.count << " Non-Terminals\n\n" + + unless unused_symbols.empty? + io << "#{unused_symbols.count} Unused Terms\n\n" + unused_symbols.each_with_index do |term, index| + io << sprintf("%5d %s", index, term.id.s_value) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb new file mode 100644 index 0000000000..50912e094e --- /dev/null +++ b/tool/lrama/lib/lrama/state.rb @@ -0,0 +1,534 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "state/action" +require_relative "state/inadequacy_annotation" +require_relative "state/item" +require_relative "state/reduce_reduce_conflict" +require_relative "state/resolved_conflict" +require_relative "state/shift_reduce_conflict" + +module Lrama + class State + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type conflict = State::ShiftReduceConflict | State::ReduceReduceConflict + # type transition = Action::Shift | Action::Goto + # type lookahead_set = Hash[Item, Array[Grammar::Symbol]] + # + # @id: Integer + # @accessing_symbol: Grammar::Symbol + # @kernels: Array[Item] + # @items: Array[Item] + # @items_to_state: Hash[Array[Item], State] + # @conflicts: Array[conflict] + # @resolved_conflicts: Array[ResolvedConflict] + # @default_reduction_rule: Grammar::Rule? + # @closure: Array[Item] + # @nterm_transitions: Array[Action::Goto] + # @term_transitions: Array[Action::Shift] + # @transitions: Array[transition] + # @internal_dependencies: Hash[Action::Goto, Array[Action::Goto]] + # @successor_dependencies: Hash[Action::Goto, Array[Action::Goto]] + + attr_reader :id #: Integer + attr_reader :accessing_symbol #: Grammar::Symbol + attr_reader :kernels #: Array[Item] + attr_reader :conflicts #: Array[conflict] + attr_reader :resolved_conflicts #: Array[ResolvedConflict] + attr_reader :default_reduction_rule #: Grammar::Rule? + attr_reader :closure #: Array[Item] + attr_reader :items #: Array[Item] + attr_reader :annotation_list #: Array[InadequacyAnnotation] + attr_reader :predecessors #: Array[State] + attr_reader :items_to_state #: Hash[Array[Item], State] + attr_reader :lane_items #: Hash[State, Array[[Item, Item]]] + + attr_accessor :_transitions #: Array[[Grammar::Symbol, Array[Item]]] + attr_accessor :reduces #: Array[Action::Reduce] + attr_accessor :ielr_isocores #: Array[State] + attr_accessor :lalr_isocore #: State + attr_accessor :lookaheads_recomputed #: bool + attr_accessor :follow_kernel_items #: Hash[Action::Goto, Hash[Item, bool]] + attr_accessor :always_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor :goto_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + + # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void + def initialize(id, accessing_symbol, kernels) + @id = id + @accessing_symbol = accessing_symbol + @kernels = kernels.freeze + @items = @kernels + # Manage relationships between items to state + # to resolve next state + @items_to_state = {} + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + @predecessors = [] + @lalr_isocore = self + @ielr_isocores = [self] + @internal_dependencies = {} + @successor_dependencies = {} + @annotation_list = [] + @lookaheads_recomputed = false + @follow_kernel_items = {} + @always_follows = {} + @goto_follows = {} + @lhs_contributions = {} + @lane_items = {} + end + + # @rbs (State other) -> bool + def ==(other) + self.id == other.id + end + + # @rbs (Array[Item] closure) -> void + def closure=(closure) + @closure = closure + @items = @kernels + @closure + end + + # @rbs () -> Array[Action::Reduce] + def non_default_reduces + reduces.reject do |reduce| + reduce.rule == @default_reduction_rule + end + end + + # @rbs () -> void + def compute_transitions_and_reduces + _transitions = {} + @_lane_items ||= {} + reduces = [] + items.each do |item| + # TODO: Consider what should be pushed + if item.end_of_rule? + reduces << Action::Reduce.new(item) + else + key = item.next_sym + _transitions[key] ||= [] + @_lane_items[key] ||= [] + next_item = item.new_by_next_position + _transitions[key] << next_item + @_lane_items[key] << [item, next_item] + end + end + + # It seems Bison 3.8.2 iterates transitions order by symbol number + transitions = _transitions.sort_by do |next_sym, to_items| + next_sym.number + end + + self._transitions = transitions.freeze + self.reduces = reduces.freeze + end + + # @rbs (Grammar::Symbol next_sym, State next_state) -> void + def set_lane_items(next_sym, next_state) + @lane_items[next_state] = @_lane_items[next_sym] + end + + # @rbs (Array[Item] items, State next_state) -> void + def set_items_to_state(items, next_state) + @items_to_state[items] = next_state + end + + # @rbs (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void + def set_look_ahead(rule, look_ahead) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead = look_ahead + end + + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void + def set_look_ahead_sources(rule, sources) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead_sources = sources + end + + # @rbs () -> Array[Action::Goto] + def nterm_transitions # steep:ignore + @nterm_transitions ||= transitions.select {|transition| transition.is_a?(Action::Goto) } + end + + # @rbs () -> Array[Action::Shift] + def term_transitions # steep:ignore + @term_transitions ||= transitions.select {|transition| transition.is_a?(Action::Shift) } + end + + # @rbs () -> Array[transition] + def transitions + @transitions ||= _transitions.map do |next_sym, to_items| + if next_sym.term? + Action::Shift.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + else + Action::Goto.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + end + end + end + + # @rbs (transition transition, State next_state) -> void + def update_transition(transition, next_state) + set_items_to_state(transition.to_items, next_state) + next_state.append_predecessor(self) + update_transitions_caches(transition) + end + + # @rbs () -> void + def update_transitions_caches(transition) + new_transition = + if transition.next_sym.term? + Action::Shift.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + else + Action::Goto.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + end + + @transitions.delete(transition) + @transitions << new_transition + @nterm_transitions = nil + @term_transitions = nil + + @follow_kernel_items[new_transition] = @follow_kernel_items.delete(transition) + @always_follows[new_transition] = @always_follows.delete(transition) + end + + # @rbs () -> Array[Action::Shift] + def selected_term_transitions + term_transitions.reject do |shift| + shift.not_selected + end + end + + # Move to next state by sym + # + # @rbs (Grammar::Symbol sym) -> State + def transition(sym) + result = nil + + if sym.term? + result = term_transitions.find {|shift| shift.next_sym == sym }.to_state + else + result = nterm_transitions.find {|goto| goto.next_sym == sym }.to_state + end + + raise "Can not transit by #{sym} #{self}" if result.nil? + + result + end + + # @rbs (Item item) -> Action::Reduce + def find_reduce_by_item!(item) + reduces.find do |r| + r.item == item + end || (raise "reduce is not found. #{item}") + end + + # @rbs (Grammar::Rule default_reduction_rule) -> void + def default_reduction_rule=(default_reduction_rule) + @default_reduction_rule = default_reduction_rule + + reduces.each do |r| + if r.rule == default_reduction_rule + r.default_reduction = true + end + end + end + + # @rbs () -> bool + def has_conflicts? + !@conflicts.empty? + end + + # @rbs () -> Array[conflict] + def sr_conflicts + @conflicts.select do |conflict| + conflict.type == :shift_reduce + end + end + + # @rbs () -> Array[conflict] + def rr_conflicts + @conflicts.select do |conflict| + conflict.type == :reduce_reduce + end + end + + # Clear information related to conflicts. + # IELR computation re-calculates conflicts and default reduction of states + # after LALR computation. + # Call this method before IELR computation to avoid duplicated conflicts information + # is stored. + # + # @rbs () -> void + def clear_conflicts + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + + term_transitions.each(&:clear_conflicts) + reduces.each(&:clear_conflicts) + end + + # @rbs () -> bool + def split_state? + @lalr_isocore != self + end + + # Definition 3.40 (propagate_lookaheads) + # + # @rbs (State next_state) -> lookahead_set + def propagate_lookaheads(next_state) + next_state.kernels.map {|next_kernel| + lookahead_sets = + if next_kernel.position > 1 + kernel = kernels.find {|k| k.predecessor_item_of?(next_kernel) } + item_lookahead_set[kernel] + else + goto_follow_set(next_kernel.lhs) + end + + [next_kernel, lookahead_sets & next_state.lookahead_set_filters[next_kernel]] + }.to_h + end + + # Definition 3.43 (is_compatible) + # + # @rbs (lookahead_set filtered_lookahead) -> bool + def is_compatible?(filtered_lookahead) + !lookaheads_recomputed || + @lalr_isocore.annotation_list.all? {|annotation| + a = annotation.dominant_contribution(item_lookahead_set) + b = annotation.dominant_contribution(filtered_lookahead) + a.nil? || b.nil? || a == b + } + end + + # Definition 3.38 (lookahead_set_filters) + # + # @rbs () -> lookahead_set + def lookahead_set_filters + @lookahead_set_filters ||= kernels.map {|kernel| + [kernel, @lalr_isocore.annotation_list.select {|annotation| annotation.contributed?(kernel) }.map(&:token)] + }.to_h + end + + # Definition 3.27 (inadequacy_lists) + # + # @rbs () -> Hash[Grammar::Symbol, Array[Action::Shift | Action::Reduce]] + def inadequacy_list + return @inadequacy_list if @inadequacy_list + + inadequacy_list = {} + + term_transitions.each do |shift| + inadequacy_list[shift.next_sym] ||= [] + inadequacy_list[shift.next_sym] << shift.dup + end + reduces.each do |reduce| + next if reduce.look_ahead.nil? + + reduce.look_ahead.each do |token| + inadequacy_list[token] ||= [] + inadequacy_list[token] << reduce.dup + end + end + + @inadequacy_list = inadequacy_list.select {|token, actions| actions.size > 1 } + end + + # Definition 3.30 (annotate_manifestation) + # + # @rbs () -> void + def annotate_manifestation + inadequacy_list.each {|token, actions| + contribution_matrix = actions.map {|action| + if action.is_a?(Action::Shift) + [action, nil] + else + [action, action.rule.empty_rule? ? lhs_contributions(action.rule.lhs, token) : kernels.map {|k| [k, k.rule == action.item.rule && k.end_of_rule?] }.to_h] + end + }.to_h + @annotation_list << InadequacyAnnotation.new(self, token, actions, contribution_matrix) + } + end + + # Definition 3.32 (annotate_predecessor) + # + # @rbs (State predecessor) -> void + def annotate_predecessor(predecessor) + propagating_list = annotation_list.map {|annotation| + contribution_matrix = annotation.contribution_matrix.map {|action, contributions| + if contributions.nil? + [action, nil] + elsif first_kernels.any? {|kernel| contributions[kernel] && predecessor.lhs_contributions(kernel.lhs, annotation.token).empty? } + [action, nil] + else + cs = predecessor.lane_items[self].map {|pred_kernel, kernel| + c = contributions[kernel] && ( + (kernel.position > 1 && predecessor.item_lookahead_set[pred_kernel].include?(annotation.token)) || + (kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, annotation.token)[pred_kernel]) + ) + [pred_kernel, c] + }.to_h + [action, cs] + end + }.to_h + + # Observation 3.33 (Simple Split-Stable Dominance) + # + # If all of contributions in the contribution_matrix are + # always contribution or never contribution, we can stop annotate propagations + # to the predecessor state. + next nil if contribution_matrix.all? {|_, contributions| contributions.nil? || contributions.all? {|_, contributed| !contributed } } + + InadequacyAnnotation.new(annotation.state, annotation.token, annotation.actions, contribution_matrix) + }.compact + predecessor.append_annotation_list(propagating_list) + end + + # @rbs () -> Array[Item] + def first_kernels + @first_kernels ||= kernels.select {|kernel| kernel.position == 1 } + end + + # @rbs (Array[InadequacyAnnotation] propagating_list) -> void + def append_annotation_list(propagating_list) + annotation_list.each do |annotation| + merging_list = propagating_list.select {|a| a.state == annotation.state && a.token == annotation.token && a.actions == annotation.actions } + annotation.merge_matrix(merging_list.map(&:contribution_matrix)) + propagating_list -= merging_list + end + + @annotation_list += propagating_list + end + + # Definition 3.31 (compute_lhs_contributions) + # + # @rbs (Grammar::Symbol sym, Grammar::Symbol token) -> (nil | Hash[Item, bool]) + def lhs_contributions(sym, token) + return @lhs_contributions[sym][token] unless @lhs_contributions.dig(sym, token).nil? + + transition = nterm_transitions.find {|goto| goto.next_sym == sym } + @lhs_contributions[sym] ||= {} + @lhs_contributions[sym][token] = + if always_follows[transition].include?(token) + {} + else + kernels.map {|kernel| [kernel, follow_kernel_items[transition][kernel] && item_lookahead_set[kernel].include?(token)] }.to_h + end + end + + # Definition 3.26 (item_lookahead_sets) + # + # @rbs () -> lookahead_set + def item_lookahead_set + return @item_lookahead_set if @item_lookahead_set + + @item_lookahead_set = kernels.map {|k| [k, []] }.to_h + @item_lookahead_set = kernels.map {|kernel| + value = + if kernel.lhs.accept_symbol? + [] + elsif kernel.position > 1 + prev_items = predecessors_with_item(kernel) + prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms } + elsif kernel.position == 1 + prev_state = @predecessors.find {|p| p.transitions.any? {|transition| transition.next_sym == kernel.lhs } } + goto = prev_state.nterm_transitions.find {|goto| goto.next_sym == kernel.lhs } + prev_state.goto_follows[goto] + end + [kernel, value] + }.to_h + end + + # @rbs (lookahead_set k) -> void + def item_lookahead_set=(k) + @item_lookahead_set = k + end + + # @rbs (Item item) -> Array[[State, Item]] + def predecessors_with_item(item) + result = [] + @predecessors.each do |pre| + pre.items.each do |i| + result << [pre, i] if i.predecessor_item_of?(item) + end + end + result + end + + # @rbs (State prev_state) -> void + def append_predecessor(prev_state) + @predecessors << prev_state + @predecessors.uniq! + end + + # Definition 3.39 (compute_goto_follow_set) + # + # @rbs (Grammar::Symbol nterm_token) -> Array[Grammar::Symbol] + def goto_follow_set(nterm_token) + return [] if nterm_token.accept_symbol? + goto = @lalr_isocore.nterm_transitions.find {|g| g.next_sym == nterm_token } + + @kernels + .select {|kernel| @lalr_isocore.follow_kernel_items[goto][kernel] } + .map {|kernel| item_lookahead_set[kernel] } + .reduce(@lalr_isocore.always_follows[goto]) {|result, terms| result |= terms } + end + + # Definition 3.8 (Goto Follows Internal Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def internal_dependencies(goto) + return @internal_dependencies[goto] if @internal_dependencies[goto] + + syms = @items.select {|i| + i.next_sym == goto.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0 + }.map(&:lhs).uniq + @internal_dependencies[goto] = nterm_transitions.select {|goto2| syms.include?(goto2.next_sym) } + end + + # Definition 3.5 (Goto Follows Successor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def successor_dependencies(goto) + return @successor_dependencies[goto] if @successor_dependencies[goto] + + @successor_dependencies[goto] = goto.to_state.nterm_transitions.select {|next_goto| next_goto.next_sym.nullable } + end + + # Definition 3.9 (Goto Follows Predecessor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def predecessor_dependencies(goto) + state_items = [] + @kernels.select {|kernel| + kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) + }.each do |item| + queue = predecessors_with_item(item) + until queue.empty? + st, i = queue.pop + if i.position == 0 + state_items << [st, i] + else + st.predecessors_with_item(i).each {|v| queue << v } + end + end + end + + state_items.map {|state, item| + state.nterm_transitions.find {|goto2| goto2.next_sym == item.lhs } + } + end + end +end diff --git a/tool/lrama/lib/lrama/state/action.rb b/tool/lrama/lib/lrama/state/action.rb new file mode 100644 index 0000000000..791685fc23 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "action/goto" +require_relative "action/reduce" +require_relative "action/shift" diff --git a/tool/lrama/lib/lrama/state/action/goto.rb b/tool/lrama/lib/lrama/state/action/goto.rb new file mode 100644 index 0000000000..4c2c82afdc --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/goto.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Goto + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/reduce.rb b/tool/lrama/lib/lrama/state/action/reduce.rb new file mode 100644 index 0000000000..9678ab0a98 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/reduce.rb @@ -0,0 +1,71 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Reduce + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @item: Item + # @look_ahead: Array[Grammar::Symbol]? + # @look_ahead_sources: Hash[Grammar::Symbol, Array[Action::Goto]]? + # @not_selected_symbols: Array[Grammar::Symbol] + + attr_reader :item #: Item + attr_reader :look_ahead #: Array[Grammar::Symbol]? + attr_reader :look_ahead_sources #: Hash[Grammar::Symbol, Array[Action::Goto]]? + attr_reader :not_selected_symbols #: Array[Grammar::Symbol] + + # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html + attr_accessor :default_reduction #: bool + + # @rbs (Item item) -> void + def initialize(item) + @item = item + @look_ahead = nil + @look_ahead_sources = nil + @not_selected_symbols = [] + end + + # @rbs () -> Grammar::Rule + def rule + @item.rule + end + + # @rbs (Array[Grammar::Symbol] look_ahead) -> Array[Grammar::Symbol] + def look_ahead=(look_ahead) + @look_ahead = look_ahead.freeze + end + + # @rbs (Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> Hash[Grammar::Symbol, Array[Action::Goto]] + def look_ahead_sources=(sources) + @look_ahead_sources = sources.freeze + end + + # @rbs (Grammar::Symbol sym) -> Array[Grammar::Symbol] + def add_not_selected_symbol(sym) + @not_selected_symbols << sym + end + + # @rbs () -> (::Array[Grammar::Symbol?]) + def selected_look_ahead + if look_ahead + look_ahead - @not_selected_symbols + else + [] + end + end + + # @rbs () -> void + def clear_conflicts + @not_selected_symbols = [] + @default_reduction = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/shift.rb b/tool/lrama/lib/lrama/state/action/shift.rb new file mode 100644 index 0000000000..52d9f8c4f0 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/shift.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Shift + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + attr_accessor :not_selected #: bool + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + + # @rbs () -> void + def clear_conflicts + @not_selected = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/inadequacy_annotation.rb b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb new file mode 100644 index 0000000000..3654fa4607 --- /dev/null +++ b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb @@ -0,0 +1,140 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class InadequacyAnnotation + # @rbs! + # type action = Action::Shift | Action::Reduce + + attr_accessor :state #: State + attr_accessor :token #: Grammar::Symbol + attr_accessor :actions #: Array[action] + attr_accessor :contribution_matrix #: Hash[action, Hash[Item, bool]] + + # @rbs (State state, Grammar::Symbol token, Array[action] actions, Hash[action, Hash[Item, bool]] contribution_matrix) -> void + def initialize(state, token, actions, contribution_matrix) + @state = state + @token = token + @actions = actions + @contribution_matrix = contribution_matrix + end + + # @rbs (Item item) -> bool + def contributed?(item) + @contribution_matrix.any? {|action, contributions| !contributions.nil? && contributions[item] } + end + + # @rbs (Array[Hash[action, Hash[Item, bool]]] another_matrixes) -> void + def merge_matrix(another_matrixes) + another_matrixes.each do |another_matrix| + @contribution_matrix.merge!(another_matrix) {|action, contributions, another_contributions| + next contributions if another_contributions.nil? + next another_contributions if contributions.nil? + + contributions.merge!(another_contributions) {|_, contributed, another_contributed| contributed || another_contributed } + } + end + end + + # Definition 3.42 (dominant_contribution) + # + # @rbs (State::lookahead_set lookaheads) -> Array[action]? + def dominant_contribution(lookaheads) + actions = @actions.select {|action| + contribution_matrix[action].nil? || contribution_matrix[action].any? {|item, contributed| contributed && lookaheads[item].include?(@token) } + } + return nil if actions.empty? + + resolve_conflict(actions) + end + + # @rbs (Array[action] actions) -> Array[action] + def resolve_conflict(actions) + # @type var shifts: Array[Action::Shift] + # @type var reduces: Array[Action::Reduce] + shifts = actions.select {|action| action.is_a?(Action::Shift)} + reduces = actions.select {|action| action.is_a?(Action::Reduce) } + + shifts.each do |shift| + reduces.each do |reduce| + sym = shift.next_sym + + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + actions.delete(shift) + next + when shift_prec > reduce_prec + # Shift is selected + actions.delete(reduce) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence&.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + next + when :right + # Shift is selected + actions.delete(reduce) + next + when :left + # Reduce is selected + actions.delete(shift) + next + when :nonassoc + # Can not resolve + # + # nonassoc creates "run-time" error, precedence creates "compile-time" error. + # Then omit both the shift and reduce. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + actions.delete(shift) + actions.delete(reduce) + else + raise "Unknown precedence type. #{sym}" + end + end + end + + actions + end + + # @rbs () -> String + def to_s + "State: #{@state.id}, Token: #{@token.id.s_value}, Actions: #{actions_to_s}, Contributions: #{contribution_matrix_to_s}" + end + + private + + # @rbs () -> String + def actions_to_s + '[' + @actions.map {|action| + if action.is_a?(Action::Shift) || action.is_a?(Action::Goto) + action.class.name + elsif action.is_a?(Action::Reduce) + "#{action.class.name}: (#{action.item})" + end + }.join(', ') + ']' + end + + # @rbs () -> String + def contribution_matrix_to_s + '[' + @contribution_matrix.map {|action, contributions| + "#{(action.is_a?(Action::Shift) || action.is_a?(Action::Goto)) ? action.class.name : "#{action.class.name}: (#{action.item})"}: " + contributions&.transform_keys(&:to_s).to_s + }.join(', ') + ']' + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/item.rb b/tool/lrama/lib/lrama/state/item.rb new file mode 100644 index 0000000000..3ecdd70b76 --- /dev/null +++ b/tool/lrama/lib/lrama/state/item.rb @@ -0,0 +1,120 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# TODO: Validate position is not over rule rhs + +require "forwardable" + +module Lrama + class State + class Item < Struct.new(:rule, :position, keyword_init: true) + # @rbs! + # include Grammar::Rule::_DelegatedMethods + # + # attr_accessor rule: Grammar::Rule + # attr_accessor position: Integer + # + # def initialize: (?rule: Grammar::Rule, ?position: Integer) -> void + + extend Forwardable + + def_delegators "rule", :lhs, :rhs + + # Optimization for States#setup_state + # + # @rbs () -> Integer + def hash + [rule_id, position].hash + end + + # @rbs () -> Integer + def rule_id + rule.id + end + + # @rbs () -> bool + def empty_rule? + rule.empty_rule? + end + + # @rbs () -> Integer + def number_of_rest_symbols + @number_of_rest_symbols ||= rhs.count - position + end + + # @rbs () -> Grammar::Symbol + def next_sym + rhs[position] + end + + # @rbs () -> Grammar::Symbol + def next_next_sym + @next_next_sym ||= rhs[position + 1] + end + + # @rbs () -> Grammar::Symbol + def previous_sym + rhs[position - 1] + end + + # @rbs () -> bool + def end_of_rule? + rhs.count == position + end + + # @rbs () -> bool + def beginning_of_rule? + position == 0 + end + + # @rbs () -> bool + def start_item? + rule.initial_rule? && beginning_of_rule? + end + + # @rbs () -> State::Item + def new_by_next_position + Item.new(rule: rule, position: position + 1) + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_before_dot # steep:ignore + rhs[0...position] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_dot # steep:ignore + rhs[position..-1] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_transition # steep:ignore + rhs[position+1..-1] + end + + # @rbs () -> ::String + def to_s + "#{lhs.id.s_value}: #{display_name}" + end + + # @rbs () -> ::String + def display_name + r = rhs.map(&:display_name).insert(position, "•").join(" ") + "#{r} (rule #{rule_id})" + end + + # Right after position + # + # @rbs () -> ::String + def display_rest + r = symbols_after_dot.map(&:display_name).join(" ") + ". #{r} (rule #{rule_id})" + end + + # @rbs (State::Item other_item) -> bool + def predecessor_item_of?(other_item) + rule == other_item.rule && position == other_item.position - 1 + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb new file mode 100644 index 0000000000..55ecad40bd --- /dev/null +++ b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class ReduceReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :reduce1 #: State::Action::Reduce + attr_reader :reduce2 #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], reduce1: State::Action::Reduce, reduce2: State::Action::Reduce) -> void + def initialize(symbols:, reduce1:, reduce2:) + @symbols = symbols + @reduce1 = reduce1 + @reduce2 = reduce2 + end + + # @rbs () -> :reduce_reduce + def type + :reduce_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/resolved_conflict.rb b/tool/lrama/lib/lrama/state/resolved_conflict.rb new file mode 100644 index 0000000000..014533c233 --- /dev/null +++ b/tool/lrama/lib/lrama/state/resolved_conflict.rb @@ -0,0 +1,65 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + # * state: A state on which the conflct is resolved + # * symbol: A symbol under discussion + # * reduce: A reduce under discussion + # * which: For which a conflict is resolved. :shift, :reduce or :error (for nonassociative) + # * resolved_by_precedence: If the conflict is resolved by precedence definition or not + class ResolvedConflict + # @rbs! + # type which_enum = :reduce | :shift | :error + + attr_reader :state #: State + attr_reader :symbol #: Grammar::Symbol + attr_reader :reduce #: State::Action::Reduce + attr_reader :which #: which_enum + attr_reader :resolved_by_precedence #: bool + + # @rbs (state: State, symbol: Grammar::Symbol, reduce: State::Action::Reduce, which: which_enum, resolved_by_precedence: bool) -> void + def initialize(state:, symbol:, reduce:, which:, resolved_by_precedence:) + @state = state + @symbol = symbol + @reduce = reduce + @which = which + @resolved_by_precedence = resolved_by_precedence + end + + # @rbs () -> (::String | bot) + def report_message + "Conflict between rule #{reduce.rule.id} and token #{symbol.display_name} #{how_resolved}." + end + + # @rbs () -> (::String | bot) + def report_precedences_message + "Conflict between reduce by \"#{reduce.rule.display_name}\" and shift #{symbol.display_name} #{how_resolved}." + end + + private + + # @rbs () -> (::String | bot) + def how_resolved + s = symbol.display_name + r = reduce.rule.precedence_sym&.display_name + case + when which == :shift && resolved_by_precedence + msg = "resolved as #{which} (%right #{s})" + when which == :shift + msg = "resolved as #{which} (#{r} < #{s})" + when which == :reduce && resolved_by_precedence + msg = "resolved as #{which} (%left #{s})" + when which == :reduce + msg = "resolved as #{which} (#{s} < #{r})" + when which == :error + msg = "resolved as an #{which} (%nonassoc #{s})" + else + raise "Unknown direction. #{self}" + end + + msg + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb new file mode 100644 index 0000000000..548f2de614 --- /dev/null +++ b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class ShiftReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :shift #: State::Action::Shift + attr_reader :reduce #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], shift: State::Action::Shift, reduce: State::Action::Reduce) -> void + def initialize(symbols:, shift:, reduce:) + @symbols = symbols + @shift = shift + @reduce = reduce + end + + # @rbs () -> :shift_reduce + def type + :shift_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb new file mode 100644 index 0000000000..ddce627df4 --- /dev/null +++ b/tool/lrama/lib/lrama/states.rb @@ -0,0 +1,867 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "tracer/duration" +require_relative "state/item" + +module Lrama + # States is passed to a template file + # + # "Efficient Computation of LALR(1) Look-Ahead Sets" + # https://dl.acm.org/doi/pdf/10.1145/69622.357187 + class States + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type state_id = Integer + # type rule_id = Integer + # + # include Grammar::_DelegatedMethods + # + # @grammar: Grammar + # @tracer: Tracer + # @states: Array[State] + # @direct_read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @reads_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @includes_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @lookback_relation: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + # @follow_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @la: Hash[state_id, Hash[rule_id, Bitmap::bitmap]] + + extend Forwardable + include Lrama::Tracer::Duration + + def_delegators "@grammar", :symbols, :terms, :nterms, :rules, :precedences, + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined? + + attr_reader :states #: Array[State] + attr_reader :reads_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :includes_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :lookback_relation #: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + + # @rbs (Grammar grammar, Tracer tracer) -> void + def initialize(grammar, tracer) + @grammar = grammar + @tracer = tracer + + @states = [] + + # `DR(p, A) = {t ∈ T | p -(A)-> r -(t)-> }` + # where p is state, A is nterm, t is term. + # + # `@direct_read_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @direct_read_sets = {} + + # Reads relation on nonterminal transitions (pair of state and nterm) + # `(p, A) reads (r, C) iff p -(A)-> r -(C)-> and C =>* ε` + # where p, r are state, A, C are nterm. + # + # `@reads_relation` is a hash whose + # key is goto, + # value is array of goto. + @reads_relation = {} + + # `Read(p, A) =s DR(p, A) ∪ ∪{Read(r, C) | (p, A) reads (r, C)}` + # + # `@read_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @read_sets = {} + + # `(p, A) includes (p', B) iff B -> βAγ, γ =>* ε, p' -(β)-> p` + # where p, p' are state, A, B are nterm, β, γ is sequence of symbol. + # + # `@includes_relation` is a hash whose + # key is goto, + # value is array of goto. + @includes_relation = {} + + # `(q, A -> ω) lookback (p, A) iff p -(ω)-> q` + # where p, q are state, A -> ω is rule, A is nterm, ω is sequence of symbol. + # + # `@lookback_relation` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is array of goto. + @lookback_relation = {} + + # `Follow(p, A) =s Read(p, A) ∪ ∪{Follow(p', B) | (p, A) includes (p', B)}` + # + # `@follow_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @follow_sets = {} + + # `LA(q, A -> ω) = ∪{Follow(p, A) | (q, A -> ω) lookback (p, A)` + # + # `@la` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is bitmap of term. + @la = {} + end + + # @rbs () -> void + def compute + report_duration(:compute_lr0_states) { compute_lr0_states } + + # Look Ahead Sets + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + + # Conflicts + report_duration(:compute_conflicts) { compute_conflicts(:lalr) } + + report_duration(:compute_default_reduction) { compute_default_reduction } + end + + # @rbs () -> void + def compute_ielr + # Preparation + report_duration(:clear_conflicts) { clear_conflicts } + # Phase 1 + report_duration(:compute_predecessors) { compute_predecessors } + report_duration(:compute_follow_kernel_items) { compute_follow_kernel_items } + report_duration(:compute_always_follows) { compute_always_follows } + report_duration(:compute_goto_follows) { compute_goto_follows } + # Phase 2 + report_duration(:compute_inadequacy_annotations) { compute_inadequacy_annotations } + # Phase 3 + report_duration(:split_states) { split_states } + # Phase 4 + report_duration(:clear_look_ahead_sets) { clear_look_ahead_sets } + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + # Phase 5 + report_duration(:compute_conflicts) { compute_conflicts(:ielr) } + report_duration(:compute_default_reduction) { compute_default_reduction } + end + + # @rbs () -> Integer + def states_count + @states.count + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def direct_read_sets + @_direct_read_sets ||= @direct_read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def read_sets + @_read_sets ||= @read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def follow_sets + @_follow_sets ||= @follow_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[state_id, Hash[rule_id, Array[Grammar::Symbol]]] + def la + @_la ||= @la.transform_values do |second_hash| + second_hash.transform_values do |v| + bitmap_to_terms(v) + end + end + end + + # @rbs () -> Integer + def sr_conflicts_count + @sr_conflicts_count ||= @states.flat_map(&:sr_conflicts).count + end + + # @rbs () -> Integer + def rr_conflicts_count + @rr_conflicts_count ||= @states.flat_map(&:rr_conflicts).count + end + + # @rbs (Logger logger) -> void + def validate!(logger) + validate_conflicts_within_threshold!(logger) + end + + def compute_la_sources_for_conflicted_states + reflexive = {} + @states.each do |state| + state.nterm_transitions.each do |goto| + reflexive[goto] = [goto] + end + end + + # compute_read_sets + read_sets = Digraph.new(nterm_transitions, @reads_relation, reflexive).compute + # compute_follow_sets + follow_sets = Digraph.new(nterm_transitions, @includes_relation, read_sets).compute + + @states.select(&:has_conflicts?).each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + sources = {} + + ary.each do |goto| + source = follow_sets[goto] + + next unless source + + source.each do |goto2| + tokens = direct_read_sets[goto2] + tokens.each do |token| + sources[token] ||= [] + sources[token] |= [goto2] + end + end + end + + state.set_look_ahead_sources(rule, sources) + end + end + end + + private + + # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] + def create_state(accessing_symbol, kernels, states_created) + # A item can appear in some states, + # so need to use `kernels` (not `kernels.first`) as a key. + # + # For example... + # + # %% + # program: '+' strings_1 + # | '-' strings_2 + # ; + # + # strings_1: string_1 + # ; + # + # strings_2: string_1 + # | string_2 + # ; + # + # string_1: string + # ; + # + # string_2: string '+' + # ; + # + # string: tSTRING + # ; + # %% + # + # For these grammar, there are 2 states + # + # State A + # string_1: string • + # + # State B + # string_1: string • + # string_2: string • '+' + # + return [states_created[kernels], false] if states_created[kernels] + + state = State.new(@states.count, accessing_symbol, kernels) + @states << state + states_created[kernels] = state + + return [state, true] + end + + # @rbs (State state) -> void + def setup_state(state) + # closure + closure = [] + queued = {} + items = state.kernels.dup + + items.each do |item| + queued[item.rule_id] = true if item.position == 0 + end + + while (item = items.shift) do + if (sym = item.next_sym) && sym.nterm? + @grammar.find_rules_by_symbol!(sym).each do |rule| + next if queued[rule.id] + i = State::Item.new(rule: rule, position: 0) + closure << i + items << i + queued[i.rule_id] = true + end + end + end + + state.closure = closure.sort_by {|i| i.rule.id } + + # Trace + @tracer.trace_closure(state) + + # shift & reduce + state.compute_transitions_and_reduces + end + + # @rbs (Array[State] states, State state) -> void + def enqueue_state(states, state) + # Trace + @tracer.trace_state_list_append(@states.count, state) + + states << state + end + + # @rbs () -> void + def compute_lr0_states + # State queue + states = [] + states_created = {} + + state, _ = create_state(symbols.first, [State::Item.new(rule: @grammar.rules.first, position: 0)], states_created) + enqueue_state(states, state) + + while (state = states.shift) do + # Trace + @tracer.trace_state(state) + + setup_state(state) + + # `State#transitions` can not be used here + # because `items_to_state` of the `state` is not set yet. + state._transitions.each do |next_sym, to_items| + new_state, created = create_state(next_sym, to_items, states_created) + state.set_items_to_state(to_items, new_state) + state.set_lane_items(next_sym, new_state) + enqueue_state(states, new_state) if created + end + end + end + + # @rbs () -> Array[State::Action::Goto] + def nterm_transitions + a = [] + + @states.each do |state| + state.nterm_transitions.each do |goto| + a << goto + end + end + + a + end + + # @rbs () -> void + def compute_look_ahead_sets + report_duration(:compute_direct_read_sets) { compute_direct_read_sets } + report_duration(:compute_reads_relation) { compute_reads_relation } + report_duration(:compute_read_sets) { compute_read_sets } + report_duration(:compute_includes_relation) { compute_includes_relation } + report_duration(:compute_lookback_relation) { compute_lookback_relation } + report_duration(:compute_follow_sets) { compute_follow_sets } + report_duration(:compute_la) { compute_la } + end + + # @rbs () -> void + def compute_direct_read_sets + @states.each do |state| + state.nterm_transitions.each do |goto| + ary = goto.to_state.term_transitions.map do |shift| + shift.next_sym.number + end + + @direct_read_sets[goto] = Bitmap.from_array(ary) + end + end + end + + # @rbs () -> void + def compute_reads_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + goto.to_state.nterm_transitions.each do |goto2| + nterm2 = goto2.next_sym + if nterm2.nullable + @reads_relation[goto] ||= [] + @reads_relation[goto] << goto2 + end + end + end + end + end + + # @rbs () -> void + def compute_read_sets + @read_sets = Digraph.new(nterm_transitions, @reads_relation, @direct_read_sets).compute + end + + # Execute transition of state by symbols + # then return final state. + # + # @rbs (State state, Array[Grammar::Symbol] symbols) -> State + def transition(state, symbols) + symbols.each do |sym| + state = state.transition(sym) + end + + state + end + + # @rbs () -> void + def compute_includes_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + nterm = goto.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + i = rule.rhs.count - 1 + + while (i > -1) do + sym = rule.rhs[i] + + break if sym.term? + state2 = transition(state, rule.rhs[0...i]) + # p' = state, B = nterm, p = state2, A = sym + key = state2.nterm_transitions.find do |goto2| + goto2.next_sym.token_id == sym.token_id + end || (raise "Goto by #{sym.name} on state #{state2.id} is not found") + # TODO: need to omit if state == state2 ? + @includes_relation[key] ||= [] + @includes_relation[key] << goto + break unless sym.nullable + i -= 1 + end + end + end + end + end + + # @rbs () -> void + def compute_lookback_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + nterm = goto.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + state2 = transition(state, rule.rhs) + # p = state, A = nterm, q = state2, A -> ω = rule + @lookback_relation[state2.id] ||= {} + @lookback_relation[state2.id][rule.id] ||= [] + @lookback_relation[state2.id][rule.id] << goto + end + end + end + end + + # @rbs () -> void + def compute_follow_sets + @follow_sets = Digraph.new(nterm_transitions, @includes_relation, @read_sets).compute + end + + # @rbs () -> void + def compute_la + @states.each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + ary.each do |goto| + # q = state, A -> ω = rule, p = state2, A = nterm + follows = @follow_sets[goto] + + next if follows == 0 + + @la[state.id] ||= {} + @la[state.id][rule.id] ||= 0 + look_ahead = @la[state.id][rule.id] | follows + @la[state.id][rule.id] |= look_ahead + + # No risk of conflict when + # * the state only has single reduce + # * the state only has nterm_transitions (GOTO) + next if state.reduces.count == 1 && state.term_transitions.count == 0 + + state.set_look_ahead(rule, bitmap_to_terms(look_ahead)) + end + end + end + end + + # @rbs (Bitmap::bitmap bit) -> Array[Grammar::Symbol] + def bitmap_to_terms(bit) + ary = Bitmap.to_array(bit) + ary.map do |i| + @grammar.find_symbol_by_number!(i) + end + end + + # @rbs () -> void + def compute_conflicts(lr_type) + compute_shift_reduce_conflicts(lr_type) + compute_reduce_reduce_conflicts + end + + # @rbs () -> void + def compute_shift_reduce_conflicts(lr_type) + states.each do |state| + state.term_transitions.each do |shift| + state.reduces.each do |reduce| + sym = shift.next_sym + + next unless reduce.look_ahead + next unless reduce.look_ahead.include?(sym) + + # Shift/Reduce conflict + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when shift_prec > reduce_prec + # Shift is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence.type + when :precedence + # Can not resolve the conflict + # + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + when :right + # Shift is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when :left + # Reduce is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when :nonassoc + # The conflict is resolved + # + # %nonassoc creates "run-time" error by removing both shift and reduce from + # the state. This makes the state to get syntax error if the conflicted token appears. + # On the other hand, %precedence creates "compile-time" error by keeping both + # shift and reduce on the state. This makes the state to be conflicted on the token. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :error, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + else + raise "Unknown precedence type. #{sym}" + end + end + end + end + end + + # @rbs (Grammar::Precedence shift_prec, Grammar::Precedence reduce_prec, State::ResolvedConflict resolved_conflict) -> void + def mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + case lr_type + when :lalr + shift_prec.mark_used_by_lalr(resolved_conflict) + reduce_prec.mark_used_by_lalr(resolved_conflict) + when :ielr + shift_prec.mark_used_by_ielr(resolved_conflict) + reduce_prec.mark_used_by_ielr(resolved_conflict) + end + end + + # @rbs () -> void + def compute_reduce_reduce_conflicts + states.each do |state| + state.reduces.combination(2) do |reduce1, reduce2| + next if reduce1.look_ahead.nil? || reduce2.look_ahead.nil? + + intersection = reduce1.look_ahead & reduce2.look_ahead + + unless intersection.empty? + state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) + end + end + end + end + + # @rbs () -> void + def compute_default_reduction + states.each do |state| + next if state.reduces.empty? + # Do not set, if conflict exist + next unless state.conflicts.empty? + # Do not set, if shift with `error` exists. + next if state.term_transitions.map {|shift| shift.next_sym }.include?(@grammar.error_symbol) + + state.default_reduction_rule = state.reduces.map do |r| + [r.rule, r.rule.id, (r.look_ahead || []).count] + end.min_by do |rule, rule_id, count| + [-count, rule_id] + end.first + end + end + + # @rbs () -> void + def clear_conflicts + states.each(&:clear_conflicts) + end + + # Definition 3.15 (Predecessors) + # + # @rbs () -> void + def compute_predecessors + @states.each do |state| + state.transitions.each do |transition| + transition.to_state.append_predecessor(state) + end + end + end + + # Definition 3.16 (follow_kernel_items) + # + # @rbs () -> void + def compute_follow_kernel_items + set = nterm_transitions + relation = compute_goto_internal_relation + base_function = compute_goto_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, follow_kernel_items| + state = goto.from_state + state.follow_kernel_items[goto] = state.kernels.map {|kernel| + [kernel, Bitmap.to_bool_array(follow_kernel_items, state.kernels.count)] + }.to_h + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_goto_bitmaps + nterm_transitions.map {|goto| + bools = goto.from_state.kernels.map.with_index {|kernel, i| i if kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) }.compact + [goto, Bitmap.from_array(bools)] + }.to_h + end + + # Definition 3.20 (always_follows, one closure) + # + # @rbs () -> void + def compute_always_follows + set = nterm_transitions + relation = compute_goto_successor_or_internal_relation + base_function = compute_transition_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, always_follows_bitmap| + goto.from_state.always_follows[goto] = bitmap_to_terms(always_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_successor_or_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.successor_dependencies(goto) + state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_transition_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.to_state.term_transitions.map {|shift| shift.next_sym.number })] + }.to_h + end + + # Definition 3.24 (goto_follows, via always_follows) + # + # @rbs () -> void + def compute_goto_follows + set = nterm_transitions + relation = compute_goto_internal_or_predecessor_dependencies + base_function = compute_always_follows_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, goto_follows_bitmap| + goto.from_state.goto_follows[goto] = bitmap_to_terms(goto_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_or_predecessor_dependencies + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + state.predecessor_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_always_follows_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.from_state.always_follows[goto].map(&:number))] + }.to_h + end + + # @rbs () -> void + def split_states + @states.each do |state| + state.transitions.each do |transition| + compute_state(state, transition, transition.to_state) + end + end + end + + # @rbs () -> void + def compute_inadequacy_annotations + @states.each do |state| + state.annotate_manifestation + end + + queue = @states.reject {|state| state.annotation_list.empty? } + + while (curr = queue.shift) do + curr.predecessors.each do |pred| + cache = pred.annotation_list.dup + curr.annotate_predecessor(pred) + queue << pred if cache != pred.annotation_list && !queue.include?(pred) + end + end + end + + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> void + def merge_lookaheads(state, filtered_lookaheads) + return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? } + + state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 } + state.transitions.each do |transition| + next if transition.to_state.lookaheads_recomputed + compute_state(state, transition, transition.to_state) + end + end + + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void + def compute_state(state, transition, next_state) + propagating_lookaheads = state.propagate_lookaheads(next_state) + s = next_state.ielr_isocores.find {|st| st.is_compatible?(propagating_lookaheads) } + + if s.nil? + s = next_state.lalr_isocore + new_state = State.new(@states.count, s.accessing_symbol, s.kernels) + new_state.closure = s.closure + new_state.compute_transitions_and_reduces + s.transitions.each do |transition| + new_state.set_items_to_state(transition.to_items, transition.to_state) + end + @states << new_state + new_state.lalr_isocore = s + s.ielr_isocores << new_state + s.ielr_isocores.each do |st| + st.ielr_isocores = s.ielr_isocores + end + new_state.lookaheads_recomputed = true + new_state.item_lookahead_set = propagating_lookaheads + state.update_transition(transition, new_state) + elsif(!s.lookaheads_recomputed) + s.lookaheads_recomputed = true + s.item_lookahead_set = propagating_lookaheads + else + merge_lookaheads(s, propagating_lookaheads) + state.update_transition(transition, s) if state.items_to_state[transition.to_items].id != s.id + end + end + + # @rbs (Logger logger) -> void + def validate_conflicts_within_threshold!(logger) + exit false unless conflicts_within_threshold?(logger) + end + + # @rbs (Logger logger) -> bool + def conflicts_within_threshold?(logger) + return true unless @grammar.expect + + [sr_conflicts_within_threshold?(logger), rr_conflicts_within_threshold?(logger)].all? + end + + # @rbs (Logger logger) -> bool + def sr_conflicts_within_threshold?(logger) + return true if @grammar.expect == sr_conflicts_count + + logger.error("shift/reduce conflicts: #{sr_conflicts_count} found, #{@grammar.expect} expected") + false + end + + # @rbs (Logger logger) -> bool + def rr_conflicts_within_threshold?(logger, expected: 0) + return true if expected == rr_conflicts_count + + logger.error("reduce/reduce conflicts: #{rr_conflicts_count} found, #{expected} expected") + false + end + + # @rbs () -> void + def clear_look_ahead_sets + @direct_read_sets.clear + @reads_relation.clear + @read_sets.clear + @includes_relation.clear + @lookback_relation.clear + @follow_sets.clear + @la.clear + + @_direct_read_sets = nil + @_read_sets = nil + @_follow_sets = nil + @_la = nil + end + end +end diff --git a/tool/lrama/lib/lrama/tracer.rb b/tool/lrama/lib/lrama/tracer.rb new file mode 100644 index 0000000000..fda699a665 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer.rb @@ -0,0 +1,51 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "tracer/actions" +require_relative "tracer/closure" +require_relative "tracer/duration" +require_relative "tracer/only_explicit_rules" +require_relative "tracer/rules" +require_relative "tracer/state" + +module Lrama + class Tracer + # @rbs (IO io, **bool options) -> void + def initialize(io, **options) + @io = io + @options = options + @only_explicit_rules = OnlyExplicitRules.new(io, **options) + @rules = Rules.new(io, **options) + @actions = Actions.new(io, **options) + @closure = Closure.new(io, **options) + @state = State.new(io, **options) + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + @only_explicit_rules.trace(grammar) + @rules.trace(grammar) + @actions.trace(grammar) + end + + # @rbs (Lrama::State state) -> void + def trace_closure(state) + @closure.trace(state) + end + + # @rbs (Lrama::State state) -> void + def trace_state(state) + @state.trace(state) + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_state_list_append(state_count, state) + @state.trace_list_append(state_count, state) + end + + # @rbs () -> void + def enable_duration + Duration.enable if @options[:time] + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/actions.rb b/tool/lrama/lib/lrama/tracer/actions.rb new file mode 100644 index 0000000000..7b9c9b9f53 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/actions.rb @@ -0,0 +1,22 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Actions + # @rbs (IO io, ?actions: bool, **bool options) -> void + def initialize(io, actions: false, **options) + @io = io + @actions = actions + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @actions + + @io << "Grammar rules with actions:" << "\n" + grammar.rules.each { |rule| @io << rule.with_actions << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/closure.rb b/tool/lrama/lib/lrama/tracer/closure.rb new file mode 100644 index 0000000000..5b2f0b27e6 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/closure.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Closure + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @closure = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @closure + + @io << "Closure: input" << "\n" + state.kernels.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + @io << "Closure: output" << "\n" + state.items.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/duration.rb b/tool/lrama/lib/lrama/tracer/duration.rb new file mode 100644 index 0000000000..91c49625b2 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/duration.rb @@ -0,0 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + module Duration + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @_report_duration_enabled: bool + + # @rbs () -> void + def self.enable + @_report_duration_enabled = true + end + + # @rbs () -> bool + def self.enabled? + !!@_report_duration_enabled + end + + # @rbs [T] (_ToS message) { -> T } -> T + def report_duration(message) + time1 = Time.now.to_f + result = yield + time2 = Time.now.to_f + + if Duration.enabled? + STDERR.puts sprintf("%s %10.5f s", message, time2 - time1) + end + + return result + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb new file mode 100644 index 0000000000..4f64e7d2f4 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class OnlyExplicitRules + # @rbs (IO io, ?only_explicit: bool, **bool) -> void + def initialize(io, only_explicit: false, **_) + @io = io + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each do |rule| + @io << rule.display_name_without_action << "\n" if rule.lhs.first_set.any? + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/rules.rb b/tool/lrama/lib/lrama/tracer/rules.rb new file mode 100644 index 0000000000..d6e85b8432 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Rules + # @rbs (IO io, ?rules: bool, ?only_explicit: bool, **bool) -> void + def initialize(io, rules: false, only_explicit: false, **_) + @io = io + @rules = rules + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return if !@rules || @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each { |rule| @io << rule.display_name << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/state.rb b/tool/lrama/lib/lrama/tracer/state.rb new file mode 100644 index 0000000000..21c0047f8e --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/state.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class State + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @state = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @state + + # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but + # I think it is not correct... + previous = state.kernels.first.previous_sym + @io << "Processing state #{state.id} (reached by #{previous.display_name})" << "\n" + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_list_append(state_count, state) + return unless @state + + previous = state.kernels.first.previous_sym + @io << sprintf("state_list_append (state = %d, symbol = %d (%s))", + state_count, previous.number, previous.display_name) << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb new file mode 100644 index 0000000000..d649b74939 --- /dev/null +++ b/tool/lrama/lib/lrama/version.rb @@ -0,0 +1,6 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + VERSION = "0.7.1".freeze #: String +end diff --git a/tool/lrama/lib/lrama/warnings.rb b/tool/lrama/lib/lrama/warnings.rb new file mode 100644 index 0000000000..52f09144ef --- /dev/null +++ b/tool/lrama/lib/lrama/warnings.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'warnings/conflicts' +require_relative 'warnings/implicit_empty' +require_relative 'warnings/name_conflicts' +require_relative 'warnings/redefined_rules' +require_relative 'warnings/required' +require_relative 'warnings/useless_precedence' + +module Lrama + class Warnings + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @conflicts = Conflicts.new(logger, warnings) + @implicit_empty = ImplicitEmpty.new(logger, warnings) + @name_conflicts = NameConflicts.new(logger, warnings) + @redefined_rules = RedefinedRules.new(logger, warnings) + @required = Required.new(logger, warnings) + @useless_precedence = UselessPrecedence.new(logger, warnings) + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + @conflicts.warn(states) + @implicit_empty.warn(grammar) + @name_conflicts.warn(grammar) + @redefined_rules.warn(grammar) + @required.warn(grammar) + @useless_precedence.warn(grammar, states) + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/conflicts.rb b/tool/lrama/lib/lrama/warnings/conflicts.rb new file mode 100644 index 0000000000..6ba0de6f9c --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/conflicts.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Conflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::States states) -> void + def warn(states) + return unless @warnings + + if states.sr_conflicts_count != 0 + @logger.warn("shift/reduce conflicts: #{states.sr_conflicts_count} found") + end + + if states.rr_conflicts_count != 0 + @logger.warn("reduce/reduce conflicts: #{states.rr_conflicts_count} found") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/implicit_empty.rb b/tool/lrama/lib/lrama/warnings/implicit_empty.rb new file mode 100644 index 0000000000..ba81adca01 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/implicit_empty.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Empty rules are easily overlooked and ambiguous + # - Empty alternatives like `rule: | "token";` can be missed during code reading + # - Difficult to distinguish between intentional empty rules vs. omissions + # - Explicit marking with %empty directive comment improves clarity + class ImplicitEmpty + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.rule_builders.each do |builder| + if builder.rhs.empty? + @logger.warn("warning: empty rule without %empty") + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/name_conflicts.rb b/tool/lrama/lib/lrama/warnings/name_conflicts.rb new file mode 100644 index 0000000000..c0754ab551 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/name_conflicts.rb @@ -0,0 +1,63 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Parameterized rule names conflicting with symbol names + # - When a %rule name is identical to a terminal or non-terminal symbol name, + # it reduces grammar readability and may cause unintended behavior + # - Detecting these conflicts helps improve grammar definition quality + class NameConflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return if grammar.parameterized_rules.empty? + + symbol_names = collect_symbol_names(grammar) + check_conflicts(grammar.parameterized_rules, symbol_names) + end + + private + + # @rbs (Lrama::Grammar grammar) -> Set[String] + def collect_symbol_names(grammar) + symbol_names = Set.new + + collect_term_names(grammar.terms, symbol_names) + collect_nterm_names(grammar.nterms, symbol_names) + + symbol_names + end + + # @rbs (Array[untyped] terms, Set[String] symbol_names) -> void + def collect_term_names(terms, symbol_names) + terms.each do |term| + symbol_names.add(term.id.s_value) + symbol_names.add(term.alias_name) if term.alias_name + end + end + + # @rbs (Array[untyped] nterms, Set[String] symbol_names) -> void + def collect_nterm_names(nterms, symbol_names) + nterms.each do |nterm| + symbol_names.add(nterm.id.s_value) + end + end + + # @rbs (Array[untyped] parameterized_rules, Set[String] symbol_names) -> void + def check_conflicts(parameterized_rules, symbol_names) + parameterized_rules.each do |param_rule| + next unless symbol_names.include?(param_rule.name) + + @logger.warn("warning: parameterized rule name \"#{param_rule.name}\" conflicts with symbol name") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/redefined_rules.rb b/tool/lrama/lib/lrama/warnings/redefined_rules.rb new file mode 100644 index 0000000000..8ac2f1f103 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/redefined_rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class RedefinedRules + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.parameterized_resolver.redefined_rules.each do |rule| + @logger.warn("parameterized rule redefined: #{rule}") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/required.rb b/tool/lrama/lib/lrama/warnings/required.rb new file mode 100644 index 0000000000..4ab1ed787e --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/required.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Required + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings = false, **_) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + if grammar.required + @logger.warn("currently, %require is simply valid as a grammar but does nothing") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/useless_precedence.rb b/tool/lrama/lib/lrama/warnings/useless_precedence.rb new file mode 100644 index 0000000000..2913d6d7e5 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/useless_precedence.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class UselessPrecedence + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + return unless @warnings + + grammar.precedences.each do |precedence| + unless precedence.used_by? + @logger.warn("Precedence #{precedence.s_value} (line: #{precedence.lineno}) is defined but not used in any rule.") + end + end + end + end + end +end diff --git a/tool/lrama/template/bison/_yacc.h b/tool/lrama/template/bison/_yacc.h new file mode 100644 index 0000000000..3e270c9171 --- /dev/null +++ b/tool/lrama/template/bison/_yacc.h @@ -0,0 +1,79 @@ +<%# b4_shared_declarations -%> + <%-# b4_cpp_guard_open([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#ifndef <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> +# define <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> + <%- end -%> + <%-# b4_declare_yydebug & b4_YYDEBUG_define -%> +/* Debug traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif +#if YYDEBUG && !defined(yydebug) +extern int yydebug; +#endif +<%= output.percent_code("requires") %> + + <%-# b4_token_enums_defines -%> +/* Token kinds. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + enum yytokentype + { +<%= output.token_enums -%> + }; + typedef enum yytokentype yytoken_kind_t; +#endif + + <%-# b4_declare_yylstype -%> + <%-# b4_value_type_define -%> +/* Value type. */ +<% if output.grammar.union %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +union YYSTYPE +{ +#line <%= output.grammar.union.lineno %> "<%= output.grammar_file_path %>" +<%= output.grammar.union.braces_less_code %> +#line [@oline@] [@ofile@] + +}; +typedef union YYSTYPE YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% else %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef int YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% end %> + + <%-# b4_location_type_define -%> +/* Location type. */ +#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED +typedef struct YYLTYPE YYLTYPE; +struct YYLTYPE +{ + int first_line; + int first_column; + int last_line; + int last_column; +}; +# define YYLTYPE_IS_DECLARED 1 +# define YYLTYPE_IS_TRIVIAL 1 +#endif + + + + + <%-# b4_declare_yyerror_and_yylex. Not supported -%> + <%-# b4_declare_yyparse -%> +int yyparse (<%= output.parse_param %>); + + +<%= output.percent_code("provides") %> + <%-# b4_cpp_guard_close([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#endif /* !<%= output.b4_cpp_guard__b4_spec_mapped_header_file %> */ + <%- end -%> diff --git a/tool/lrama/template/bison/yacc.c b/tool/lrama/template/bison/yacc.c new file mode 100644 index 0000000000..6edd59a0d5 --- /dev/null +++ b/tool/lrama/template/bison/yacc.c @@ -0,0 +1,2068 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison implementation for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* C LALR(1) parser skeleton written by Richard Stallman, by + simplifying the original so-called "semantic" parser. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + +/* All symbols defined below should begin with yy or YY, to avoid + infringing on user name space. This should be done even for local + variables, as they might otherwise be expanded by user macros. + There are some unavoidable exceptions within include files to + define necessary library symbols; they are noted "INFRINGES ON + USER NAME SPACE" below. */ + +<%# b4_identification -%> +/* Identify Bison output, and Bison version. */ +#define YYBISON 30802 + +/* Bison version string. */ +#define YYBISON_VERSION "3.8.2" + +/* Skeleton name. */ +#define YYSKELETON_NAME "<%= output.template_basename %>" + +/* Pure parsers. */ +#define YYPURE 1 + +/* Push parsers. */ +#define YYPUSH 0 + +/* Pull parsers. */ +#define YYPULL 1 + + +<%# b4_user_pre_prologue -%> +<%- if output.aux.prologue -%> +/* First part of user prologue. */ +#line <%= output.aux.prologue_first_lineno %> "<%= output.grammar_file_path %>" +<%= output.aux.prologue %> +#line [@oline@] [@ofile@] +<%- end -%> + +<%# b4_cast_define -%> +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast<Type> (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast<Type> (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif +<%# b4_null_define -%> +# ifndef YY_NULLPTR +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif +# else +# define YY_NULLPTR ((void*)0) +# endif +# endif + +<%# b4_header_include_if -%> +<%- if output.include_header -%> +#include "<%= output.include_header %>" +<%- else -%> +/* Use api.header.include to #include this header + instead of duplicating it here. */ +<%= output.render_partial("bison/_yacc.h") %> +<%- end -%> +<%# b4_declare_symbol_enum -%> +/* Symbol kind. */ +enum yysymbol_kind_t +{ +<%= output.symbol_enum -%> +}; +typedef enum yysymbol_kind_t yysymbol_kind_t; + + + + +<%# b4_user_post_prologue -%> +<%# b4_c99_int_type_define -%> +#ifdef short +# undef short +#endif + +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + <limits.h> and (if available) <stdint.h> are included + so that the code can choose integer types of a good width. */ + +#ifndef __PTRDIFF_MAX__ +# include <limits.h> /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stdint.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif + +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ + +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; +#endif + +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; +#else +typedef short yytype_int16; +#endif + +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + <https://h20195.www2.hpe.com/V2/getpdf.aspx/4AA4-7673ENW.pdf>. */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 +#endif + +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; +#else +typedef short yytype_uint8; +#endif + +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; +#else +typedef int yytype_uint16; +#endif + +<%# b4_sizes_types_define -%> +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif +#endif + +#ifndef YYSIZE_T +# ifdef __SIZE_TYPE__ +# define YYSIZE_T __SIZE_TYPE__ +# elif defined size_t +# define YYSIZE_T size_t +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# define YYSIZE_T size_t +# else +# define YYSIZE_T unsigned +# endif +#endif + +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef <%= output.int_type_for([output.yynstates - 1]) %> yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include <libintl.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_(Msgid) dgettext ("bison-runtime", Msgid) +# endif +# endif +# ifndef YY_ +# define YY_(Msgid) Msgid +# endif +#endif + + +<%# b4_attribute_define -%> +#ifndef YY_ATTRIBUTE_PURE +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif +#endif + +#ifndef YY_ATTRIBUTE_UNUSED +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) +# else +# define YY_ATTRIBUTE_UNUSED +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YY_USE(E) ((void) (E)) +#else +# define YY_USE(E) /* empty */ +#endif + +/* Suppress an incorrect diagnostic about yylval being uninitialized. */ +#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__ +# if __GNUC__ * 100 + __GNUC_MINOR__ < 407 +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") +# else +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ + _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") +# endif +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ + _Pragma ("GCC diagnostic pop") +#else +# define YY_INITIAL_VALUE(Value) Value +#endif +#ifndef YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_END +#endif +#ifndef YY_INITIAL_VALUE +# define YY_INITIAL_VALUE(Value) /* Nothing. */ +#endif + +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + + +#define YY_ASSERT(E) ((void) (0 && (E))) + +#if 1 + +/* The parser invokes alloca or malloc; define the necessary symbols. */ + +# ifdef YYSTACK_USE_ALLOCA +# if YYSTACK_USE_ALLOCA +# ifdef __GNUC__ +# define YYSTACK_ALLOC __builtin_alloca +# elif defined __BUILTIN_VA_ARG_INCR +# include <alloca.h> /* INFRINGES ON USER NAME SPACE */ +# elif defined _AIX +# define YYSTACK_ALLOC __alloca +# elif defined _MSC_VER +# include <malloc.h> /* INFRINGES ON USER NAME SPACE */ +# define alloca _alloca +# else +# define YYSTACK_ALLOC alloca +# if ! defined _ALLOCA_H && ! defined EXIT_SUCCESS +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ + /* Use EXIT_SUCCESS as a witness for stdlib.h. */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# endif +# endif +# endif + +# ifdef YYSTACK_ALLOC + /* Pacify GCC's 'empty if-body' warning. */ +# define YYSTACK_FREE(Ptr) do { /* empty */; } while (0) +# ifndef YYSTACK_ALLOC_MAXIMUM + /* The OS might guarantee only one guard page at the bottom of the stack, + and a page size can be as small as 4096 bytes. So we cannot safely + invoke alloca (N) if N exceeds 4096. Use a slightly smaller number + to allow for a few compiler-allocated temporary stack slots. */ +# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +# endif +# else +# define YYSTACK_ALLOC YYMALLOC +# define YYSTACK_FREE YYFREE +# ifndef YYSTACK_ALLOC_MAXIMUM +# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +# endif +# if (defined __cplusplus && ! defined EXIT_SUCCESS \ + && ! ((defined YYMALLOC || defined malloc) \ + && (defined YYFREE || defined free))) +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# ifndef YYMALLOC +# define YYMALLOC malloc +# if ! defined malloc && ! defined EXIT_SUCCESS +void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# ifndef YYFREE +# define YYFREE free +# if ! defined free && ! defined EXIT_SUCCESS +void free (void *); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# endif +#endif /* 1 */ + +#if (! defined yyoverflow \ + && (! defined __cplusplus \ + || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL \ + && defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) + +/* A type that is properly aligned for any stack member. */ +union yyalloc +{ + yy_state_t yyss_alloc; + YYSTYPE yyvs_alloc; + YYLTYPE yyls_alloc; +}; + +/* The size of the maximum gap between one aligned stack and the next. */ +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) + +/* The size of an array large to enough to hold all stacks, each with + N elements. */ +# define YYSTACK_BYTES(N) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE) \ + + YYSIZEOF (YYLTYPE)) \ + + 2 * YYSTACK_GAP_MAXIMUM) + +# define YYCOPY_NEEDED 1 + +/* Relocate STACK from its old location to the new one. The + local variables YYSIZE and YYSTACKSIZE give the old and new number of + elements in the stack, and YYPTR gives the new location of the + stack. Advance YYPTR to a properly aligned location for the next + stack. */ +# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYPTRDIFF_T yynewbytes; \ + YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ + } \ + while (0) + +#endif + +#if defined YYCOPY_NEEDED && YYCOPY_NEEDED +/* Copy COUNT objects from SRC to DST. The source and destination do + not overlap. */ +# ifndef YYCOPY +# if defined __GNUC__ && 1 < __GNUC__ +# define YYCOPY(Dst, Src, Count) \ + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) +# else +# define YYCOPY(Dst, Src, Count) \ + do \ + { \ + YYPTRDIFF_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (Dst)[yyi] = (Src)[yyi]; \ + } \ + while (0) +# endif +# endif +#endif /* !YYCOPY_NEEDED */ + +/* YYFINAL -- State number of the termination state. */ +#define YYFINAL <%= output.yyfinal %> +/* YYLAST -- Last index in YYTABLE. */ +#define YYLAST <%= output.yylast %> + +/* YYNTOKENS -- Number of terminals. */ +#define YYNTOKENS <%= output.yyntokens %> +/* YYNNTS -- Number of nonterminals. */ +#define YYNNTS <%= output.yynnts %> +/* YYNRULES -- Number of rules. */ +#define YYNRULES <%= output.yynrules %> +/* YYNSTATES -- Number of states. */ +#define YYNSTATES <%= output.yynstates %> + +/* YYMAXUTOK -- Last valid token kind. */ +#define YYMAXUTOK <%= output.yymaxutok %> + + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) + +/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM + as returned by yylex. */ +static const <%= output.int_type_for(output.context.yytranslate) %> yytranslate[] = +{ +<%= output.yytranslate %> +}; + +<%- if output.error_recovery -%> +/* YYTRANSLATE_INVERTED[SYMBOL-NUM] -- Token number corresponding to SYMBOL-NUM */ +static const <%= output.int_type_for(output.context.yytranslate_inverted) %> yytranslate_inverted[] = +{ +<%= output.yytranslate_inverted %> +}; +<%- end -%> +#if YYDEBUG +/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ +static const <%= output.int_type_for(output.context.yyrline) %> yyrline[] = +{ +<%= output.yyrline %> +}; +#endif + +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if 1 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + +/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. + First, the terminals, then, starting at YYNTOKENS, nonterminals. */ +static const char *const yytname[] = +{ +<%= output.yytname %> +}; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} +#endif + +#define YYPACT_NINF (<%= output.yypact_ninf %>) + +#define yypact_value_is_default(Yyn) \ + <%= output.table_value_equals(output.context.yypact, "Yyn", output.yypact_ninf, "YYPACT_NINF") %> + +#define YYTABLE_NINF (<%= output.yytable_ninf %>) + +#define yytable_value_is_error(Yyn) \ + <%= output.table_value_equals(output.context.yytable, "Yyn", output.yytable_ninf, "YYTABLE_NINF") %> + +<%# b4_parser_tables_define -%> +/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + STATE-NUM. */ +static const <%= output.int_type_for(output.context.yypact) %> yypact[] = +{ +<%= output.int_array_to_string(output.context.yypact) %> +}; + +/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. + Performed when YYTABLE does not specify something else to do. Zero + means the default is an error. */ +static const <%= output.int_type_for(output.context.yydefact) %> yydefact[] = +{ +<%= output.int_array_to_string(output.context.yydefact) %> +}; + +/* YYPGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yypgoto) %> yypgoto[] = +{ +<%= output.int_array_to_string(output.context.yypgoto) %> +}; + +/* YYDEFGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yydefgoto) %> yydefgoto[] = +{ +<%= output.int_array_to_string(output.context.yydefgoto) %> +}; + +/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If + positive, shift that token. If negative, reduce the rule whose + number is the opposite. If YYTABLE_NINF, syntax error. */ +static const <%= output.int_type_for(output.context.yytable) %> yytable[] = +{ +<%= output.int_array_to_string(output.context.yytable) %> +}; + +static const <%= output.int_type_for(output.context.yycheck) %> yycheck[] = +{ +<%= output.int_array_to_string(output.context.yycheck) %> +}; + +/* YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of + state STATE-NUM. */ +static const <%= output.int_type_for(output.context.yystos) %> yystos[] = +{ +<%= output.int_array_to_string(output.context.yystos) %> +}; + +/* YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr1) %> yyr1[] = +{ +<%= output.int_array_to_string(output.context.yyr1) %> +}; + +/* YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr2) %> yyr2[] = +{ +<%= output.int_array_to_string(output.context.yyr2) %> +}; + + +enum { YYENOMEM = -2 }; + +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab +#define YYNOMEM goto yyexhaustedlab + + +#define YYRECOVERING() (!!yyerrstatus) + +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (<%= output.yyerror_args %>, YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF + +<%# b4_yylloc_default_define -%> +/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. + If N is 0, then set CURRENT to the empty location which ends + the previous symbol: RHS[0] (always defined). */ + +#ifndef YYLLOC_DEFAULT +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = \ + YYRHSLOC (Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC (Rhs, 0).last_column; \ + } \ + while (0) +#endif + +#define YYRHSLOC(Rhs, K) ((Rhs)[K]) + + +/* Enable debugging if requested. */ +#if YYDEBUG + +# ifndef YYFPRINTF +# include <stdio.h> /* INFRINGES ON USER NAME SPACE */ +# define YYFPRINTF fprintf +# endif + +# define YYDPRINTF(Args) \ +do { \ + if (yydebug) \ + YYFPRINTF Args; \ +} while (0) + + +<%# b4_yylocation_print_define -%> +/* YYLOCATION_PRINT -- Print the location on the stream. + This macro was not mandated originally: define only if we know + we won't break user code: when these are the locations we know. */ + +# ifndef YYLOCATION_PRINT + +# if defined YY_LOCATION_PRINT + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) YY_LOCATION_PRINT(File, *(Loc)<%= output.user_args %>) + +# elif defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + +/* Print *YYLOCP on YYO. Private, do not rely on its existence. */ + +YY_ATTRIBUTE_UNUSED +static int +yy_location_print_ (FILE *yyo, YYLTYPE const * const yylocp) +{ + int res = 0; + int end_col = 0 != yylocp->last_column ? yylocp->last_column - 1 : 0; + if (0 <= yylocp->first_line) + { + res += YYFPRINTF (yyo, "%d", yylocp->first_line); + if (0 <= yylocp->first_column) + res += YYFPRINTF (yyo, ".%d", yylocp->first_column); + } + if (0 <= yylocp->last_line) + { + if (yylocp->first_line < yylocp->last_line) + { + res += YYFPRINTF (yyo, "-%d", yylocp->last_line); + if (0 <= end_col) + res += YYFPRINTF (yyo, ".%d", end_col); + } + else if (0 <= end_col && yylocp->first_column < end_col) + res += YYFPRINTF (yyo, "-%d", end_col); + } + return res; +} + +# define YYLOCATION_PRINT yy_location_print_ + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT(File, Loc<%= output.user_args %>) YYLOCATION_PRINT(File, &(Loc)<%= output.user_args %>) + +# else + +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) ((void) 0) + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT YYLOCATION_PRINT + +# endif +# endif /* !defined YYLOCATION_PRINT */ + + +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) \ +do { \ + if (yydebug) \ + { \ + YYFPRINTF (stderr, "%s ", Title); \ + yy_symbol_print (stderr, \ + Kind, Value, Location<%= output.user_args %>); \ + YYFPRINTF (stderr, "\n"); \ + } \ +} while (0) + + +<%# b4_yy_symbol_print_define -%> +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ + +static void +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + FILE *yyoutput = yyo; +<%= output.parse_param_use("yyoutput", "yylocationp") %> + if (!yyvaluep) + return; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +<%# b4_symbol_actions(printer) -%> +switch (yykind) + { +<%= output.symbol_actions_for_printer -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ + +static void +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); + + YYLOCATION_PRINT (yyo, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ": "); + yy_symbol_value_print (yyo, yykind, yyvaluep, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ")"); +} + +/*------------------------------------------------------------------. +| yy_stack_print -- Print the state stack from its BOTTOM up to its | +| TOP (included). | +`------------------------------------------------------------------*/ + +static void +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop<%= output.user_formals %>) +{ + YYFPRINTF (stderr, "Stack now"); + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF (stderr, " %d", yybot); + } + YYFPRINTF (stderr, "\n"); +} + +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_stack_print ((Bottom), (Top)<%= output.user_args %>); \ +} while (0) + + +/*------------------------------------------------. +| Report that the YYRULE is going to be reduced. | +`------------------------------------------------*/ + +static void +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, YYLTYPE *yylsp, + int yyrule<%= output.user_formals %>) +{ + int yylno = yyrline[yyrule]; + int yynrhs = yyr2[yyrule]; + int yyi; + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", + yyrule - 1, yylno); + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF (stderr, " $%d = ", yyi + 1); + yy_symbol_print (stderr, + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)], + &(yylsp[(yyi + 1) - (yynrhs)])<%= output.user_args %>); + YYFPRINTF (stderr, "\n"); + } +} + +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_reduce_print (yyssp, yyvsp, yylsp, Rule<%= output.user_args %>); \ +} while (0) + +/* Nonzero means print parse trace. It is left uninitialized so that + multiple parsers can coexist. */ +#ifndef yydebug +int yydebug; +#endif +#else /* !YYDEBUG */ +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) +#endif /* !YYDEBUG */ + + +/* YYINITDEPTH -- initial size of the parser's stacks. */ +#ifndef YYINITDEPTH +# define YYINITDEPTH 200 +#endif + +/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only + if the built-in stack extension method is used). + + Do not make this value too large; the results are undefined if + YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) + evaluated with infinite-precision integer arithmetic. */ + +#ifndef YYMAXDEPTH +# define YYMAXDEPTH 10000 +#endif + + +/* Context of a parse error. */ +typedef struct +{ + yy_state_t *yyssp; + yysymbol_kind_t yytoken; + YYLTYPE *yylloc; +} yypcontext_t; + +/* Put in YYARG at most YYARGN of the expected tokens given the + current YYCTX, and return the number of tokens stored in YYARG. If + YYARG is null, return the number of expected tokens (guaranteed to + be less than YYNTOKENS). Return YYENOMEM on memory exhaustion. + Return 0 if there are more than YYARGN expected tokens, yet fill + YYARG up to YYARGN. */ +static int +yypcontext_expected_tokens (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + int yyn = yypact[+*yyctx->yyssp]; + if (!yypact_value_is_default (yyn)) + { + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. In other words, skip the first -YYN actions for + this state because they are default actions. */ + int yyxbegin = yyn < 0 ? -yyn : 0; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYSYMBOL_YYerror + && !yytable_value_is_error (yytable[yyx + yyn])) + { + if (!yyarg) + ++yycount; + else if (yycount == yyargn) + return 0; + else + yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx); + } + } + if (yyarg && yycount == 0 && 0 < yyargn) + yyarg[0] = YYSYMBOL_YYEMPTY; + return yycount; +} + + + + +#ifndef yystrlen +# if defined __GLIBC__ && defined _STRING_H +# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S))) +# else +/* Return the length of YYSTR. */ +static YYPTRDIFF_T +yystrlen (const char *yystr) +{ + YYPTRDIFF_T yylen; + for (yylen = 0; yystr[yylen]; yylen++) + continue; + return yylen; +} +# endif +#endif + +#ifndef yystpcpy +# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +# define yystpcpy stpcpy +# else +/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in + YYDEST. */ +static char * +yystpcpy (char *yydest, const char *yysrc) +{ + char *yyd = yydest; + const char *yys = yysrc; + + while ((*yyd++ = *yys++) != '\0') + continue; + + return yyd - 1; +} +# endif +#endif + +#ifndef yytnamerr +/* Copy to YYRES the contents of YYSTR after stripping away unnecessary + quotes and backslashes, so that it's suitable for yyerror. The + heuristic is that double-quoting is unnecessary unless the string + contains an apostrophe, a comma, or backslash (other than + backslash-backslash). YYSTR is taken from yytname. If YYRES is + null, do not copy; instead, return the length of what the result + would have been. */ +static YYPTRDIFF_T +yytnamerr (char *yyres, const char *yystr) +{ + if (*yystr == '"') + { + YYPTRDIFF_T yyn = 0; + char const *yyp = yystr; + for (;;) + switch (*++yyp) + { + case '\'': + case ',': + goto do_not_strip_quotes; + + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; + else + goto append; + + append: + default: + if (yyres) + yyres[yyn] = *yyp; + yyn++; + break; + + case '"': + if (yyres) + yyres[yyn] = '\0'; + return yyn; + } + do_not_strip_quotes: ; + } + + if (yyres) + return yystpcpy (yyres, yystr) - yyres; + else + return yystrlen (yystr); +} +#endif + + +static int +yy_syntax_error_arguments (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + /* There are many possibilities here to consider: + - If this state is a consistent state with a default action, then + the only way this function was invoked is if the default action + is an error action. In that case, don't check for expected + tokens because there are none. + - The only way there can be no lookahead present (in yychar) is if + this state is a consistent state with a default action. Thus, + detecting the absence of a lookahead is sufficient to determine + that there is no unexpected or expected token to report. In that + case, just report a simple "syntax error". + - Don't assume there isn't a lookahead just because this state is a + consistent state with a default action. There might have been a + previous inconsistent state, consistent state with a non-default + action, or user semantic action that manipulated yychar. + - Of course, the expected token list depends on states to have + correct lookahead information, and it depends on the parser not + to perform extra reductions after fetching a lookahead from the + scanner and before detecting a syntax error. Thus, state merging + (from LALR or IELR) and default reductions corrupt the expected + token list. However, the list is correct for canonical LR with + one exception: it will still contain any token that will not be + accepted due to an error action in a later state. + */ + if (yyctx->yytoken != YYSYMBOL_YYEMPTY) + { + int yyn; + if (yyarg) + yyarg[yycount] = yyctx->yytoken; + ++yycount; + yyn = yypcontext_expected_tokens (yyctx, + yyarg ? yyarg + 1 : yyarg, yyargn - 1); + if (yyn == YYENOMEM) + return YYENOMEM; + else + yycount += yyn; + } + return yycount; +} + +/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message + about the unexpected token YYTOKEN for the state stack whose top is + YYSSP. + + Return 0 if *YYMSG was successfully written. Return -1 if *YYMSG is + not large enough to hold the message. In that case, also set + *YYMSG_ALLOC to the required number of bytes. Return YYENOMEM if the + required number of bytes is too large to store. */ +static int +yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg, + const yypcontext_t *yyctx<%= output.user_formals %>) +{ + enum { YYARGS_MAX = 5 }; + /* Internationalized format string. */ + const char *yyformat = YY_NULLPTR; + /* Arguments of yyformat: reported tokens (one for the "unexpected", + one per "expected"). */ + yysymbol_kind_t yyarg[YYARGS_MAX]; + /* Cumulated lengths of YYARG. */ + YYPTRDIFF_T yysize = 0; + + /* Actual size of YYARG. */ + int yycount = yy_syntax_error_arguments (yyctx, yyarg, YYARGS_MAX); + if (yycount == YYENOMEM) + return YYENOMEM; + + switch (yycount) + { +#define YYCASE_(N, S) \ + case N: \ + yyformat = S; \ + break + default: /* Avoid compiler warnings. */ + YYCASE_(0, YY_("syntax error")); + YYCASE_(1, YY_("syntax error, unexpected %s")); + YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); + YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s")); + YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); + YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); +#undef YYCASE_ + } + + /* Compute error message size. Don't count the "%s"s, but reserve + room for the terminator. */ + yysize = yystrlen (yyformat) - 2 * yycount + 1; + { + int yyi; + for (yyi = 0; yyi < yycount; ++yyi) + { + YYPTRDIFF_T yysize1 + = yysize + yytnamerr (YY_NULLPTR, yytname[yyarg[yyi]]); + if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM) + yysize = yysize1; + else + return YYENOMEM; + } + } + + if (*yymsg_alloc < yysize) + { + *yymsg_alloc = 2 * yysize; + if (! (yysize <= *yymsg_alloc + && *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM)) + *yymsg_alloc = YYSTACK_ALLOC_MAXIMUM; + return -1; + } + + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + { + char *yyp = *yymsg; + int yyi = 0; + while ((*yyp = *yyformat) != '\0') + if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount) + { + yyp += yytnamerr (yyp, yytname[yyarg[yyi++]]); + yyformat += 2; + } + else + { + ++yyp; + ++yyformat; + } + } + return 0; +} + +<%# b4_yydestruct_define %> +/*-----------------------------------------------. +| Release the memory associated to this symbol. | +`-----------------------------------------------*/ + +static void +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep, YYLTYPE *yylocationp<%= output.user_formals %>) +{ +<%= output.parse_param_use("yyvaluep", "yylocationp") %> + if (!yymsg) + yymsg = "Deleting"; + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp<%= output.user_args %>); + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + switch (yykind) + { +<%= output.symbol_actions_for_destructor -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + + +<%- if output.error_recovery -%> +#ifndef YYMAXREPAIR +# define YYMAXREPAIR(<%= output.parse_param_name %>) (3) +#endif + +#ifndef YYERROR_RECOVERY_ENABLED +# define YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>) (1) +#endif + +enum yy_repair_type { + inserting, + deleting, + shifting, +}; + +struct yy_repair { + enum yy_repair_type type; + yysymbol_kind_t term; +}; +typedef struct yy_repair yy_repair; + +struct yy_repairs { + /* For debug */ + int id; + /* For breadth-first traversing */ + struct yy_repairs *next; + YYPTRDIFF_T stack_length; + /* Bottom of states */ + yy_state_t *states; + /* Top of states */ + yy_state_t *state; + /* repair length */ + int repair_length; + /* */ + struct yy_repairs *prev_repair; + struct yy_repair repair; +}; +typedef struct yy_repairs yy_repairs; + +struct yy_term { + yysymbol_kind_t kind; + YYSTYPE value; + YYLTYPE location; +}; +typedef struct yy_term yy_term; + +struct yy_repair_terms { + int id; + int length; + yy_term terms[]; +}; +typedef struct yy_repair_terms yy_repair_terms; + +static void +yy_error_token_initialize (yysymbol_kind_t yykind, YYSTYPE * const yyvaluep, YYLTYPE * const yylocationp<%= output.user_formals %>) +{ + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +switch (yykind) + { +<%= output.symbol_actions_for_error_token -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + +static yy_repair_terms * +yy_create_repair_terms(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + yy_repair_terms *rep_terms; + int count = 0; + + while (r->prev_repair) + { + count++; + r = r->prev_repair; + } + + rep_terms = (yy_repair_terms *) YYMALLOC (sizeof (yy_repair_terms) + sizeof (yy_term) * count); + rep_terms->id = reps->id; + rep_terms->length = count; + + r = reps; + while (r->prev_repair) + { + rep_terms->terms[count-1].kind = r->repair.term; + count--; + r = r->prev_repair; + } + + return rep_terms; +} + +static void +yy_print_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + + YYDPRINTF ((stderr, + "id: %d, repair_length: %d, repair_state: %d, prev_repair_id: %d\n", + reps->id, reps->repair_length, *reps->state, reps->prev_repair->id)); + + while (r->prev_repair) + { + YYDPRINTF ((stderr, "%s ", yysymbol_name (r->repair.term))); + r = r->prev_repair; + } + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_print_repair_terms(yy_repair_terms *rep_terms<%= output.user_formals %>) +{ + for (int i = 0; i < rep_terms->length; i++) + YYDPRINTF ((stderr, "%s ", yysymbol_name (rep_terms->terms[i].kind))); + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_free_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + while (reps) + { + yy_repairs *r = reps; + reps = reps->next; + YYFREE (r->states); + YYFREE (r); + } +} + +static int +yy_process_repairs(yy_repairs *reps, yysymbol_kind_t token) +{ + int yyn; + int yystate = *reps->state; + int yylen = 0; + yysymbol_kind_t yytoken = token; + + goto yyrecover_backup; + +yyrecover_newstate: + // TODO: check reps->stack_length + reps->state += 1; + *reps->state = (yy_state_t) yystate; + + +yyrecover_backup: + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yyrecover_default; + + /* "Reading a token" */ + if (yytoken == YYSYMBOL_YYEMPTY) + return 1; + + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yyrecover_default; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyrecover_errlab; + yyn = -yyn; + goto yyrecover_reduce; + } + + /* shift */ + yystate = yyn; + yytoken = YYSYMBOL_YYEMPTY; + goto yyrecover_newstate; + + +yyrecover_default: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyrecover_errlab; + goto yyrecover_reduce; + + +yyrecover_reduce: + yylen = yyr2[yyn]; + /* YYPOPSTACK */ + reps->state -= yylen; + yylen = 0; + + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *reps->state; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *reps->state + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yyrecover_newstate; + +yyrecover_errlab: + return 0; +} + +static yy_repair_terms * +yyrecover(yy_state_t *yyss, yy_state_t *yyssp, int yychar<%= output.user_formals %>) +{ + yysymbol_kind_t yytoken = YYTRANSLATE (yychar); + yy_repair_terms *rep_terms = YY_NULLPTR; + int count = 0; + + yy_repairs *head = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + yy_repairs *current = head; + yy_repairs *tail = head; + YYPTRDIFF_T stack_length = yyssp - yyss + 1; + + head->id = count; + head->next = 0; + head->stack_length = stack_length; + head->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + head->state = head->states + (yyssp - yyss); + YYCOPY (head->states, yyss, stack_length); + head->repair_length = 0; + head->prev_repair = 0; + + stack_length = (stack_length * 2 > 100) ? (stack_length * 2) : 100; + count++; + + while (current) + { + int yystate = *current->state; + int yyn = yypact[yystate]; + /* See also: yypcontext_expected_tokens */ + if (!yypact_value_is_default (yyn)) + { + int yyxbegin = yyn < 0 ? -yyn : 0; + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + { + if (yyx != YYSYMBOL_YYerror) + { + if (current->repair_length + 1 > YYMAXREPAIR(<%= output.parse_param_name %>)) + continue; + + yy_repairs *reps = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + reps->id = count; + reps->next = 0; + reps->stack_length = stack_length; + reps->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + reps->state = reps->states + (current->state - current->states); + YYCOPY (reps->states, current->states, current->state - current->states + 1); + reps->repair_length = current->repair_length + 1; + reps->prev_repair = current; + reps->repair.type = inserting; + reps->repair.term = (yysymbol_kind_t) yyx; + + /* Process PDA assuming next token is yyx */ + if (! yy_process_repairs (reps, (yysymbol_kind_t)yyx)) + { + YYFREE (reps); + continue; + } + + tail->next = reps; + tail = reps; + count++; + + if (yyx == yytoken) + { + rep_terms = yy_create_repair_terms (current<%= output.user_args %>); + YYDPRINTF ((stderr, "repair_terms found. id: %d, length: %d\n", rep_terms->id, rep_terms->length)); + yy_print_repairs (current<%= output.user_args %>); + yy_print_repair_terms (rep_terms<%= output.user_args %>); + + goto done; + } + + YYDPRINTF ((stderr, + "New repairs is enqueued. count: %d, yystate: %d, yyx: %d\n", + count, yystate, yyx)); + yy_print_repairs (reps<%= output.user_args %>); + } + } + } + + current = current->next; + } + +done: + + yy_free_repairs(head<%= output.user_args %>); + + if (!rep_terms) + { + YYDPRINTF ((stderr, "repair_terms not found\n")); + } + + return rep_terms; +} +<%- end -%> + + + +/*----------. +| yyparse. | +`----------*/ + +int +yyparse (<%= output.parse_param %>) +{ +<%# b4_declare_scanner_communication_variables -%> +/* Lookahead token kind. */ +int yychar; + + +/* The semantic value of the lookahead symbol. */ +/* Default value used for initialization, for pacifying older GCCs + or non-GCC compilers. */ +#ifdef __cplusplus +static const YYSTYPE yyval_default = {}; +(void) yyval_default; +#else +YY_INITIAL_VALUE (static const YYSTYPE yyval_default;) +#endif +YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); + +/* Location data for the lookahead symbol. */ +static const YYLTYPE yyloc_default +# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + = { 1, 1, 1, 1 } +# endif +; +YYLTYPE yylloc = yyloc_default; + +<%# b4_declare_parser_state_variables -%> + /* Number of syntax errors so far. */ + int yynerrs = 0; + YY_USE (yynerrs); /* Silence compiler warning. */ + + yy_state_fast_t yystate = 0; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus = 0; + + /* Refer to the stacks through separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ + + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; + + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; + + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; + + /* The location stack: array, bottom, top. */ + YYLTYPE yylsa[YYINITDEPTH]; + YYLTYPE *yyls = yylsa; + YYLTYPE *yylsp = yyls; + + int yyn; + /* The return value of yyparse. */ + int yyresult; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; + YYLTYPE yyloc; + + /* The locations where the error started and ended. */ + YYLTYPE yyerror_range[3]; +<%- if output.error_recovery -%> + yy_repair_terms *rep_terms = 0; + yy_term term_backup; + int rep_terms_index; + int yychar_backup; +<%- end -%> + + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char *yymsg = yymsgbuf; + YYPTRDIFF_T yymsg_alloc = sizeof yymsgbuf; + +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N), yylsp -= (N)) + + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; + + YYDPRINTF ((stderr, "Starting parse\n")); + + yychar = YYEMPTY; /* Cause a token to be read. */ + + +<%# b4_user_initial_action -%> +<%= output.user_initial_action("/* User initialization code. */") %> +#line [@oline@] [@ofile@] + + yylsp[0] = yylloc; + goto yysetstate; + + +/*------------------------------------------------------------. +| yynewstate -- push a new state, which is found in yystate. | +`------------------------------------------------------------*/ +yynewstate: + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; + + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + + if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + YYNOMEM; +#else + { + /* Get the current used size of the three stacks, in elements. */ + YYPTRDIFF_T yysize = yyssp - yyss + 1; + +# if defined yyoverflow + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + yy_state_t *yyss1 = yyss; + YYSTYPE *yyvs1 = yyvs; + YYLTYPE *yyls1 = yyls; + + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow (YY_("memory exhausted"), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), + &yyls1, yysize * YYSIZEOF (*yylsp), + &yystacksize); + yyss = yyss1; + yyvs = yyvs1; + yyls = yyls1; + } +# else /* defined YYSTACK_RELOCATE */ + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + YYNOMEM; + yystacksize *= 2; + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yy_state_t *yyss1 = yyss; + union yyalloc *yyptr = + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); + if (! yyptr) + YYNOMEM; + YYSTACK_RELOCATE (yyss_alloc, yyss); + YYSTACK_RELOCATE (yyvs_alloc, yyvs); + YYSTACK_RELOCATE (yyls_alloc, yyls); +# undef YYSTACK_RELOCATE + if (yyss1 != yyssa) + YYSTACK_FREE (yyss1); + } +# endif + + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; + yylsp = yyls + yysize - 1; + + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END + + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ + + + if (yystate == YYFINAL) + YYACCEPT; + + goto yybackup; + + +/*-----------. +| yybackup. | +`-----------*/ +yybackup: + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ + + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yydefault; + + /* Not known => get a lookahead token if don't already have one. */ + +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + if (yychar == YYEMPTY && rep_terms) + { + + if (rep_terms_index < rep_terms->length) + { + YYDPRINTF ((stderr, "An error recovery token is used\n")); + yy_term term = rep_terms->terms[rep_terms_index]; + yytoken = term.kind; + yylval = term.value; + yylloc = term.location; + yychar = yytranslate_inverted[yytoken]; + YY_SYMBOL_PRINT ("Next error recovery token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + rep_terms_index++; + } + else + { + YYDPRINTF ((stderr, "Error recovery is completed\n")); + yytoken = term_backup.kind; + yylval = term_backup.value; + yylloc = term_backup.location; + yychar = yychar_backup; + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + + YYFREE (rep_terms); + rep_terms = 0; + yychar_backup = 0; + } + } + } +<%- end -%> + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token\n")); + yychar = yylex <%= output.yylex_formals %>; + } + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + yychar = <%= output.eof_symbol.id.s_value %>; + yytoken = <%= output.eof_symbol.enum_name %>; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else if (yychar == <%= output.error_symbol.id.s_value %>) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = <%= output.undef_symbol.id.s_value %>; + yytoken = <%= output.error_symbol.enum_name %>; + yyerror_range[1] = yylloc; + goto yyerrlab1; + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyerrlab; + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc<%= output.user_args %>); + yystate = yyn; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + *++yylsp = yylloc; +<%= output.after_shift_function("/* %after-shift code. */") %> + + /* Discard the shifted token. */ + yychar = YYEMPTY; + goto yynewstate; + + +/*-----------------------------------------------------------. +| yydefault -- do the default action for the current state. | +`-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + +/*-----------------------------. +| yyreduce -- do a reduction. | +`-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + '$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1-yylen]; +<%= output.before_reduce_function("/* %before-reduce function. */") %> + + /* Default location. */ + YYLLOC_DEFAULT (yyloc, (yylsp - yylen), yylen); + yyerror_range[1] = yyloc; + YY_REDUCE_PRINT (yyn<%= output.user_args %>); + switch (yyn) + { +<%= output.user_actions -%> + + default: break; + } + /* User semantic actions sometimes alter yychar, and that requires + that yytoken be updated with the new translation. We take the + approach of translating immediately before every use of yytoken. + One alternative is translating here after every semantic action, + but that translation would be missed if the semantic action invokes + YYABORT, YYACCEPT, or YYERROR immediately after altering yychar or + if it invokes YYBACKUP. In the case of YYABORT or YYACCEPT, an + incorrect destructor might then be invoked immediately. In the + case of YYERROR or YYBACKUP, subsequent parser actions might lead + to an incorrect destructor call or verbose syntax error message + before the lookahead is translated. */ + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc<%= output.user_args %>); + + YYPOPSTACK (yylen); +<%= output.after_reduce_function("/* %after-reduce function. */") %> + yylen = 0; + + *++yyvsp = yyval; + *++yylsp = yyloc; + + /* Now 'shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yynewstate; + + +/*--------------------------------------. +| yyerrlab -- here on detecting error. | +`--------------------------------------*/ +yyerrlab: + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; + { + yypcontext_t yyctx + = {yyssp, yytoken, &yylloc}; + char const *yymsgp = YY_("syntax error"); + int yysyntax_error_status; + yysyntax_error_status = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + if (yysyntax_error_status == 0) + yymsgp = yymsg; + else if (yysyntax_error_status == -1) + { + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + yymsg = YY_CAST (char *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, yymsg_alloc))); + if (yymsg) + { + yysyntax_error_status + = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + yymsgp = yymsg; + } + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + yysyntax_error_status = YYENOMEM; + } + } + yyerror (<%= output.yyerror_args %>, yymsgp); + if (yysyntax_error_status == YYENOMEM) + YYNOMEM; + } + } + + yyerror_range[1] = yylloc; + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + /* Return failure if at end of input. */ + if (yychar == <%= output.eof_symbol.id.s_value %>) + YYABORT; + } + else + { + yydestruct ("Error: discarding", + yytoken, &yylval, &yylloc<%= output.user_args %>); + yychar = YYEMPTY; + } + } + + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; + + +/*---------------------------------------------------. +| yyerrorlab -- error raised explicitly by YYERROR. | +`---------------------------------------------------*/ +yyerrorlab: + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; + ++yynerrs; + + /* Do not reclaim the symbols of the rule whose action triggered + this YYERROR. */ + YYPOPSTACK (yylen); +<%= output.after_pop_stack_function("yylen", "/* %after-pop-stack function. */") %> + yylen = 0; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + yystate = *yyssp; + goto yyerrlab1; + + +/*-------------------------------------------------------------. +| yyerrlab1 -- common code for both syntax error and YYERROR. | +`-------------------------------------------------------------*/ +yyerrlab1: +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + rep_terms = yyrecover (yyss, yyssp, yychar<%= output.user_args %>); + if (rep_terms) + { + for (int i = 0; i < rep_terms->length; i++) + { + yy_term *term = &rep_terms->terms[i]; + yy_error_token_initialize (term->kind, &term->value, &term->location<%= output.user_args %>); + } + + yychar_backup = yychar; + /* Can be packed into (the tail of) rep_terms? */ + term_backup.kind = yytoken; + term_backup.value = yylval; + term_backup.location = yylloc; + rep_terms_index = 0; + yychar = YYEMPTY; + + goto yybackup; + } + } +<%- end -%> + yyerrstatus = 3; /* Each real token shifted decrements this. */ + + /* Pop stack until we find a state that shifts the error token. */ + for (;;) + { + yyn = yypact[yystate]; + if (!yypact_value_is_default (yyn)) + { + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) + { + yyn = yytable[yyn]; + if (0 < yyn) + break; + } + } + + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + + yyerror_range[1] = *yylsp; + yydestruct ("Error: popping", + YY_ACCESSING_SYMBOL (yystate), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); +<%= output.after_pop_stack_function(1, "/* %after-pop-stack function. */") %> + yystate = *yyssp; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + } + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + + yyerror_range[2] = yylloc; + ++yylsp; + YYLLOC_DEFAULT (*yylsp, yyerror_range, 2); + + /* Shift the error token. */ + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp<%= output.user_args %>); +<%= output.after_shift_error_token_function("/* %after-shift-error-token code. */") %> + + yystate = yyn; + goto yynewstate; + + +/*-------------------------------------. +| yyacceptlab -- YYACCEPT comes here. | +`-------------------------------------*/ +yyacceptlab: + yyresult = 0; + goto yyreturnlab; + + +/*-----------------------------------. +| yyabortlab -- YYABORT comes here. | +`-----------------------------------*/ +yyabortlab: + yyresult = 1; + goto yyreturnlab; + + +/*-----------------------------------------------------------. +| yyexhaustedlab -- YYNOMEM (memory exhaustion) comes here. | +`-----------------------------------------------------------*/ +yyexhaustedlab: + yyerror (<%= output.yyerror_args %>, YY_("memory exhausted")); + yyresult = 2; + goto yyreturnlab; + + +/*----------------------------------------------------------. +| yyreturnlab -- parsing is finished, clean up and return. | +`----------------------------------------------------------*/ +yyreturnlab: + if (yychar != YYEMPTY) + { + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = YYTRANSLATE (yychar); + yydestruct ("Cleanup: discarding lookahead", + yytoken, &yylval, &yylloc<%= output.user_args %>); + } + /* Do not reclaim the symbols of the rule whose action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK (yylen); + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + while (yyssp != yyss) + { + yydestruct ("Cleanup: popping", + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); + } +#ifndef yyoverflow + if (yyss != yyssa) + YYSTACK_FREE (yyss); +#endif + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + return yyresult; +} + +<%# b4_percent_code_get([[epilogue]]) -%> +<%- if output.aux.epilogue -%> +#line <%= output.aux.epilogue_first_lineno - 1 %> "<%= output.grammar_file_path %>" +<%= output.aux.epilogue -%> +<%- end -%> + diff --git a/tool/lrama/template/bison/yacc.h b/tool/lrama/template/bison/yacc.h new file mode 100644 index 0000000000..848dbf5961 --- /dev/null +++ b/tool/lrama/template/bison/yacc.h @@ -0,0 +1,40 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison interface for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ +<%= output.render_partial("bison/_yacc.h") %> diff --git a/tool/lrama/template/diagram/diagram.html b/tool/lrama/template/diagram/diagram.html new file mode 100644 index 0000000000..3e87e6e519 --- /dev/null +++ b/tool/lrama/template/diagram/diagram.html @@ -0,0 +1,102 @@ +<!DOCTYPE html> +<html> +<head> + <title>Lrama syntax diagrams</title> + + <style> + <%= output.default_style %> + .diagram-header { + display: inline-block; + font-weight: bold; + font-size: 18px; + margin-bottom: -8px; + text-align: center; + } + + svg { + width: 100%; + } + + svg.railroad-diagram g.non-terminal text { + cursor: pointer; + } + + h2.hover-header { + background-color: #90ee90; + } + + svg.railroad-diagram g.non-terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + + svg.railroad-diagram g.terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + </style> +</head> + +<body align="center"> + <%= output.diagrams %> + <script> + document.addEventListener("DOMContentLoaded", () => { + function addHoverEffect(selector, hoverClass, relatedSelector, relatedHoverClass, getTextElements) { + document.querySelectorAll(selector).forEach(element => { + element.addEventListener("mouseenter", () => { + element.classList.add(hoverClass); + getTextElements(element).forEach(textEl => { + if (!relatedSelector) return; + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.add(relatedHoverClass); + }); + }); + }); + + element.addEventListener("mouseleave", () => { + element.classList.remove(hoverClass); + if (!relatedSelector) return; + getTextElements(element).forEach(textEl => { + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.remove(relatedHoverClass); + }); + }); + }); + }); + } + + function getElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => el.textContent.trim() === text.trim()); + } + + function getParentElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => + [...el.querySelectorAll("text")].some(textEl => textEl.textContent.trim() === text.trim()) + ); + } + + function scrollToMatchingHeader() { + document.querySelectorAll("g.non-terminal").forEach(element => { + element.addEventListener("click", () => { + const textElements = [...element.querySelectorAll("text")]; + for (const textEl of textElements) { + const targetHeader = getElementsByText("h2", textEl.textContent)[0]; + if (targetHeader) { + targetHeader.scrollIntoView({ behavior: "smooth", block: "start" }); + break; + } + } + }); + }); + } + + addHoverEffect("h2", "hover-header", "g.non-terminal", "hover-g", element => [element]); + addHoverEffect("g.non-terminal", "hover-g", "h2", "hover-header", + element => [...element.querySelectorAll("text")] + ); + addHoverEffect("g.terminal", "hover-g", "", "", element => [element]); + scrollToMatchingHeader(); + }); + </script> +</body> +</html> diff --git a/tool/m4/ruby_append_option.m4 b/tool/m4/ruby_append_option.m4 index ff828d2162..8cd2741ae8 100644 --- a/tool/m4/ruby_append_option.m4 +++ b/tool/m4/ruby_append_option.m4 @@ -3,3 +3,7 @@ AC_DEFUN([RUBY_APPEND_OPTION], [# RUBY_APPEND_OPTION($1) AS_CASE([" [$]{$1-} "], [*" $2 "*], [], [' '], [ $1="$2"], [ $1="[$]$1 $2"])])dnl +AC_DEFUN([RUBY_PREPEND_OPTION], + [# RUBY_PREPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_check_builtin_overflow.m4 b/tool/m4/ruby_check_builtin_overflow.m4 new file mode 100644 index 0000000000..8568d2c6d9 --- /dev/null +++ b/tool/m4/ruby_check_builtin_overflow.m4 @@ -0,0 +1,28 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_OVERFLOW], [dnl +{ # $0($1) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow, [int x;__builtin_[$1]_overflow(0,0,&x)]) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow_p, [__builtin_[$1]_overflow_p(0,0,(int)0)]) + + AS_IF([test "$rb_cv_builtin___builtin_[$1]_overflow" != no], [ + AC_CACHE_CHECK(for __builtin_[$1]_overflow with long long arguments, rb_cv_use___builtin_[$1]_overflow_long_long, [ + AC_LINK_IFELSE([AC_LANG_SOURCE([[ +@%:@pragma clang optimize off + +int +main(void) +{ + long long x = 0, y; + __builtin_$1_overflow(x, x, &y); + + return 0; +} +]])], + rb_cv_use___builtin_[$1]_overflow_long_long=yes, + rb_cv_use___builtin_[$1]_overflow_long_long=no)]) + ]) + AS_IF([test "$rb_cv_use___builtin_[$1]_overflow_long_long" = yes], [ + AC_DEFINE(USE___BUILTIN_[]AS_TR_CPP($1)_OVERFLOW_LONG_LONG, 1) + ]) +} +])dnl diff --git a/tool/m4/ruby_check_builtin_setjmp.m4 b/tool/m4/ruby_check_builtin_setjmp.m4 index 05118e2243..1e5d9b3028 100644 --- a/tool/m4/ruby_check_builtin_setjmp.m4 +++ b/tool/m4/ruby_check_builtin_setjmp.m4 @@ -20,7 +20,7 @@ AC_CACHE_CHECK(for __builtin_setjmp, ac_cv_func___builtin_setjmp, void (*volatile f)(void) = t; if (!jump()) printf("%d\n", f != 0); ]])], - [ac_cv_func___builtin_setjmp="yes with cast ($cast)"]) + [ac_cv_func___builtin_setjmp="yes${cast:+ with cast ($cast)}"]) ]) test "$ac_cv_func___builtin_setjmp" = no || break done]) diff --git a/tool/m4/ruby_check_header.m4 b/tool/m4/ruby_check_header.m4 new file mode 100644 index 0000000000..6fec9d16c5 --- /dev/null +++ b/tool/m4/ruby_check_header.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_HEADER], + [# RUBY_CHECK_HEADER($@) + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS m4_if([$5], [], [$INCFLAGS], [$5])" + AC_CHECK_HEADERS([$1], [$2], [$3], [$4]) + CPPFLAGS="$save_CPPFLAGS" + unset save_CPPFLAGS]) diff --git a/tool/m4/ruby_default_arch.m4 b/tool/m4/ruby_default_arch.m4 index 03e52f7776..2f25ba81ee 100644 --- a/tool/m4/ruby_default_arch.m4 +++ b/tool/m4/ruby_default_arch.m4 @@ -1,11 +1,21 @@ dnl -*- Autoconf -*- AC_DEFUN([RUBY_DEFAULT_ARCH], [ +# Set ARCH_FLAG for different width but family CPU AC_MSG_CHECKING([arch option]) -AS_CASE([$1], - [arm64], [], - [*64], [ARCH_FLAG=-m64], - [[i[3-6]86]], [ARCH_FLAG=-m32], - [AC_MSG_ERROR(unknown target architecture: $target_archs)] - ) -AC_MSG_RESULT([$ARCH_FLAG]) +AS_CASE([$1:"$host_cpu"], + [arm64:arm*], [ARCH_FLAG=-m64], + [arm*:arm*], [ARCH_FLAG=-m32], + [x86_64:[i[3-6]86]], [ARCH_FLAG=-m64], + [x64:x86_64], [], + [[i[3-6]86]:x86_64], [ARCH_FLAG=-m32], + [ppc64:ppc*], [ARCH_FLAG=-m64], + [ppc*:ppc64], [ARCH_FLAG=-m32], + [ + ARCH_FLAG= + for flag in "-arch "$1 -march=$1; do + _RUBY_TRY_CFLAGS([$]flag, [ARCH_FLAG="[$]flag"]) + test x"$ARCH_FLAG" = x || break + done] +) +AC_MSG_RESULT([${ARCH_FLAG:-'(none)'}]) ])dnl diff --git a/tool/m4/ruby_define_if.m4 b/tool/m4/ruby_define_if.m4 index 1cfd07b0c8..aba55783a2 100644 --- a/tool/m4/ruby_define_if.m4 +++ b/tool/m4/ruby_define_if.m4 @@ -1,12 +1,6 @@ dnl -*- Autoconf -*- AC_DEFUN([RUBY_DEFINE_IF], [dnl - m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])cat <<EOH >> confdefs.h -@%:@if $1 -EOH -])dnl + m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])printf "@%:@if %s\n" "$1" >>confdefs.h]) AC_DEFINE_UNQUOTED($2, $3)dnl - m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])cat <<EOH >> confdefs.h -@%:@endif /* $1 */ -EOH -])dnl + m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])printf "@%:@endif /* %s */\n" "$1" >>confdefs.h]) ])dnl diff --git a/tool/m4/ruby_defint.m4 b/tool/m4/ruby_defint.m4 index e9ed68e5b8..7f262a73fc 100644 --- a/tool/m4/ruby_defint.m4 +++ b/tool/m4/ruby_defint.m4 @@ -17,7 +17,8 @@ typedef $1 t; int s = sizeof(t) == 42;])], ["${ac_cv_sizeof___int128@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int128"], [ rb_cv_type_$1=no])])]) AS_IF([test "${rb_cv_type_$1}" != no], [ - type="${rb_cv_type_$1@%:@@%:@unsigned }" + type="${rb_cv_type_$1@%:@@%:@*signed }" + AS_IF([test "$type" = "long long"], [type=long_long]) AS_IF([test "$type" != yes && eval 'test -n "${ac_cv_sizeof_'$type'+set}"'], [ eval cond='"${ac_cv_sizeof_'$type'}"' AS_CASE([$cond], [*:*], [ diff --git a/tool/m4/ruby_modular_gc.m4 b/tool/m4/ruby_modular_gc.m4 new file mode 100644 index 0000000000..661fce2e60 --- /dev/null +++ b/tool/m4/ruby_modular_gc.m4 @@ -0,0 +1,41 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_MODULAR_GC],[ +AC_ARG_WITH(modular-gc, + AS_HELP_STRING([--with-modular-gc=DIR], + [Enable replacement of Ruby's GC from a modular library in the specified directory.]), + [modular_gc_dir=$withval], [unset modular_gc_dir] +) + +AS_IF([test "$modular_gc_dir" = yes], [ + AC_MSG_ERROR(you must specify a directory when using --with-modular-gc) +]) + +AC_MSG_CHECKING([if building with modular GC support]) +AS_IF([test x"$modular_gc_dir" != x], [ + AC_MSG_RESULT([yes]) + + # Ensure that modular_gc_dir is always an absolute path so that Ruby + # never loads a modular GC from a relative path + AS_CASE(["$modular_gc_dir"], + [/*], [], + [test "$load_relative" = yes || modular_gc_dir="$prefix/$modular_gc_dir"] + ) + + # Ensure that modular_gc_dir always terminates with a / + AS_CASE(["$modular_gc_dir"], + [*/], [], + [modular_gc_dir="$modular_gc_dir/"] + ) + + AC_DEFINE([USE_MODULAR_GC], [1]) + + modular_gc_summary="yes (in $modular_gc_dir)" +], [ + AC_MSG_RESULT([no]) + AC_DEFINE([USE_MODULAR_GC], [0]) + + modular_gc_summary="no" +]) + +AC_SUBST(modular_gc_dir, "${modular_gc_dir}") +])dnl diff --git a/tool/m4/ruby_prog_makedirs.m4 b/tool/m4/ruby_prog_makedirs.m4 new file mode 100644 index 0000000000..132ff7768f --- /dev/null +++ b/tool/m4/ruby_prog_makedirs.m4 @@ -0,0 +1,9 @@ +dnl -*- Autoconf -*- +m4_defun([RUBY_PROG_MAKEDIRS], + [m4_bpatsubst(m4_defn([AC_PROG_MKDIR_P]), + [MKDIR_P=\"$ac_install_sh -d\"], [ + AS_IF([test "x$MKDIR_P" = "xfalse"], [AC_MSG_ERROR([mkdir -p is required])]) + MKDIR_P="mkdir -p"]) + ]dnl + AC_SUBST(MAKEDIRS, ["$MKDIR_P"]) +) diff --git a/tool/m4/ruby_replace_funcs.m4 b/tool/m4/ruby_replace_funcs.m4 index 10e85f1ac9..33b3ca70c1 100644 --- a/tool/m4/ruby_replace_funcs.m4 +++ b/tool/m4/ruby_replace_funcs.m4 @@ -8,6 +8,6 @@ AC_DEFUN([RUBY_REPLACE_FUNC], [dnl ])dnl dnl dnl RUBY_REPLACE_FUNCS [funcs] [included] -AC_DEFUN([RUBY_REPLACE_FUNCS] [dnl - m4_map_args_w([$1], [RUBY_REPLACE_FUNC(], [), [$2]])dnl +AC_DEFUN([RUBY_REPLACE_FUNCS], [dnl + m4_map_args_w([$1], [RUBY_REPLACE_FUNC(], [, [$2])])dnl ])dnl diff --git a/tool/m4/ruby_replace_type.m4 b/tool/m4/ruby_replace_type.m4 index 70674b6cc7..3df0f3994f 100644 --- a/tool/m4/ruby_replace_type.m4 +++ b/tool/m4/ruby_replace_type.m4 @@ -6,6 +6,13 @@ AC_DEFUN([RUBY_REPLACE_TYPE], [dnl [n="patsubst([$2],["],[\\"])"], [$4]) AC_CACHE_CHECK([for convertible type of [$1]], rb_cv_[$1]_convertible, [ + AC_COMPILE_IFELSE( + [AC_LANG_BOOL_COMPILE_TRY([AC_INCLUDES_DEFAULT([$4])] + [typedef $n rbcv_conftest_target_type; + extern rbcv_conftest_target_type rbcv_conftest_var; + ], [sizeof(&*rbcv_conftest_var)])], + [rb_cv_[$1]_convertible=PTR], + [ u= t= AS_CASE(["$n "], [*" signed "*], [ ], @@ -37,6 +44,7 @@ AC_DEFUN([RUBY_REPLACE_TYPE], [dnl [ t=INT]) rb_cv_[$1]_convertible=${u}${t}]) + ]) AS_IF([test "${AS_TR_SH(ac_cv_type_[$1])}" = "yes"], [ n="$1" ], [ @@ -48,11 +56,13 @@ AC_DEFUN([RUBY_REPLACE_TYPE], [dnl AS_CASE(["${rb_cv_[$1]_convertible}"], [U*], [n="unsigned $n"]) ]) - AS_CASE("${rb_cv_[$1]_convertible}", [U*], [u=+1], [u=-1]) + AS_CASE("${rb_cv_[$1]_convertible}", [PTR], [u=], [U*], [u=+1], [u=-1]) AC_DEFINE_UNQUOTED(rb_[$1], $n) + AS_IF([test $u], [ AC_DEFINE_UNQUOTED([SIGNEDNESS_OF_]AS_TR_CPP($1), $u) AC_DEFINE_UNQUOTED([$3]2NUM[(v)], [${rb_cv_[$1]_convertible}2NUM(v)]) AC_DEFINE_UNQUOTED(NUM2[$3][(v)], [NUM2${rb_cv_[$1]_convertible}(v)]) AC_DEFINE_UNQUOTED(PRI_[$3]_PREFIX, [PRI_`echo ${rb_cv_[$1]_convertible} | sed ['s/^U//']`_PREFIX]) + ]) ])dnl diff --git a/tool/m4/ruby_require_funcs.m4 b/tool/m4/ruby_require_funcs.m4 new file mode 100644 index 0000000000..78372d57d8 --- /dev/null +++ b/tool/m4/ruby_require_funcs.m4 @@ -0,0 +1,13 @@ +dnl -*- Autoconf -*- +dnl RUBY_REQUIRE_FUNC [func] [included] +AC_DEFUN([RUBY_REQUIRE_FUNC], [ +# RUBY_REQUIRE_FUNC([$1], [$2]) + AC_CHECK_FUNCS([$1]) + AS_IF([test "$ac_cv_func_[]AS_TR_SH($1)" = yes], [], + [AC_MSG_ERROR($1[() must be supported])]) +])dnl +dnl +dnl RUBY_REQUIRE_FUNCS [funcs] [included] +AC_DEFUN([RUBY_REQUIRE_FUNCS], [dnl + m4_map_args_w([$1], [RUBY_REQUIRE_FUNC(], [, [$2])])dnl +])dnl diff --git a/tool/m4/ruby_setjmp_type.m4 b/tool/m4/ruby_setjmp_type.m4 index 4ae26fe5cd..d26af34ea0 100644 --- a/tool/m4/ruby_setjmp_type.m4 +++ b/tool/m4/ruby_setjmp_type.m4 @@ -3,18 +3,15 @@ AC_DEFUN([RUBY_SETJMP_TYPE], [ RUBY_CHECK_BUILTIN_SETJMP RUBY_CHECK_SETJMP(_setjmpex, [], [@%:@include <setjmpex.h>]) RUBY_CHECK_SETJMP(_setjmp) -RUBY_CHECK_SETJMP(sigsetjmp, [sigjmp_buf]) AC_MSG_CHECKING(for setjmp type) setjmp_suffix= -unset setjmp_sigmask AC_ARG_WITH(setjmp-type, - AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), + AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), [ AS_CASE([$withval], [__builtin_setjmp], [setjmp=__builtin_setjmp], [_setjmp], [ setjmp_prefix=_], - [sigsetjmp,*], [ setjmp_prefix=sig setjmp_sigmask=`expr "$withval" : 'sigsetjmp\(,.*\)'`], - [sigsetjmp], [ setjmp_prefix=sig], + [sigsetjmp*], [ AC_MSG_WARN(No longer use sigsetjmp; use setjmp instead); setjmp_prefix=], [setjmp], [ setjmp_prefix=], [setjmpex], [ setjmp_prefix= setjmp_suffix=ex], [''], [ unset setjmp_prefix], @@ -34,19 +31,13 @@ AS_IF([test ${setjmp_prefix+set}], [ ], [test "$ac_cv_func__setjmp" = yes], [ setjmp_prefix=_ setjmp_suffix= -], [test "$ac_cv_func_sigsetjmp" = yes], [ - AS_CASE([$target_os],[solaris*|cygwin*],[setjmp_prefix=],[setjmp_prefix=sig]) - setjmp_suffix= ], [ setjmp_prefix= setjmp_suffix= ]) -AS_IF([test x$setjmp_prefix:$setjmp_sigmask = xsig:], [ - setjmp_sigmask=,0 -]) -AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}${setjmp_sigmask}) -AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env)${setjmp_sigmask})]) +AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}) +AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env))]) AC_DEFINE_UNQUOTED([RUBY_LONGJMP(env,val)], [${setjmp_prefix}longjmp($setjmp_cast(env),val)]) -AS_IF([test "(" "$GCC" != yes ")" -o x$setjmp_prefix != x__builtin_], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, ${setjmp_sigmask+${setjmp_prefix}}jmp_buf)) +AS_CASE(["$GCC:$setjmp_prefix"], [yes:__builtin_], [], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, jmp_buf)) AS_IF([test x$setjmp_suffix = xex], [AC_DEFINE_UNQUOTED(RUBY_USE_SETJMPEX, 1)]) ])dnl diff --git a/tool/m4/ruby_stack_grow_direction.m4 b/tool/m4/ruby_stack_grow_direction.m4 index a4d205cc3c..8c6fdd5722 100644 --- a/tool/m4/ruby_stack_grow_direction.m4 +++ b/tool/m4/ruby_stack_grow_direction.m4 @@ -3,7 +3,7 @@ AC_DEFUN([RUBY_STACK_GROW_DIRECTION], [ AS_VAR_PUSHDEF([stack_grow_dir], [rb_cv_stack_grow_dir_$1]) AC_CACHE_CHECK(stack growing direction on $1, stack_grow_dir, [ AS_CASE(["$1"], -[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*], [ $2=-1], +[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*|arm*|aarch*], [ $2=-1], [hppa*], [ $2=+1], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ diff --git a/tool/m4/ruby_thread.m4 b/tool/m4/ruby_thread.m4 new file mode 100644 index 0000000000..439c63bc22 --- /dev/null +++ b/tool/m4/ruby_thread.m4 @@ -0,0 +1,80 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_THREAD], [ +AC_ARG_WITH(thread, + AS_HELP_STRING([--with-thread=IMPLEMENTATION], [specify the thread implementation to use]), + [THREAD_MODEL=$withval], [ + THREAD_MODEL= + AS_CASE(["$target_os"], + [freebsd*], [ + AC_CACHE_CHECK([whether pthread should be enabled by default], + rb_cv_enable_pthread_default, + [AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ +@%:@include <osreldate.h> +@%:@if __FreeBSD_version < 502102 +@%:@error pthread should be disabled on this platform +@%:@endif + ]])], + rb_cv_enable_pthread_default=yes, + rb_cv_enable_pthread_default=no)]) + AS_IF([test $rb_cv_enable_pthread_default = yes], + [THREAD_MODEL=pthread], + [THREAD_MODEL=none]) + ], + [mingw*], [ + THREAD_MODEL=win32 + ], + [wasi*], [ + THREAD_MODEL=none + ], + [ + THREAD_MODEL=pthread + ] + ) +]) + +AS_IF([test x"$THREAD_MODEL" = xpthread], [ + AC_CHECK_HEADERS(pthread.h) + AS_IF([test x"$ac_cv_header_pthread_h" = xyes], [], [ + AC_MSG_WARN("Don't know how to find pthread header on your system -- thread support disabled") + THREAD_MODEL=none + ]) +]) +AS_IF([test x"$THREAD_MODEL" = xpthread], [ + THREAD_MODEL=none + for pthread_lib in thr pthread pthreads c c_r root; do + AC_CHECK_LIB($pthread_lib, pthread_create, + [THREAD_MODEL=pthread; break]) + done + AS_IF([test x"$THREAD_MODEL" = xpthread], [ + AC_DEFINE(_REENTRANT) + AC_DEFINE(_THREAD_SAFE) + AC_DEFINE(HAVE_LIBPTHREAD) + AC_CHECK_HEADERS(pthread_np.h, [], [], [@%:@include <pthread.h>]) + AS_CASE(["$pthread_lib:$target_os"], + [c:*], [], + [root:*], [], + [c_r:*|*:openbsd*|*:mirbsd*], [LIBS="-pthread $LIBS"], + [LIBS="-l$pthread_lib $LIBS"]) + ], [ + AC_MSG_WARN("Don't know how to find pthread library on your system -- thread support disabled") + ]) +]) + +AS_CASE(["$THREAD_MODEL"], +[pthread], [], +[win32], [], +[none], [], +[""], [AC_MSG_ERROR(thread model is missing)], + [AC_MSG_ERROR(unknown thread model $THREAD_MODEL)]) +AC_MSG_CHECKING(thread model) +AC_MSG_RESULT($THREAD_MODEL) + +THREAD_IMPL_H=thread_$THREAD_MODEL.h +AS_IF([test ! -f "$srcdir/$THREAD_IMPL_H"], + [AC_MSG_ERROR('$srcdir/$THREAD_IMPL_H' must exist)]) +THREAD_IMPL_SRC=thread_$THREAD_MODEL.c +AS_IF([test ! -f "$srcdir/$THREAD_IMPL_SRC"], + [AC_MSG_ERROR('$srcdir/$THREAD_IMPL_SRC' must exist)]) +AC_DEFINE_UNQUOTED(THREAD_IMPL_H, ["$THREAD_IMPL_H"]) +AC_DEFINE_UNQUOTED(THREAD_IMPL_SRC, ["$THREAD_IMPL_SRC"]) +])dnl diff --git a/tool/m4/ruby_try_cflags.m4 b/tool/m4/ruby_try_cflags.m4 index f2c6a3094e..b397642aad 100644 --- a/tool/m4/ruby_try_cflags.m4 +++ b/tool/m4/ruby_try_cflags.m4 @@ -1,12 +1,41 @@ dnl -*- Autoconf -*- -AC_DEFUN([RUBY_TRY_CFLAGS], [ - AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS]) +dnl +dnl Autoconf 2.67 fails to detect `-Werror=old-style-definition` due +dnl to the old style definition of `main`. +m4_version_prereq([2.70], [], [ +m4_defun([AC_LANG_PROGRAM(C)], m4_bpatsubst(m4_defn([AC_LANG_PROGRAM(C)]), [main ()], [main (void)])) +])dnl +dnl +AC_DEFUN([_RUBY_TRY_CFLAGS], [ RUBY_WERROR_FLAG([ CFLAGS="[$]CFLAGS $1" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS([$1], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)], + [$4], [$5]) +])dnl + +AC_DEFUN([_RUBY_TRY_CFLAGS_PREPEND], [ + RUBY_WERROR_FLAG([ + CFLAGS="$1 [$]CFLAGS" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS_PREPEND], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS_PREPEND([$1], [$2 AC_MSG_RESULT(yes)], [$3 - AC_MSG_RESULT(no)]) - ]) + AC_MSG_RESULT(no)], + [$4], [$5]) ])dnl diff --git a/tool/m4/ruby_universal_arch.m4 b/tool/m4/ruby_universal_arch.m4 index a2ee9e02be..d3e0dd0b47 100644 --- a/tool/m4/ruby_universal_arch.m4 +++ b/tool/m4/ruby_universal_arch.m4 @@ -17,7 +17,7 @@ AS_IF([test ${target_archs+set}], [ cpu=$archs cpu=`echo $cpu | sed 's/-.*-.*//'` universal_binary="${universal_binary+$universal_binary,}$cpu" - universal_archnames="${universal_archnames} ${archs}=${cpu}" + universal_archnames="${universal_archnames:+$universal_archnames }${archs}=${cpu}" ARCH_FLAG="${ARCH_FLAG+$ARCH_FLAG }-arch $archs" ]) done @@ -40,7 +40,7 @@ AS_IF([test ${target_archs+set}], [ AS_IF([$CC $CFLAGS $ARCH_FLAG -o conftest conftest.c > /dev/null 2>&1], [ rm -fr conftest.* ], [test -z "$ARCH_FLAG"], [ - RUBY_DEFAULT_ARCH("$target_archs") + RUBY_DEFAULT_ARCH($target_archs) ]) ]) target_cpu=${target_archs} @@ -73,7 +73,7 @@ EOF sed -n 's/^"processor-name=\(.*\)"/\1/p'` target="$target_cpu${target}" AC_MSG_RESULT([$target_cpu]) - ]) + ]) ]) target_archs="$target_cpu" ]) @@ -88,3 +88,35 @@ AS_IF([test "x${ARCH_FLAG}" != x], [ ]) # RUBY_UNIVERSAL_ARCH end ])dnl +dnl +AC_DEFUN([RUBY_UNIVERSAL_CHECK_HEADER_COND], [ dnl + AC_CACHE_CHECK([for $2 when $1], [$3], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [AC_INCLUDES_DEFAULT([$6])[ + @%:@if ]$1[ + @%:@include <]$2[> + @%:@endif]], [[]])], + [AS_VAR_SET($3, yes)], + [AS_VAR_SET($3, no)])) + AS_VAR_IF([$3], [yes], [dnl + printf "@%:@if %s\n" "$1" >>confdefs.h + AC_DEFINE_UNQUOTED(HAVE_[]AS_TR_CPP($2), 1)dnl + printf "@%:@endif\n" >>confdefs.h dnl + $4], [$5]) +])dnl +dnl +dnl RUBY_UNIVERSAL_CHECK_HEADER(CPU-LIST, HEADER, +dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND], +dnl [INCLUDES = DEFAULT-INCLUDES]) +AC_DEFUN([RUBY_UNIVERSAL_CHECK_HEADER], [ dnl + m4_if([$# dnl + ], [0], [], [ dnl + m4_foreach([rb_Header], [$1], + [AS_CASE([",$target_archs,"], [*,]rb_Header[,*], + [RUBY_UNIVERSAL_CHECK_HEADER_COND]([defined(__[]rb_Header[]__)], + [$2], [rb_cv_header_[]AS_TR_SH($2)_on_[]AS_TR_SH(rb_Header)], + [$3], [$4], [$5]) + ) + ]) + ])dnl +])dnl diff --git a/tool/m4/ruby_wasm_tools.m4 b/tool/m4/ruby_wasm_tools.m4 new file mode 100644 index 0000000000..efc017e771 --- /dev/null +++ b/tool/m4/ruby_wasm_tools.m4 @@ -0,0 +1,25 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_WASM_TOOLS], +[AS_CASE(["$target_os"], +[wasi*], [ + AC_CHECK_TOOL(WASMOPT, wasm-opt) + AS_IF([test x"${WASMOPT}" = x], [ + AC_MSG_ERROR([wasm-opt is required]) + ]) + AC_SUBST(wasmoptflags) + : ${wasmoptflags=-O3} + + AC_MSG_CHECKING([whether \$WASI_SDK_PATH is set]) + AS_IF([test x"${WASI_SDK_PATH}" = x], [ + AC_MSG_RESULT([no]) + AC_MSG_ERROR([WASI_SDK_PATH environment variable is required]) + ], [ + AC_MSG_RESULT([yes]) + CC="${CC:-${WASI_SDK_PATH}/bin/clang}" + LD="${LD:-${WASI_SDK_PATH}/bin/clang}" + AR="${AR:-${WASI_SDK_PATH}/bin/llvm-ar}" + RANLIB="${RANLIB:-${WASI_SDK_PATH}/bin/llvm-ranlib}" + OBJCOPY="${OBJCOPY:-${WASI_SDK_PATH}/bin/llvm-objcopy}" + ]) +]) +])dnl diff --git a/tool/make-snapshot b/tool/make-snapshot index 589d742e50..4af6a855eb 100755 --- a/tool/make-snapshot +++ b/tool/make-snapshot @@ -10,6 +10,7 @@ require 'fileutils' require 'shellwords' require 'tmpdir' require 'pathname' +require 'date' require 'yaml' require 'json' require File.expand_path("../lib/vcs", __FILE__) @@ -22,9 +23,9 @@ $keep_temp ||= nil $patch_file ||= nil $packages ||= nil $digests ||= nil +$no7z ||= nil $tooldir = File.expand_path("..", __FILE__) $unicode_version = nil if ($unicode_version ||= nil) == "" -$extlibs ||= "no" $colorize = Colorize.new def usage @@ -38,8 +39,6 @@ options: -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")}) -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")}) -unicode_version=VER Unicode version to generate encodings - -svn[=URL] make snapshot from SVN repository - (#{SVNURL}) -help, --help show this message version: master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL @@ -58,7 +57,7 @@ PACKAGES = { DEFAULT_PACKAGES = PACKAGES.keys - ["tar"] if !$no7z and system("7z", out: IO::NULL) PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" - PACKAGES["zip"] = %w".zip 7z a -tzip -l -mx -mtc=off" << {out: IO::NULL} + PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL} elsif gzip = ENV.delete("GZIP") PACKAGES["gzip"].concat(gzip.shellsplit) end @@ -69,21 +68,10 @@ if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?) end ENV["LC_ALL"] = ENV["LANG"] = "C" -SVNURL = URI.parse("https://svn.ruby-lang.org/repos/ruby/") # https git clone is disabled at git.ruby-lang.org/ruby.git. GITURL = URI.parse("https://github.com/ruby/ruby.git") RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ -ENV["VPATH"] ||= "include/ruby" -YACC = ENV["YACC"] ||= "bison" -ENV["BASERUBY"] ||= "ruby" -ENV["RUBY"] ||= "ruby" -ENV["MV"] ||= "mv" -ENV["RM"] ||= "rm -f" -ENV["MINIRUBY"] ||= "ruby" -ENV["PROGRAM"] ||= "ruby" -ENV["AUTOCONF"] ||= "autoconf" -ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" ENV["TZ"] = "UTC" class String @@ -120,22 +108,32 @@ $digests &&= $digests.split(/[, ]+/).tap {|dig| $digests ||= DIGESTS $patch_file &&= File.expand_path($patch_file) -path = ENV["PATH"].split(File::PATH_SEPARATOR) -%w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| - cmd, = ENV[var].shellsplit - unless path.any? {|dir| +PATH = ENV["PATH"].split(File::PATH_SEPARATOR) +def PATH.executable_env(var, command = nil) + command = ENV[var] ||= (command or return) + cmd, = command.shellsplit + unless any? {|dir| file = File.expand_path(cmd, dir) File.file?(file) and File.executable?(file) } abort "#{File.basename $0}: #{var} command not found - #{cmd}" end + command end +PATH.executable_env("MV", "mv") +PATH.executable_env("RM", "rm -f") +PATH.executable_env("AUTOCONF", "autoconf") + %w[BASERUBY RUBY MINIRUBY].each do |var| - %x[#{ENV[var]} --disable-gem -e1 2>&1] - if $?.success? - ENV[var] += ' --disable-gem' + cmd = PATH.executable_env(var, "ruby") + help = IO.popen("#{cmd} --help", err: %i[child out], &:read) + unless $?.success? and /ruby/ =~ help + abort "#{File.basename $0}: #{var} ruby not found - #{cmd}" end + IO.popen("#{cmd} --disable-gem -eexit", err: %i[child out], &:read) + cmd += ' --disable-gem' if $?.success? + ENV[var] = cmd end if defined?($help) or defined?($_help) @@ -147,7 +145,7 @@ unless destdir = ARGV.shift end revisions = ARGV.empty? ? [nil] : ARGV -if $exported +if defined?($exported) abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead" end @@ -255,7 +253,6 @@ end def package(vcs, rev, destdir, tmp = nil) pwd = Dir.pwd - patchlevel = false prerelease = false if rev and revision = rev[/@(\h+)\z/, 1] rev = $` @@ -272,22 +269,23 @@ def package(vcs, rev, destdir, tmp = nil) when /\Astable\z/ vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} url &&= vcs.branch(url) - when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + when /\A(\d+)\.(\d+)\.(\d+)-(preview|rc)(\d+)/ prerelease = true tag = "#{$4}#{$5}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") - when /\A(.*)\.(.*)\.(.*)-p(\d+)/ - patchlevel = true - tag = "p#{$4}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") - when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ - if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") - patchlevel = true - tag = "" - url = vcs.tag("v#{$1}_#{$2}_#{$3}") + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") + else + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") + end + when /\A(\d+)\.(\d+)\.(\d+)\z/ + tag = "" + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") else - url = vcs.branch("ruby_#{rev.tr('.', '_')}") + url = vcs.tag("v#{$1}_#{$2}_#{$3}") end + when /\A(\d+)\.(\d+)\z/ + url = vcs.branch("ruby_#{rev.tr('.', '_')}") else warn "#{$0}: unknown version - #{rev}" return @@ -295,7 +293,7 @@ def package(vcs, rev, destdir, tmp = nil) if info = vcs.get_revisions(url) modified = info[2] else - modified = Time.now - 10 + _, _, modified = VCS::Null.new(nil).get_revisions(url) end if !revision and info revision = info @@ -333,7 +331,11 @@ def package(vcs, rev, destdir, tmp = nil) end end - status = IO.read(File.dirname(__FILE__) + "/prereq.status") + Dir.glob("#{exported}/.*.yml") do |file| + FileUtils.rm(file, verbose: $VERBOSE) + end + + status = File.read(File.dirname(__FILE__) + "/prereq.status") Dir.chdir(tmp) if tmp if !File.directory?(v) @@ -342,33 +344,23 @@ def package(vcs, rev, destdir, tmp = nil) v = v[0] end - open("#{v}/revision.h", "wb") {|f| - short = vcs.short_revision(revision) - f.puts "#define RUBY_REVISION #{short.inspect}" - unless short == revision - f.puts "#define RUBY_FULL_REVISION #{revision.inspect}" - end + File.open("#{v}/revision.h", "wb") {|f| + f.puts vcs.revision_header(revision, modified) } - version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version ||= (versionhdr = File.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] version ||= begin - include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h") + include_ruby_versionhdr = File.read("#{v}/include/ruby/version.h") api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] [api_major_version, api_minor_version, version_teeny].join('.') end version or return - if patchlevel - unless tag.empty? - versionhdr ||= IO.read("#{v}/version.h") - patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] - tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision)) - end - elsif prerelease - versionhdr ||= IO.read("#{v}/version.h") - versionhdr.sub!(/^\#define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) - IO.write("#{v}/version.h", versionhdr) + if prerelease + versionhdr ||= File.read("#{v}/version.h") + versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace" + File.write("#{v}/version.h", versionhdr) else tag ||= vcs.revision_name(revision) end @@ -387,7 +379,21 @@ def package(vcs, rev, destdir, tmp = nil) puts $colorize.fail("patching failed") return end - def (clean = []).add(n) push(n); n end + + class << (clean = []) + def add(n) push(n) + n + end + def create(file, content = "", &block) + add(file) + if block + File.open(file, "wb", &block) + else + File.binwrite(file, content) + end + end + end + Dir.chdir(v) do unless File.exist?("ChangeLog") vcs.export_changelog(url, nil, revision, "ChangeLog") @@ -408,7 +414,7 @@ def package(vcs, rev, destdir, tmp = nil) puts end - File.open(clean.add("cross.rb"), "w") do |f| + clean.create("cross.rb") do |f| f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)" f.puts "CROSS_COMPILING=true" f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)" @@ -416,9 +422,10 @@ def package(vcs, rev, destdir, tmp = nil) f.puts "Object.__send__(:remove_const, :RUBY_VERSION)" f.puts "RUBY_VERSION='#{version}'" end + puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE unless File.exist?("configure") print "creating configure..." - unless system([ENV["AUTOCONF"]]*2) + unless system(File.exist?(gen = "./autogen.sh") ? gen : [ENV["AUTOCONF"]]*2) puts $colorize.fail(" failed") return end @@ -427,25 +434,24 @@ def package(vcs, rev, destdir, tmp = nil) clean.add("autom4te.cache") clean.add("enc/unicode/data") print "creating prerequisites..." - if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + if File.file?("common.mk") && /^prereq/ =~ commonmk = File.read("common.mk") puts extout = clean.add('tmp') begin - status = IO.read("tool/prereq.status") + status = File.read("tool/prereq.status") rescue Errno::ENOENT # use fallback file end - File.open(clean.add("config.status"), "w") {|f| - f.print status - } + clean.create("config.status", status) + clean.create("noarch-fake.rb", "require_relative 'cross'\n") FileUtils.mkpath(hdrdir = "#{extout}/include/ruby") - File.open("#{hdrdir}/config.h", "w") {} + File.binwrite("#{hdrdir}/config.h", "") FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults") - File.open("#{defaults}/operating_system.rb", "w") {} - File.open("#{defaults}/ruby.rb", "w") {} + File.binwrite("#{defaults}/operating_system.rb", "") + File.binwrite("#{defaults}/ruby.rb", "") miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] - mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")). + mk = (File.read("template/Makefile.in") rescue File.read("Makefile.in")). gsub(/^@.*\n/, '') vars = { "EXTOUT"=>extout, @@ -458,15 +464,24 @@ def package(vcs, rev, destdir, tmp = nil) "MAJOR"=>api_major_version, "MINOR"=>api_minor_version, "TEENY"=>version_teeny, + "VPATH"=>(ENV["VPATH"] || "include/ruby"), + "PROGRAM"=>(ENV["PROGRAM"] || "ruby"), + "BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"), } status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do vars[$2] ||= $3 end vars.delete("UNICODE_FILES") # for stable branches vars["UNICODE_VERSION"] = $unicode_version if $unicode_version - vars["EXTRACT_EXTLIBS"] = "no" if $extlibs == "no" args = vars.dup mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} + commonmk.gsub!(/^!(?:include \$\(srcdir\)\/(.*))?/) do + if inc = $1 and File.exist?(inc) + File.binread(inc).gsub(/^!/, '# !') + else + "#" + end + end mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') mk << <<-'APPEND' @@ -479,11 +494,9 @@ update-gems: $(UNICODE_SRC_DATA_DIR)/.unicode-tables.time: touch-unicode-files: APPEND - open(clean.add("Makefile"), "w") do |f| - f.puts mk - end - File.open(clean.add("revision.tmp"), "w") {} - File.open(clean.add(".revision.time"), "w") {} + clean.create("Makefile", mk) + clean.create("revision.tmp") + clean.create(".revision.time") ENV["CACHE_SAVE"] = "no" make = MAKE.new(args) return unless make.run("update-download") @@ -506,12 +519,11 @@ touch-unicode-files: end print "prerequisites" else - system(*%W"#{YACC} -o parse.c parse.y") + system(*%W[#{PATH.executable_env("YACC", "bison")} -o parse.c parse.y]) end vcs.after_export(".") if exported clean.concat(Dir.glob("ext/**/autom4te.cache")) - FileUtils.rm_rf(clean) unless $keep_temp - FileUtils.rm_rf(".downloaded-cache") + clean.add(".downloaded-cache") if File.exist?("gems/bundled_gems") gems = Dir.glob("gems/*.gem") gems -= File.readlines("gems/bundled_gems").map {|line| @@ -519,10 +531,11 @@ touch-unicode-files: name, version, _ = line.split(' ') "gems/#{name}-#{version}.gem" } - FileUtils.rm_f(gems) + clean.concat(gems) else - FileUtils.rm_rf("gems") + clean.add("gems") end + FileUtils.rm_rf(clean) if modified touch_all(modified, "**/*/", 0) do |name, stat| stat.mtime > modified @@ -591,20 +604,18 @@ ensure Dir.chdir(pwd) end -if [$srcdir, ($svn||=nil), ($git||=nil)].compact.size > 1 - abort "#{File.basename $0}: -srcdir, -svn, and -git are exclusive" +if [$srcdir, ($git||=nil)].compact.size > 1 + abort "#{File.basename $0}: -srcdir and -git are exclusive" end if $srcdir vcs = VCS.detect($srcdir) -elsif $svn - vcs = VCS::SVN.new($svn == true ? SVNURL : URI.parse($svn)) elsif $git abort "#{File.basename $0}: use -srcdir with cloned local repository" else begin vcs = VCS.detect(File.expand_path("../..", __FILE__)) rescue VCS::NotFoundError - vcs = VCS::SVN.new(SVNURL) + abort "#{File.expand_path("../..", __FILE__)}: cannot find git repository" end end @@ -617,7 +628,7 @@ revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| success = false next end - str = open(name, "rb") {|f| f.read} + str = File.binread(name) pathname = Pathname(name) basename = pathname.basename.to_s extname = pathname.extname.sub(/\A\./, '') @@ -625,7 +636,7 @@ revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| key = basename[/\A(.*)\.(?:tar|zip)/, 1] info[key] ||= Hash.new{|h,k|h[k]={}} info[key]['version'] = version if version - info[key]['date'] = release_date.strftime('%Y-%m-%d') + info[key]['date'] = release_date.to_date if version info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" diff --git a/tool/make_hgraph.rb b/tool/make_hgraph.rb index 0f388814dd..174fa5dd2f 100644 --- a/tool/make_hgraph.rb +++ b/tool/make_hgraph.rb @@ -83,13 +83,12 @@ module ObjectSpace def self.module_refenreces_image klass, file dot = module_refenreces_dot(klass) - img = nil - IO.popen("dot -Tpng", 'r+'){|io| + img = IO.popen(%W"dot -Tpng", 'r+b') {|io| # io.puts dot io.close_write - img = io.read + io.read } - open(File.expand_path(file), 'w+'){|f| f.puts img} + File.binwrite(file, img) end end diff --git a/tool/merger.rb b/tool/merger.rb index d38f00b0fd..4c096087fc 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -2,9 +2,8 @@ # -*- ruby -*- exec "${RUBY-ruby}" "-x" "$0" "$@" && [ ] if false #!ruby -# This needs ruby 2.0, Subversion and Git. -# As a Ruby committer, run this in an SVN repository -# to commit a change. +# This needs ruby 2.0 and Git. +# As a Ruby committer, run this in a git repository to commit a change. require 'tempfile' require 'net/http' @@ -15,20 +14,11 @@ ENV['LC_ALL'] = 'C' ORIGIN = 'git@git.ruby-lang.org:ruby.git' GITHUB = 'git@github.com:ruby/ruby.git' -module Merger - REPOS = 'svn+ssh://svn@ci.ruby-lang.org/ruby/' -end - -class << Merger - include Merger - +class << Merger = Object.new def help puts <<-HELP \e[1msimple backport\e[0m - ruby #$0 1234 - -\e[1mbackport from other branch\e[0m - ruby #$0 17502 mvm + ruby #$0 1234abc \e[1mrevision increment\e[0m ruby #$0 revisionup @@ -37,16 +27,16 @@ class << Merger ruby #$0 teenyup \e[1mtagging major release\e[0m - ruby #$0 tag 2.2.0 + ruby #$0 tag 3.2.0 -\e[1mtagging patch release\e[0m (about 2.1.0 or later, it means X.Y.Z (Z > 0) release) +\e[1mtagging patch release\e[0m (for 2.1.0 or later, it means X.Y.Z (Z > 0) release) ruby #$0 tag \e[1mtagging preview/RC\e[0m - ruby #$0 tag 2.2.0-preview1 + ruby #$0 tag 3.2.0-preview1 \e[1mremove tag\e[0m - ruby #$0 removetag 2.2.9 + ruby #$0 removetag 3.2.9 \e[33;1m* all operations shall be applied to the working directory.\e[0m HELP @@ -57,11 +47,11 @@ class << Merger yield if block_given? STDERR.puts "\e[1;33m#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})\e[0m" case STDIN.gets - when /\Aa/i then exit + when /\Aa/i then exit 1 when /\Ar/i then redo when /\Ay/i then break when /\Ae/i then system(ENV['EDITOR'], editfile) - else exit + else exit 1 end end end @@ -69,17 +59,14 @@ class << Merger def version_up(teeny: false) now = Time.now now = now.localtime(9*60*60) # server is Japan Standard Time +09:00 - if svn_mode? - system('svn', 'revert', 'version.h') - else - system('git', 'checkout', 'HEAD', 'version.h') - end + system('git', 'checkout', 'HEAD', 'version.h') v, pl = version if teeny v[2].succ! end - if pl != '-1' # trunk does not have patchlevel + # We stopped bumping RUBY_PATCHLEVEL at Ruby 4.0.0. + if Integer(v[0]) <= 3 pl.succ! end @@ -127,33 +114,21 @@ class << Merger abort 'no relname is given and not in a release branch even if this is patch release' end end - tagname = "v#{v.join('_')}#{("_#{pl}" if v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl)}" - - if svn_mode? - if relname - branch_url = `svn info`[/URL: (.*)/, 1] - else - branch_url = "#{REPOS}branches/ruby_" - if v[0] < '2' || (v[0] == '2' && v[1] < '1') - abort 'patchlevel must be greater than 0 for patch release' if pl == '0' - branch_url << v.join('_') - else - abort 'teeny must be greater than 0 for patch release' if v[2] == '0' - branch_url << v.join('_').sub(/_\d+\z/, '') - end - end - tag_url = "#{REPOS}tags/#{tagname}" - system('svn', 'info', tag_url, out: IO::NULL, err: IO::NULL) - if $?.success? - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('svn', 'cp', '-m', "add tag #{tagname}", branch_url, tag_url, interactive: true) + if /^(?:preview|rc)/ =~ pl + tagname = "v#{v.join('.')}-#{pl}" + elsif Integer(v[0]) >= 4 + tagname = "v#{v.join('.')}" else - unless execute('git', 'tag', tagname) - abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' - end - execute('git', 'push', ORIGIN, tagname, interactive: true) + tagname = "v#{v.join('_')}" end + + unless execute('git', 'diff', '--exit-code') + abort 'uncommitted changes' + end + unless execute('git', 'tag', tagname) + abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' + end + execute('git', 'push', ORIGIN, tagname, interactive: true) end def remove_tag(relname) @@ -167,70 +142,52 @@ class << Merger unless relname raise ArgumentError, 'relname is not specified' end - if /^v/ !~ relname - tagname = "v#{relname.gsub(/[.-]/, '_')}" - else + if relname.start_with?('v') tagname = relname - end - - if svn_mode? - tag_url = "#{REPOS}tags/#{tagname}" - execute('svn', 'rm', '-m', "remove tag #{tagname}", tag_url, interactive: true) + elsif Integer(relname.split('.', 2).first) >= 4 + tagname = "v#{relname}" else - execute('git', 'tag', '-d', tagname) - execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) - execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) + tagname = "v#{relname.gsub(/[.-]/, '_')}" end + + execute('git', 'tag', '-d', tagname) + execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) + execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) end def update_revision_h - if svn_mode? - execute('svn', 'up') - end execute('ruby tool/file2lastrev.rb --revision.h . > revision.tmp') execute('tool/ifchange', '--timestamp=.revision.time', 'revision.h', 'revision.tmp') execute('rm', '-f', 'revision.tmp') end def stat - if svn_mode? - `svn stat` - else - `git status --short` - end + `git status --short` end def diff(file = nil) - if svn_mode? - command = %w[svn diff --diff-cmd=diff -x -upw] - else - command = %w[git diff --color] - end + command = %w[git diff --color HEAD] IO.popen(command + [file].compact, &:read) end def commit(file) - if svn_mode? - begin - execute('svn', 'ci', '-F', file) - execute('svn', 'update') # svn ci doesn't update revision info on working copy - ensure - execute('rm', '-f', 'subversion.commitlog') - end - else - current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip - execute('git', 'add', '.') && - execute('git', 'commit', '-F', file) - end + current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip + execute('git', 'add', '.') && execute('git', 'commit', '-F', file) end - private - - def svn_mode? - return @svn_mode if defined?(@svn_mode) - @svn_mode = system("svn info", %i(out err) => IO::NULL) + def has_conflicts? + changes = IO.popen(%w[git status --porcelain -z]) { |io| io.readlines("\0", chomp: true) } + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + !conflict.empty? end + private + # Prints the version of Ruby found in version.h def version v = p = nil @@ -289,21 +246,21 @@ else case ARGV[0] when /--ticket=(.*)/ - tickets = $1.split(/,/).map{|num| " [Backport ##{num}]"}.join + tickets = $1.split(/,/) ARGV.shift else - tickets = '' + tickets = [] + detect_ticket = true end - revstr = ARGV[0].delete('^, :\-0-9a-fA-F') + revstr = ARGV[0].gsub(%r!https://github\.com/ruby/ruby/commit/|https://bugs\.ruby-lang\.org/projects/ruby-master/repository/git/revisions/!, '') + revstr = revstr.delete('^, :\-0-9a-fA-F') revs = revstr.split(/[,\s]+/) commit_message = '' revs.each do |rev| git_rev = nil case rev - when /\A\d{1,6}\z/ - svn_rev = rev when /\A\h{7,40}\z/ git_rev = rev when nil then @@ -314,28 +271,25 @@ else exit end - # Merge revision from Git patch or SVN - if git_rev - git_uri = "https://git.ruby-lang.org/ruby.git/patch/?id=#{git_rev}" - resp = Net::HTTP.get_response(URI(git_uri)) - if resp.code != '200' - abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" - end - patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - - message = "\n\n#{(patch[/^Subject: (.*)\n\ndiff --git/m, 1] || "Message not found for revision: #{git_rev}\n")}" - puts '+ git apply' - IO.popen(['git', 'apply'], 'wb') { |f| f.write(patch) } - else - default_merge_branch = (%r{^URL: .*/branches/ruby_1_8_} =~ `svn info` ? 'branches/ruby_1_8' : 'trunk') - svn_src = "#{Merger::REPOS}#{ARGV[1] || default_merge_branch}" - message = IO.popen(['svn', 'log', '-c', svn_rev, svn_src], &:read) + # Merge revision from Git patch + git_uri = "https://github.com/ruby/ruby/commit/#{git_rev}.patch" + resp = Net::HTTP.get_response(URI(git_uri)) + if resp.code != '200' + abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" + end + patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - cmd = ['svn', 'merge', '--accept=postpone', '-c', svn_rev, svn_src] - puts "+ #{cmd.join(' ')}" - system(*cmd) + if detect_ticket + tickets += patch.scan(/\[(?:Bug|Feature|Misc) #(\d+)\]/i).map(&:first) end + message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" + message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") + message = "\n\n#{message}" + + puts '+ git apply' + IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } + commit_message << message.sub(/\A-+\nr.*/, '').sub(/\n-+\n\z/, '').gsub(/^./, "\t\\&") end @@ -345,20 +299,22 @@ else Merger.version_up f = Tempfile.new 'merger.rb' - f.printf "merge revision(s) %s:%s", revstr, tickets + f.printf "merge revision(s) %s:%s", revs.join(', '), tickets.map{|num| " [Backport ##{num}]"}.join f.write commit_message f.flush f.close - Merger.interactive('conflicts resolved?', f.path) do - IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| - g << Merger.stat - g << "\n\n" - f.open - g << f.read - f.close - g << "\n\n" - g << Merger.diff + if Merger.has_conflicts? + Merger.interactive('conflicts resolved?', f.path) do + IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| + g << Merger.stat + g << "\n\n" + f.open + g << f.read + f.close + g << "\n\n" + g << Merger.diff + end end end diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat new file mode 100755 index 0000000000..d39568fe86 --- /dev/null +++ b/tool/missing-baseruby.bat @@ -0,0 +1,30 @@ +:"" == " +@echo off || ( + :warn + echo>&2.%~1 + goto :eof + :abort + exit /b 1 +)||( +:)"||( + # necessary libraries + require 'erb' + require 'fileutils' + require 'tempfile' + s = %^# +) +: ; call() { local call=${1#:}; shift; $call "$@"; } +: ; warn() { echo "$1" >&2; } +: ; abort () { exit 1; } + +call :warn "executable host ruby is required. use --with-baseruby option." +call :warn "Note that BASERUBY must be Ruby 3.1.0 or later." +call :abort +(goto :eof ^;) +verbose = true if ARGV[0] == "--verbose" +case +when !defined?(RubyVM::InstructionSequence) + abort(*(["BASERUBY must be CRuby"] if verbose)) +when RUBY_VERSION < s[%r[warn .*\KBASERUBY .*Ruby ([\d.]+)(?:\.0)?.*(?=\")],1] + abort(*(["#{$&}. Found: #{RUBY_VERSION}"] if verbose)) +end diff --git a/tool/mjit_archflag.sh b/tool/mjit_archflag.sh deleted file mode 100644 index 082fb4bcd0..0000000000 --- a/tool/mjit_archflag.sh +++ /dev/null @@ -1,40 +0,0 @@ -# -*- sh -*- - -quote() { - printf "#${indent}define $1" - shift - ${1+printf} ${1+' "%s"'$sep} ${1+"$@"} - echo -} - -archs="" -arch_flag="" - -parse_arch_flags() { - for arch in $1; do - archs="${archs:+$archs }${arch%=*}" - done - - while shift && [ "$#" -gt 0 ]; do - case "$1" in - -arch) - shift - archs="${archs:+$archs }$1" - ;; - *) - arch_flag="${arch_flag:+${arch_flag} }$1" - ;; - esac - done -} - -define_arch_flags() { - ${archs:+echo} ${archs:+'#if 0'} - for arch in $archs; do - echo "#elif defined __${arch}__" - quote "MJIT_ARCHFLAG " -arch "${arch}" - done - ${archs:+echo} ${archs:+'#else'} - quote "MJIT_ARCHFLAG /* ${arch_flag:-no flag} */" ${arch_flag} - ${archs:+echo} ${archs:+'#endif'} -} diff --git a/tool/mjit_tabs.rb b/tool/mjit_tabs.rb deleted file mode 100644 index edcbf6cfcb..0000000000 --- a/tool/mjit_tabs.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true -# This is a script to run a command in ARGV, expanding tabs in some files -# included by vm.c to normalize indentation of MJIT header. You can enable -# this feature by passing `--without-mjit-tabs` in configure. -# -# Note that preprocessor of GCC converts a hard tab to one spaces, where -# we expect it to be shown as 8 spaces. To obviate this script, we need -# to convert all tabs to spaces in these files. - -require 'fileutils' - -EXPAND_TARGETS = %w[ - vm*.* - include/ruby/ruby.h -] - -# These files have no hard tab indentations. Skip normalizing these files from the glob result. -SKIPPED_FILES = %w[ - vm_callinfo.h - vm_debug.h - vm_exec.h - vm_opts.h - vm_sync.h - vm_sync.c -] - -srcdir = File.expand_path('..', __dir__) -targets = EXPAND_TARGETS.flat_map { |t| Dir.glob(File.join(srcdir, t)) } - SKIPPED_FILES.map { |f| File.join(srcdir, f) } -sources = {} -mtimes = {} - -mjit_tabs, *command = ARGV - -targets.each do |target| - next if mjit_tabs != 'false' - unless File.writable?(target) - puts "tool/mjit_tabs.rb: Skipping #{target.dump} as it's not writable." - next - end - source = File.read(target) - begin - expanded = source.gsub(/^\t+/) { |tab| ' ' * 8 * tab.length } - rescue ArgumentError # invalid byte sequence in UTF-8 (Travis, RubyCI) - puts "tool/mjit_tabs.rb: Skipping #{target.dump} as the encoding is #{source.encoding}." - next - end - - sources[target] = source - mtimes[target] = File.mtime(target) - - if sources[target] == expanded - puts "#{target.dump} has no hard tab indentation. This should be ignored in tool/mjit_tabs.rb." - end - File.write(target, expanded) - FileUtils.touch(target, mtime: mtimes[target]) -end - -result = system(*command) - -targets.each do |target| - if sources.key?(target) - File.write(target, sources[target]) - FileUtils.touch(target, mtime: mtimes.fetch(target)) - end -end - -exit result diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index 6740ec5c17..5aa07962f9 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -4,6 +4,26 @@ require 'ripper' require 'stringio' require_relative 'ruby_vm/helpers/c_escape' +SUBLIBS = {} +REQUIRED = {} +BUILTIN_ATTRS = %w[leaf inline_block use_block c_trace] + +module CompileWarning + @@warnings = 0 + + def warn(message) + @@warnings += 1 + super + end + + def self.reset + w, @@warnings = @@warnings, 0 + w.nonzero? + end +end + +Warning.extend CompileWarning + def string_literal(lit, str = []) while lit case lit.first @@ -22,6 +42,17 @@ def string_literal(lit, str = []) end end +# e.g. [:symbol_literal, [:symbol, [:@ident, "inline", [19, 21]]]] +def symbol_literal(lit) + symbol_literal, symbol_lit = lit + raise "#{lit.inspect} was not :symbol_literal" if symbol_literal != :symbol_literal + symbol, ident_lit = symbol_lit + raise "#{symbol_lit.inspect} was not :symbol" if symbol != :symbol + ident, symbol_name, = ident_lit + raise "#{ident.inspect} was not :@ident" if ident != :@ident + symbol_name +end + def inline_text argc, arg1 raise "argc (#{argc}) of inline! should be 1" unless argc == 1 arg1 = string_literal(arg1) @@ -29,6 +60,16 @@ def inline_text argc, arg1 arg1.join("").rstrip end +def inline_attrs(args) + raise "args was empty" if args.empty? + args.each do |arg| + attr = symbol_literal(arg) + unless BUILTIN_ATTRS.include?(attr) + raise "attr (#{attr}) was not in: #{BUILTIN_ATTRS.join(', ')}" + end + end +end + def make_cfunc_name inlines, name, lineno case name when /\[\]/ @@ -84,7 +125,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil tree = tree[2] next when :method_add_arg - _, mid, (_, (_, args)) = tree + _method_add_arg, mid, (_arg_paren, args) = tree case mid.first when :call _, recv, sep, mid = mid @@ -93,6 +134,11 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil else mid = nil end + # w/ trailing comma: [[:method_add_arg, ...]] + # w/o trailing comma: [:args_add_block, [[:method_add_arg, ...]], false] + if args && args.first == :args_add_block + args = args[1] + end when :vcall _, mid = tree when :command # FCALL @@ -100,6 +146,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil when :call, :command_call # CALL _, recv, sep, mid, (_, args) = tree end + if mid raise "unknown sexp: #{mid.inspect}" unless %i[@ident @const].include?(mid.first) _, mid, (lineno,) = mid @@ -126,18 +173,16 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil args.pop unless (args ||= []).last argc = args.size - if /(.+)\!\z/ =~ func_name + if /(.+)[\!\?]\z/ =~ func_name case $1 when 'attr' - text = inline_text(argc, args.first) - if text != 'inline' - raise "Only 'inline' is allowed to be annotated (but got: '#{text}')" - end + # Compile-time validation only. compile.c will parse them. + inline_attrs(args) break when 'cstmt' text = inline_text argc, args.first - func_name = "_bi#{inlines.size}" + func_name = "_bi#{lineno}" cfunc_name = make_cfunc_name(inlines, name, lineno) inlines[cfunc_name] = [lineno, text, locals, func_name] argc -= 1 @@ -145,7 +190,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil text = inline_text argc, args.first code = "return #{text};" - func_name = "_bi#{inlines.size}" + func_name = "_bi#{lineno}" cfunc_name = make_cfunc_name(inlines, name, lineno) locals = [] if $1 == 'cconst' @@ -156,6 +201,8 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil func_name = nil # required inlines[inlines.size] = [lineno, text, nil, nil] argc -= 1 + when 'mandatory_only' + func_name = nil when 'arg' argc == 1 or raise "unexpected argument number #{argc}" (arg = args.first)[0] == :symbol_literal or raise "symbol literal expected #{args}" @@ -171,6 +218,21 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil end bs[func_name] = [argc, cfunc_name] if func_name + elsif /\Arequire(?:_relative)\z/ =~ mid and args.size == 1 and + (arg1 = args[0])[0] == :string_literal and + (arg1 = arg1[1])[0] == :string_content and + (arg1 = arg1[1])[0] == :@tstring_content and + sublib = arg1[1] + if File.exist?(f = File.join(@dir, sublib)+".rb") + puts "- #{@base}.rb requires #{sublib}" + if REQUIRED[sublib] + warn "!!! #{sublib} is required from #{REQUIRED[sublib]} already; ignored" + else + REQUIRED[sublib] = @base + (SUBLIBS[@base] ||= []) << sublib + end + ARGV.push(f) + end end break unless tree = args end @@ -217,12 +279,25 @@ end def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) f = StringIO.new + + # Avoid generating fetches of lvars we don't need. This is imperfect as it + # will match text inside strings or other false positives. + local_ptrs = [] + local_candidates = text.gsub(/\bLOCAL_PTR\(\K[a-zA-Z_][a-zA-Z0-9_]*(?=\))/) { + local_ptrs << $&; '' + }.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + f.puts '{' lineno += 1 - locals.reverse_each.with_index{|param, i| + # locals is nil outside methods + locals&.reverse_each&.with_index{|param, i| next unless Symbol === param - f.puts "MAYBE_UNUSED(const VALUE) #{param} = rb_vm_lvar(ec, #{-3 - i});" - lineno += 1 + param = param.to_s + lvar = local_candidates.include?(param) + next unless lvar or local_ptrs.include?(param) + f.puts "VALUE *const #{param}__ptr = (VALUE *)&ec->cfp->ep[#{-3 - i}];" + f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" if lvar + lineno += lvar ? 2 : 1 } f.puts "#line #{body_lineno} \"#{line_file}\"" lineno += 1 @@ -239,21 +314,25 @@ def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_nam end def mk_builtin_header file + @dir = File.dirname(file) base = File.basename(file, '.rb') + @base = base ofile = "#{file}inc" # bs = { func_name => argc } code = File.read(file) - collect_iseq RubyVM::InstructionSequence.compile(code).to_a - collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) - begin - f = open(ofile, 'w') - rescue Errno::EACCES - # Fall back to the current directory - f = open(File.basename(ofile), 'w') + verbose, $VERBOSE = $VERBOSE, true + collect_iseq RubyVM::InstructionSequence.compile(code, base).to_a + ensure + $VERBOSE = verbose end - begin + if warnings = CompileWarning.reset + raise "#{warnings} warnings in #{file}" + end + collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) + + StringIO.open do |f| if File::ALT_SEPARATOR file = file.tr(File::ALT_SEPARATOR, File::SEPARATOR) ofile = ofile.tr(File::ALT_SEPARATOR, File::SEPARATOR) @@ -290,43 +369,13 @@ def mk_builtin_header file end } - bs.each_pair{|func, (argc, cfunc_name)| - decl = ', VALUE' * argc - argv = argc \ - . times \ - . map {|i|", argv[#{i}]"} \ - . join('') - f.puts %'static void' - f.puts %'mjit_compile_invokebuiltin_for_#{func}(FILE *f, long index, unsigned stack_size, bool inlinable_p)' - f.puts %'{' - f.puts %' fprintf(f, " VALUE self = GET_SELF();\\n");' - f.puts %' fprintf(f, " typedef VALUE (*func)(rb_execution_context_t *, VALUE#{decl});\\n");' - if inlines.has_key? cfunc_name - body_lineno, text, locals, func_name = inlines[cfunc_name] - lineno, str = generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) - f.puts %' if (inlinable_p) {' - str.gsub(/^(?!#)/, ' ').each_line {|i| - j = RubyVM::CEscape.rstring2cstr(i).dup - j.sub!(/^ return\b/ , ' val =') - f.printf(%' fprintf(f, "%%s", %s);\n', j) - } - f.puts(%' return;') - f.puts(%' }') - end - if argc > 0 - f.puts %' if (index == -1) {' - f.puts %' fprintf(f, " const VALUE *argv = &stack[%d];\\n", stack_size - #{argc});' - f.puts %' }' - f.puts %' else {' - f.puts %' fprintf(f, " const unsigned int lnum = GET_ISEQ()->body->local_table_size;\\n");' - f.puts %' fprintf(f, " const VALUE *argv = GET_EP() - lnum - VM_ENV_DATA_SIZE + 1 + %ld;\\n", index);' - f.puts %' }' + if SUBLIBS[base] + f.puts "// sub libraries" + SUBLIBS[base].each do |sub| + f.puts %[#include #{(sub+".rbinc").dump}] end - f.puts %' fprintf(f, " func f = (func)%"PRIuVALUE"; /* == #{cfunc_name} */\\n", (VALUE)#{cfunc_name});' - f.puts %' fprintf(f, " val = f(ec, self#{argv});\\n");' - f.puts %'}' f.puts - } + end f.puts "void Init_builtin_#{base}(void)" f.puts "{" @@ -335,9 +384,9 @@ def mk_builtin_header file f.puts " // table definition" f.puts " static const struct rb_builtin_function #{table}[] = {" bs.each.with_index{|(func, (argc, cfunc_name)), i| - f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc}, mjit_compile_invokebuiltin_for_#{func})," + f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc})," } - f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0, 0)," + f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0)," f.puts " };" f.puts @@ -351,13 +400,26 @@ def mk_builtin_header file } f.puts "COMPILER_WARNING_POP" + if SUBLIBS[base] + f.puts + f.puts " // sub libraries" + SUBLIBS[base].each do |sub| + f.puts " Init_builtin_#{sub}();" + end + end + f.puts f.puts " // load" f.puts " rb_load_with_builtin_functions(#{base.dump}, #{table});" f.puts "}" - ensure - f.close + + begin + File.write(ofile, f.string) + rescue SystemCallError # EACCES, EPERM, EROFS, etc. + # Fall back to the current directory + File.write(File.basename(ofile), f.string) + end end end diff --git a/tool/mk_rbbin.rb b/tool/mk_rbbin.rb new file mode 100755 index 0000000000..991230f094 --- /dev/null +++ b/tool/mk_rbbin.rb @@ -0,0 +1,48 @@ +#!ruby -s + +OPTIMIZATION = { + inline_const_cache: true, + peephole_optimization: true, + tailcall_optimization: false, + specialized_instruction: true, + operands_unification: true, + instructions_unification: true, + frozen_string_literal: true, + debug_frozen_string_literal: false, + coverage_enabled: false, + debug_level: 0, +} + +file = File.basename(ARGV[0], ".rb") +name = "<internal:#{file}>" +iseq = RubyVM::InstructionSequence.compile(ARGF.read, name, name, **OPTIMIZATION) +puts <<C +/* -*- C -*- */ + +static const char #{file}_builtin[] = { +C +iseq.to_binary.bytes.each_slice(8) do |b| + print " ", b.map {|c| "0x%.2x," % c}.join(" ") + if $comment + print " /* ", b.pack("C*").gsub(/([[ -~]&&[^\\]])|(?m:.)/) { + (c = $1) ? "#{c} " : (c = $&.dump).size == 2 ? c : ". " + }, "*/" + end + puts +end +puts <<C +}; + +#include "ruby/ruby.h" +#include "vm_core.h" + +void +Init_#{file}(void) +{ + const char *builtin = #{file}_builtin; + size_t size = sizeof(#{file}_builtin); + VALUE code = rb_str_new_static(builtin, (long)size); + VALUE iseq = rb_funcallv(rb_cISeq, rb_intern_const("load_from_binary"), 1, &code); + rb_funcallv(iseq, rb_intern_const("eval"), 0, 0); +} +C diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb index 6db60724e9..db74115730 100755 --- a/tool/mkconfig.rb +++ b/tool/mkconfig.rb @@ -1,4 +1,5 @@ #!./miniruby -s +# frozen-string-literal: true # This script, which is run when ruby is built, generates rbconfig.rb by # parsing information from config.status. rbconfig.rb contains build @@ -63,8 +64,6 @@ File.foreach "config.status" do |line| when /^(?:X|(?:MINI|RUN|(?:HAVE_)?BASE|BOOTSTRAP|BTEST)RUBY(?:_COMMAND)?$)/; next when /^INSTALLDOC|TARGET$/; next when /^DTRACE/; next - when /^MJIT_(CC|SUPPORT)$/; # pass - when /^MJIT_/; next when /^(?:MAJOR|MINOR|TEENY)$/; vars[name] = val; next when /^LIBRUBY_D?LD/; next when /^RUBY_INSTALL_NAME$/; next vars[name] = (install_name = val).dup if $install_name @@ -88,7 +87,7 @@ File.foreach "config.status" do |line| unless $install_name $install_name = "ruby" val.gsub!(/\$\$/, '$') - val.scan(%r[\G[\s;]*(/(?:\\.|[^/])*/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*)\3((?:(?!\3)(?:\\.|.))*)\3([gi]*)]) do + val.scan(%r[\G[\s;]*(/(?:\\.|[^/])*+/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*+)\3((?:(?!\3)(?:\\.|.))*+)\3([gi]*)]) do |addr, cmd, sep, pat, rep, opt| if addr Regexp.new(addr[/\A\/(.*)\/\z/, 1]) =~ $install_name or next @@ -122,8 +121,16 @@ File.foreach "config.status" do |line| universal, val = val, 'universal' if universal when /^arch$/ if universal - platform = val.sub(/universal/, %q[#{arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/]}]) + platform = val.sub(/universal/, '$(arch)') end + when /^target_cpu$/ + if universal + val = 'cpu' + end + when /^target$/ + val = '"$(target_cpu)-$(target_vendor)-$(target_os)"' + when /^host(?:_(?:os|vendor|cpu|alias))?$/ + val = %["$(#{name.sub(/^host/, 'target')})"] when /^includedir$/ val = '"$(SDKROOT)"'+val if /darwin/ =~ arch end @@ -161,8 +168,8 @@ def vars.expand(val, config = self) val.replace(newval) unless newval == val val end -prefix = vars.expand(vars["prefix"] ||= "") -rubyarchdir = vars.expand(vars["rubyarchdir"] ||= "") +prefix = vars.expand(vars["prefix"] ||= +"") +rubyarchdir = vars.expand(vars["rubyarchdir"] ||= +"") relative_archdir = rubyarchdir.rindex(prefix, 0) ? rubyarchdir[prefix.size..-1] : rubyarchdir puts %[\ @@ -185,17 +192,19 @@ print " # Ruby installed directory.\n" print " TOPDIR = File.dirname(__FILE__).chomp!(#{relative_archdir.dump})\n" print " # DESTDIR on make install.\n" print " DESTDIR = ", (drive ? "TOPDIR && TOPDIR[/\\A[a-z]:/i] || " : ""), "'' unless defined? DESTDIR\n" -print <<'ARCH' if universal +print <<"UNIVERSAL", <<'ARCH' if universal + universal = #{universal} +UNIVERSAL arch_flag = ENV['ARCHFLAGS'] || ((e = ENV['RC_ARCHS']) && e.split.uniq.map {|a| "-arch #{a}"}.join(' ')) arch = arch_flag && arch_flag[/\A\s*-arch\s+(\S+)\s*\z/, 1] + cpu = arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/] ARCH -print " universal = #{universal}\n" if universal print " # The hash configurations stored.\n" print " CONFIG = {}\n" print " CONFIG[\"DESTDIR\"] = DESTDIR\n" versions = {} -IO.foreach(File.join(srcdir, "version.h")) do |l| +File.foreach(File.join(srcdir, "version.h")) do |l| m = /^\s*#\s*define\s+RUBY_(PATCHLEVEL)\s+(-?\d+)/.match(l) if m versions[m[1]] = m[2] @@ -216,7 +225,7 @@ IO.foreach(File.join(srcdir, "version.h")) do |l| end end if versions.size != 4 - IO.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| + File.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| m = /^\s*#\s*define\s+RUBY_API_VERSION_(\w+)\s+(-?\d+)/.match(l) if m versions[m[1]] ||= m[2] @@ -247,14 +256,14 @@ end v_others.compact! if $install_name - if install_name and vars.expand("$(RUBY_INSTALL_NAME)") == $install_name + if install_name and vars.expand(+"$(RUBY_INSTALL_NAME)") == $install_name $install_name = install_name end v_fast << " CONFIG[\"ruby_install_name\"] = \"" + $install_name + "\"\n" v_fast << " CONFIG[\"RUBY_INSTALL_NAME\"] = \"" + $install_name + "\"\n" end if $so_name - if so_name and vars.expand("$(RUBY_SO_NAME)") == $so_name + if so_name and vars.expand(+"$(RUBY_SO_NAME)") == $so_name $so_name = so_name end v_fast << " CONFIG[\"RUBY_SO_NAME\"] = \"" + $so_name + "\"\n" @@ -268,7 +277,7 @@ EOS print <<EOS if $unicode_emoji_version CONFIG["UNICODE_EMOJI_VERSION"] = #{$unicode_emoji_version.dump} EOS -print <<EOS if /darwin/ =~ arch +print prefix.start_with?("/System/") ? <<EOS : <<EOS if /darwin/ =~ arch if sdkroot = ENV["SDKROOT"] sdkroot = sdkroot.dup elsif File.exist?(File.join(CONFIG["prefix"], "include")) || @@ -279,6 +288,8 @@ print <<EOS if /darwin/ =~ arch end CONFIG["SDKROOT"] = sdkroot EOS + CONFIG["SDKROOT"] = "" +EOS print <<EOS CONFIG["platform"] = #{platform || '"$(arch)"'} CONFIG["archdir"] = "$(rubyarchdir)" @@ -340,7 +351,6 @@ print <<EOS RbConfig::expand(val) end - # :nodoc: # call-seq: # # RbConfig.fire_update!(key, val) -> array @@ -356,7 +366,7 @@ print <<EOS # RbConfig::CONFIG.values_at("CC", "LDSHARED") # => ["gcc-8", "gcc-8 -shared"] # # returns updated keys list, or +nil+ if nothing changed. - def RbConfig.fire_update!(key, val, mkconf = MAKEFILE_CONFIG, conf = CONFIG) + def RbConfig.fire_update!(key, val, mkconf = MAKEFILE_CONFIG, conf = CONFIG) # :nodoc: return if mkconf[key] == val mkconf[key] = val keys = [key] @@ -384,6 +394,7 @@ print <<EOS ) end end +# Non-nil if configured for cross compiling. CROSS_COMPILING = nil unless defined? CROSS_COMPILING EOS diff --git a/tool/mkrunnable.rb b/tool/mkrunnable.rb index 62ace28980..ef358e2425 100755 --- a/tool/mkrunnable.rb +++ b/tool/mkrunnable.rb @@ -6,6 +6,7 @@ require './rbconfig' require 'fileutils' +require_relative 'lib/path' case ARGV[0] when "-n" @@ -18,91 +19,7 @@ else include FileUtils end -module Mswin - def ln_safe(src, dest, *opt) - cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] - cmd[1, 0] = opt - return if system("cmd", "/c", *cmd) - # TODO: use RUNAS or something - puts cmd.join(" ") - end - - def ln_dir_safe(src, dest) - ln_safe(src, dest, "/d") - end -end - -def clean_link(src, dest) - begin - link = File.readlink(dest) - rescue - else - return if link == src - File.unlink(dest) - end - yield src, dest -end - -def ln_safe(src, dest) - ln_sf(src, dest) -rescue Errno::ENOENT - # Windows disallows to create broken symboic links, probably because - # it is a kind of reparse points. - raise if File.exist?(src) -end - -alias ln_dir_safe ln_safe - -case RUBY_PLATFORM -when /linux/ - def ln_exe(src, dest) - ln(src, dest, force: true) - end -else - alias ln_exe ln_safe -end - -if !File.respond_to?(:symlink) && /mingw|mswin/ =~ (CROSS_COMPILING || RUBY_PLATFORM) - extend Mswin -end - -def clean_path(path) - path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') - nil while path.sub!(/[^\/]+\/\.\.\//, '') - path -end - -def relative_path_from(path, base) - path = clean_path(path) - base = clean_path(base) - path, base = [path, base].map{|s|s.split("/")} - until path.empty? or base.empty? or path[0] != base[0] - path.shift - base.shift - end - path, base = [path, base].map{|s|s.join("/")} - if /(\A|\/)\.\.\// =~ base - File.expand_path(path) - else - base.gsub!(/[^\/]+/, '..') - File.join(base, path) - end -end - -def ln_relative(src, dest, executable = false) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - return ln_exe(src, dest) if executable - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_safe(s, d)} -end - -def ln_dir_relative(src, dest) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - clean_link(relative_path_from(src, parent), dest) {|s, d| ln_dir_safe(s, d)} -end +include Path config = RbConfig::MAKEFILE_CONFIG.merge("prefix" => ".", "exec_prefix" => ".") config.each_value {|s| RbConfig.expand(s, config)} @@ -117,18 +34,25 @@ vendordir = config["vendordir"] rubylibdir = config["rubylibdir"] rubyarchdir = config["rubyarchdir"] archdir = "#{extout}/#{arch}" -[bindir, libdir, archdir].uniq.each do |dir| +exedir = bindir +if libdirname == "archlibdir" + exedir = exedir.sub(%r[/\K(?=[^/]+\z)]) {extout+"/"} +end +[exedir, libdir, archdir].uniq.each do |dir| File.directory?(dir) or mkdir_p(dir) end +unless exedir == bindir + ln_dir_relative(exedir, bindir) +end exeext = config["EXEEXT"] ruby_install_name = config["ruby_install_name"] rubyw_install_name = config["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name -[ruby_install_name, rubyw_install_name, goruby_install_name].map do |ruby| +[ruby_install_name, rubyw_install_name, goruby_install_name].each do |ruby| if ruby and !ruby.empty? ruby += exeext - ln_relative(ruby, "#{bindir}/#{ruby}", true) + ln_relative(ruby, "#{exedir}/#{ruby}", true) end end so = config["LIBRUBY_SO"] diff --git a/tool/notes-github-pr.rb b/tool/notes-github-pr.rb new file mode 100644 index 0000000000..d69d479cdf --- /dev/null +++ b/tool/notes-github-pr.rb @@ -0,0 +1,138 @@ +#!/usr/bin/env ruby +# Add GitHub pull request reference / author info to git notes. + +require 'net/http' +require 'uri' +require 'tmpdir' +require 'json' +require 'yaml' + +# Conversion for people whose GitHub account name and SVN_ACCOUNT_NAME are different. +GITHUB_TO_SVN = { + 'amatsuda' => 'a_matsuda', + 'matzbot' => 'git', + 'jeremyevans' => 'jeremy', + 'znz' => 'kazu', + 'k-tsj' => 'ktsj', + 'nurse' => 'naruse', + 'ioquatix' => 'samuel', + 'suketa' => 'suke', + 'unak' => 'usa', +} + +EMAIL_YML_URL = 'https://raw.githubusercontent.com/ruby/git.ruby-lang.org/refs/heads/master/config/email.yml' +SVN_TO_EMAILS = YAML.safe_load(Net::HTTP.get_response(URI(EMAIL_YML_URL)).tap(&:value).body) + +class GitHub + ENDPOINT = URI.parse('https://api.github.com') + + def initialize(access_token) + @access_token = access_token + end + + # https://developer.github.com/changes/2019-04-11-pulls-branches-for-commit/ + def pulls(owner:, repo:, commit_sha:) + resp = get("/repos/#{owner}/#{repo}/commits/#{commit_sha}/pulls", accept: 'application/vnd.github.groot-preview+json') + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/pulls/#get-a-single-pull-request + def pull_request(owner:, repo:, number:) + resp = get("/repos/#{owner}/#{repo}/pulls/#{number}") + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/users/#get-a-single-user + def user(username:) + resp = get("/users/#{username}") + JSON.parse(resp.body) + end + + private + + def get(path, accept: 'application/vnd.github.v3+json') + Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: ENDPOINT.scheme == 'https') do |http| + headers = { 'Accept': accept, 'Authorization': "bearer #{@access_token}" } + http.get(path, headers).tap(&:value) + end + end +end + +module Git + class << self + def abbrev_ref(refname, repo_path:) + git('rev-parse', '--symbolic', '--abbrev-ref', refname, repo_path: repo_path).strip + end + + def rev_list(arg, first_parent: false, repo_path: nil) + git('rev-list', *[('--first-parent' if first_parent)].compact, arg, repo_path: repo_path).lines.map(&:chomp) + end + + def commit_message(sha) + git('log', '-1', '--pretty=format:%B', sha) + end + + def notes_message(sha) + git('log', '-1', '--pretty=format:%N', sha) + end + + def committer_name(sha) + git('log', '-1', '--pretty=format:%cn', sha) + end + + def committer_email(sha) + git('log', '-1', '--pretty=format:%cE', sha) + end + + private + + def git(*cmd, repo_path: nil) + env = {} + if repo_path + env['GIT_DIR'] = repo_path + end + out = IO.popen(env, ['git', *cmd], &:read) + unless $?.success? + abort "Failed to execute: git #{cmd.join(' ')}\n#{out}" + end + out + end + end +end + +github = GitHub.new(ENV.fetch('GITHUB_TOKEN')) + +repo_path, *rest = ARGV +rest.each_slice(3).map do |oldrev, newrev, _refname| + system('git', 'fetch', 'origin', 'refs/notes/commits:refs/notes/commits', exception: true) + + updated = false + Git.rev_list("#{oldrev}..#{newrev}", first_parent: true).each do |sha| + github.pulls(owner: 'ruby', repo: 'ruby', commit_sha: sha).each do |pull| + number = pull.fetch('number') + url = pull.fetch('html_url') + next unless url.start_with?('https://github.com/ruby/ruby/pull/') + + # "Merged" notes for "Squash and merge" + message = Git.commit_message(sha) + notes = Git.notes_message(sha) + if !message.include?(url) && !message.match(/[ (]##{number}[) ]/) && !notes.include?(url) + system('git', 'notes', 'append', '-m', "Merged: #{url}", sha, exception: true) + updated = true + end + + # "Merged-By" notes for "Rebase and merge" + if Git.committer_name(sha) == 'GitHub' && Git.committer_email(sha) == 'noreply@github.com' + username = github.pull_request(owner: 'ruby', repo: 'ruby', number: number).fetch('merged_by').fetch('login') + email = github.user(username: username).fetch('email') + email ||= SVN_TO_EMAILS[GITHUB_TO_SVN.fetch(username, username)]&.first + system('git', 'notes', 'append', '-m', "Merged-By: #{username}#{(" <#{email}>" if email)}", sha, exception: true) + updated = true + end + end + end + + if updated + system('git', 'push', 'origin', 'refs/notes/commits', exception: true) + end +end diff --git a/tool/notify-slack-commits.rb b/tool/notify-slack-commits.rb new file mode 100644 index 0000000000..73e22b9a03 --- /dev/null +++ b/tool/notify-slack-commits.rb @@ -0,0 +1,87 @@ +#!/usr/bin/env ruby + +require "net/https" +require "open3" +require "json" +require "digest/md5" + +SLACK_WEBHOOK_URLS = [ + ENV.fetch("SLACK_WEBHOOK_URL_ALERTS").chomp, # ruby-lang#alerts + ENV.fetch("SLACK_WEBHOOK_URL_COMMITS").chomp, # ruby-lang#commits + ENV.fetch("SLACK_WEBHOOK_URL_RUBY_JP").chomp, # ruby-jp#ruby-commits +] +GRAVATAR_OVERRIDES = { + "nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars0.githubusercontent.com/u/21976", + "noreply@github.com" => "https://avatars1.githubusercontent.com/u/9919", + "nurse@users.noreply.github.com" => "https://avatars1.githubusercontent.com/u/13423", + "svn-admin@ruby-lang.org" => "https://avatars1.githubusercontent.com/u/29403229", + "svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars1.githubusercontent.com/u/29403229", + "usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars2.githubusercontent.com/u/17790", + "usa@ruby-lang.org" => "https://avatars2.githubusercontent.com/u/17790", + "yui-knk@ruby-lang.org" => "https://avatars0.githubusercontent.com/u/5356517", + "znz@users.noreply.github.com" => "https://avatars3.githubusercontent.com/u/11857", +} + +def escape(s) + s.gsub(/[&<>]/, "&" => "&", "<" => "<", ">" => ">") +end + +ARGV.each_slice(3) do |oldrev, newrev, refname| + out, = Open3.capture2("git", "rev-parse", "--symbolic", "--abbrev-ref", refname) + branch = out.strip + + out, = Open3.capture2("git", "log", "--pretty=format:%H\n%h\n%cn\n%ce\n%ct\n%B", "--abbrev=10", "-z", "#{oldrev}..#{newrev}") + + attachments = [] + out.split("\0").reverse_each do |s| + sha, sha_abbr, committer, committeremail, committertime, body = s.split("\n", 6) + subject, body = body.split("\n", 2) + + # Append notes content to `body` if it's notes + if refname.match(%r[\Arefs/notes/\w+\z]) + # `--diff-filter=AM -M` to exclude rename by git's directory optimization + object = IO.popen(["git", "diff", "--diff-filter=AM", "-M", "--name-only", "#{sha}^..#{sha}"], &:read).chomp + if md = object.match(/\A(?<prefix>\h{2})\/?(?<rest>\h{38})\z/) + body = [body, IO.popen(["git", "notes", "show", md[:prefix] + md[:rest]], &:read)].join + end + end + + gravatar = GRAVATAR_OVERRIDES.fetch(committeremail) do + "https://www.gravatar.com/avatar/#{ Digest::MD5.hexdigest(committeremail.downcase) }" + end + + attachments << { + title: "#{ sha_abbr } (#{ branch }): #{ escape(subject) }", + title_link: "https://github.com/ruby/ruby/commit/#{ sha }", + text: escape((body || "").strip), + footer: committer, + footer_icon: gravatar, + ts: committertime.to_i, + color: '#24282D', + } + end + + # 100 attachments cannot be exceeded. 20 is recommended. https://api.slack.com/docs/message-attachments + attachments.each_slice(20).each do |attachments_group| + payload = { attachments: attachments_group } + + #Net::HTTP.post( + # URI.parse(SLACK_WEBHOOK_URL), + # JSON.generate(payload), + # "Content-Type" => "application/json" + #) + responses = SLACK_WEBHOOK_URLS.map do |url| + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.start do + req = Net::HTTP::Post.new(uri.path) + req.set_form_data(payload: payload.to_json) + http.request(req) + end + end + + results = responses.map { |resp| "#{resp.code} (#{resp.body})" }.join(', ') + puts "#{results} -- #{payload.to_json}" + end +end diff --git a/tool/outdate-bundled-gems.rb b/tool/outdate-bundled-gems.rb new file mode 100755 index 0000000000..47ee80bc89 --- /dev/null +++ b/tool/outdate-bundled-gems.rb @@ -0,0 +1,190 @@ +#!/usr/bin/ruby +require 'fileutils' +require 'rubygems' + +fu = FileUtils::Verbose + +until ARGV.empty? + case ARGV.first + when '--' + ARGV.shift + break + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Don't remove + fu = FileUtils::DryRun + when /\A--make=/ + # just to run when `make -n` + when /\A--mflags=(.*)/ + fu = FileUtils::DryRun if /\A-\S*n/ =~ $1 + when /\A--gem[-_]platform=(.*)/im + ## --gem-platform=PLATFORM Platform in RubyGems style + gem_platform = $1 + ruby_platform = nil + when /\A--ruby[-_]platform=(.*)/im + ## --ruby-platform=PLATFORM Platform in Ruby style + ruby_platform = $1 + gem_platform = nil + when /\A--ruby[-_]version=(.*)/im + ## --ruby-version=VERSION Ruby version to keep + ruby_version = $1 + when /\A--only=(?:(curdir|srcdir)|all)\z/im + ## --only=(curdir|srcdir|all) Specify directory to remove gems from + only = $1&.downcase + when /\A--all\z/im + ## --all Remove all gems not only bundled gems + all = true + when /\A--help\z/im + ## --help Print this message + puts "Usage: #$0 [options] [srcdir]" + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit + when /\A-/ + raise "#{$0}: unknown option: #{ARGV.first}" + else + break + end + ## + ARGV.shift +end + +gem_platform ||= Gem::Platform.new(ruby_platform).to_s if ruby_platform + +class Removal + attr_reader :base + + def initialize(base = nil) + @base = (File.join(base, "/") if base) + @remove = {} + end + + def prefixed(name) + @base ? File.join(@base, name) : name + end + + def stripped(name) + if @base && name.start_with?(@base) + name[@base.size..-1] + else + name + end + end + + def slash(name) + name.sub(%r[[^/]\K\z], '/') + end + + def exist?(name) + !@remove.fetch(name) {|k| @remove[k] = !File.exist?(prefixed(name))} + end + def directory?(name) + !@remove.fetch(slash(name)) {|k| @remove[k] = !File.directory?(prefixed(name))} + end + + def unlink(name) + @remove[stripped(name)] = :rm_f + end + def rmdir(name) + @remove[slash(stripped(name))] = :rm_rf + end + + def glob(pattern, *rest) + Dir.glob(prefixed(pattern), *rest) {|n| + yield stripped(n) + } + end + + def sorted + @remove.sort_by {|k, | [-k.count("/"), k]} + end + + def each_file + sorted.each {|k, v| yield prefixed(k) if v == :rm_f} + end + + def each_directory + sorted.each {|k, v| yield prefixed(k) if v == :rm_rf} + end +end + +srcdir = Removal.new(ARGV.shift) +curdir = !srcdir.base || File.identical?(srcdir.base, ".") ? srcdir : Removal.new + +bundled = File.readlines("#{srcdir.base}gems/bundled_gems"). + grep(/^(\w[^\#\s]+)\s+[^\#\s]+(?:\s+[^\#\s]+\s+([^\#\s]+))?/) {$~.captures}.to_h rescue nil + +srcdir.glob(".bundle/gems/*/") do |dir| + base = File.basename(dir) + next if !all && bundled && !bundled.key?(base[/\A.+(?=-)/]) + unless srcdir.exist?("gems/#{base}.gem") + srcdir.rmdir(dir) + end +end + +srcdir.glob(".bundle/.timestamp/*.revision") do |file| + unless bundled&.fetch(File.basename(file, ".revision"), nil) + srcdir.unlink(file) + end +end + +srcdir.glob(".bundle/specifications/*.gemspec") do |spec| + unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}/") + srcdir.unlink(spec) + end +end + +curdir.glob(".bundle/specifications/*.gemspec") do |spec| + unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}") + curdir.unlink(spec) + end +end + +curdir.glob(".bundle/gems/*/") do |dir| + base = File.basename(dir) + unless curdir.exist?(".bundle/specifications/#{base}.gemspec") or + curdir.exist?("#{dir}/.bundled.#{base}.gemspec") + curdir.rmdir(dir) + end +end + +curdir.glob(".bundle/{extensions,.timestamp}/*/") do |dir| + unless gem_platform and File.fnmatch?(gem_platform, File.basename(dir)) + curdir.rmdir(dir) + end +end + +if gem_platform + curdir.glob(".bundle/{extensions,.timestamp}/#{gem_platform}/*/") do |dir| + unless ruby_version and File.fnmatch?(ruby_version, File.basename(dir, '-static')) + curdir.rmdir(dir) + end + end +end + +if ruby_version + curdir.glob(".bundle/extensions/#{gem_platform || '*'}/#{ruby_version}/*/") do |dir| + unless curdir.exist?(".bundle/specifications/#{File.basename(dir)}.gemspec") + curdir.rmdir(dir) + end + end + + curdir.glob(".bundle/.timestamp/#{gem_platform || '*'}/#{ruby_version}/.*.time") do |stamp| + dir = stamp[%r[/\.([^/]+)\.time\z], 1].gsub('.-.', '/')[%r[\A[^/]+/[^/]+]] + unless curdir.directory?(File.join(".bundle", dir)) + curdir.unlink(stamp) + end + end +end + +unless only == "curdir" + srcdir.each_file {|f| fu.rm_f(f)} + srcdir.each_directory {|d| fu.rm_rf(d)} +end +unless only == "srcdir" or curdir.equal?(srcdir) + curdir.each_file {|f| fu.rm_f(f)} + curdir.each_directory {|d| fu.rm_rf(d)} +end diff --git a/tool/prereq.status b/tool/prereq.status index 6de00c8a92..6aca615e90 100644 --- a/tool/prereq.status +++ b/tool/prereq.status @@ -41,4 +41,5 @@ s,@rubylibprefix@,,g s,@srcdir@,.,g s/@[A-Za-z][A-Za-z0-9_]*@//g -s/{\$([A-Za-z]*)}//g +s/{\$([^(){}]*)}//g +s/^!/#!/ diff --git a/tool/pure_parser.rb b/tool/pure_parser.rb deleted file mode 100755 index 21c87cc5d6..0000000000 --- a/tool/pure_parser.rb +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/ruby -pi.bak -BEGIN { - # pathological setting - ENV['LANG'] = ENV['LC_MESSAGES'] = ENV['LC_ALL'] = 'C' - - require_relative 'lib/colorize' - - colorize = Colorize.new - file = ARGV.shift - begin - version = IO.popen(ARGV+%w[--version], "rb", &:read) - rescue Errno::ENOENT - abort "Failed to run `#{colorize.fail ARGV.join(' ')}'; You may have to install it." - end - unless /\Abison .* (\d+)\.\d+/ =~ version - puts colorize.fail("not bison") - exit - end - exit if $1.to_i >= 3 - ARGV.clear - ARGV.push(file) -} -$_.sub!(/^%define\s+api\.pure/, '%pure-parser') -$_.sub!(/^%define\s+.*/, '') diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 2f70f46ad3..874c3ef1d9 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -28,6 +28,7 @@ begin rescue LoadError $" << "zlib.rb" end +require_relative 'lib/path' INDENT = " "*36 STDOUT.sync = true @@ -61,6 +62,7 @@ def parse_args(argv = ARGV) opt.on('-n', '--dry-run') {$dryrun = true} opt.on('--dest-dir=DIR') {|dir| $destdir = dir} opt.on('--extout=DIR') {|dir| $extout = (dir unless dir.empty?)} + opt.on('--ext-build-dir=DIR') {|v| $ext_build_dir = v } opt.on('--make=COMMAND') {|make| $make = make} opt.on('--mantype=MAN') {|man| $mantype = man} opt.on('--make-flags=FLAGS', '--mflags', Shellwords) do |v| @@ -95,11 +97,25 @@ def parse_args(argv = ARGV) opt.on('--gnumake') {gnumake = true} opt.on('--debug-symbols=SUFFIX', /\w+/) {|name| $debug_symbols = ".#{name}"} + unless $install_procs.empty? + w = (w = ENV["COLUMNS"] and (w = w.to_i) > 80) ? w - 30 : 50 + opt.on("\n""Types for --install and --exclude:") + mesg = +" " + $install_procs.each_key do |t| + if mesg.size + t.size > w + opt.on(mesg) + mesg = +" " + end + mesg << " " << t.to_s + end + opt.on(mesg) + end + opt.order!(argv) do |v| case v when /\AINSTALL[-_]([-\w]+)=(.*)/ argv.unshift("--#{$1.tr('_', '-')}=#{$2}") - when /\A\w[-\w+]*=\z/ + when /\A\w[-\w]*=/ mflags << v when /\A\w[-\w+]*\z/ $install << v.intern @@ -134,6 +150,7 @@ def parse_args(argv = ARGV) end $destdir ||= $mflags.defined?("DESTDIR") + $destdir = File.expand_path($destdir) unless $destdir.empty? if $extout ||= $mflags.defined?("EXTOUT") RbConfig.expand($extout) end @@ -142,7 +159,7 @@ def parse_args(argv = ARGV) if $installed_list ||= $mflags.defined?('INSTALLED_LIST') RbConfig.expand($installed_list, RbConfig::CONFIG) - $installed_list = open($installed_list, "ab") + $installed_list = File.open($installed_list, "ab") $installed_list.sync = true end @@ -151,6 +168,19 @@ def parse_args(argv = ARGV) $dir_mode ||= $prog_mode | 0700 $script_mode ||= $prog_mode + if $ext_build_dir.nil? + raise OptionParser::MissingArgument.new("--ext-build-dir=DIR") + end +end + +Compressors = {".gz"=>"gzip", ".bz2"=>"bzip2"} +def Compressors.for(type) + ext = File.extname(type) + if compress = fetch(ext, nil) + [type.chomp(ext), ext, compress] + else + [type, *find {|_, z| system(z, in: IO::NULL, out: IO::NULL)}] + end end $install_procs = Hash.new {[]} @@ -201,15 +231,20 @@ def ln_sf(src, dest) end $made_dirs = {} + +def dir_creating(dir) + $made_dirs.fetch(dir) do + $made_dirs[dir] = true + $installed_list.puts(File.join(dir, "")) if $installed_list + yield if defined?(yield) + end +end + def makedirs(dirs) dirs = fu_list(dirs) dirs.collect! do |dir| realdir = with_destdir(dir) - realdir unless $made_dirs.fetch(dir) do - $made_dirs[dir] = true - $installed_list.puts(File.join(dir, "")) if $installed_list - File.directory?(realdir) - end + realdir unless dir_creating(dir) {File.directory?(realdir)} end.compact! super(dirs, :mode => $dir_mode) unless dirs.empty? end @@ -287,11 +322,11 @@ def install_recursive(srcdir, dest, options = {}) end def open_for_install(path, mode) - data = open(realpath = with_destdir(path), "rb") {|f| f.read} rescue nil + data = File.binread(realpath = with_destdir(path)) rescue nil newdata = yield unless $dryrun unless newdata == data - open(realpath, "wb", mode) {|f| f.write newdata} + File.open(realpath, "wb", mode) {|f| f.write newdata} end File.chmod(mode, realpath) end @@ -342,6 +377,9 @@ rubyw_install_name = CONFIG["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name bindir = CONFIG["bindir", true] +if CONFIG["libdirname"] == "archlibdir" + archbindir = bindir.sub(%r[/\K(?=[^/]+\z)]) {CONFIG["config_target"] + "/"} +end libdir = CONFIG[CONFIG.fetch("libdirname", "libdir"), true] rubyhdrdir = CONFIG["rubyhdrdir", true] archhdrdir = CONFIG["rubyarchhdrdir"] || (rubyhdrdir + "/" + CONFIG['arch']) @@ -365,109 +403,13 @@ load_relative = CONFIG["LIBRUBY_RELATIVE"] == 'yes' rdoc_noinst = %w[created.rid] -install?(:local, :arch, :bin, :'bin-arch') do - prepare "binary commands", bindir - - install ruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - if rubyw_install_name and !rubyw_install_name.empty? - install rubyw_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - if File.exist? goruby_install_name+exeext - install goruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip - end - if enable_shared and dll != lib - install dll, bindir, :mode => $prog_mode, :strip => $strip - end -end - -install?(:local, :arch, :lib, :'lib-arch') do - prepare "base libraries", libdir - - install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc - install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" - if dll == lib and dll != arc - for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] - ln_sf(dll, File.join(libdir, link)) - end - end - - prepare "arch files", archlibdir - install "rbconfig.rb", archlibdir, :mode => $data_mode - if CONFIG["ARCHFILE"] - for file in CONFIG["ARCHFILE"].split - install file, archlibdir, :mode => $data_mode - end - end -end - -install?(:local, :arch, :data) do - pc = CONFIG["ruby_pc"] - if pc and File.file?(pc) and File.size?(pc) - prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") - install pc, pkgconfigdir, :mode => $data_mode - end -end - -install?(:ext, :arch, :'ext-arch') do - prepare "extension objects", archlibdir - noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split - install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) - prepare "extension objects", sitearchlibdir - prepare "extension objects", vendorarchlibdir - if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and - !(extso = extso.gsub(/\\\n/, '').split).empty? - libpathenv = CONFIG["LIBPATHENV"] - dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] - prepare "external libraries", dest - for file in extso - install file, dest, :mode => $prog_mode - end - end -end -install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do - prepare "extension headers", archhdrdir - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.obj", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.pch", :mode => $data_mode) - install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "rb_mjit_header-*.pdb", :mode => $data_mode) -end -install?(:ext, :comm, :'ext-comm') do - prepare "extension scripts", rubylibdir - install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) - prepare "extension scripts", sitelibdir - prepare "extension scripts", vendorlibdir -end -install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do - hdrdir = rubyhdrdir + "/ruby" - prepare "extension headers", hdrdir - install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) -end - -install?(:doc, :rdoc) do - if $rdocdir - ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") - prepare "rdoc", ridatadir - install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :html) do - if $htmldir - prepare "html-docs", docdir - install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) - end -end -install?(:doc, :capi) do - prepare "capi-docs", docdir - install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode -end - prolog_script = <<EOS bindir="#{load_relative ? '${0%/*}' : bindir.gsub(/\"/, '\\\\"')}" EOS -if CONFIG["LIBRUBY_RELATIVE"] != 'yes' and libpathenv = CONFIG["LIBPATHENV"] +if !load_relative and libpathenv = CONFIG["LIBPATHENV"] pathsep = File::PATH_SEPARATOR prolog_script << <<EOS -libdir="#{load_relative ? '$\{bindir%/bin\}/lib' : libdir.gsub(/\"/, '\\\\"')}" +libdir="#{libdir.gsub(/\"/, '\\\\"')}" export #{libpathenv}="$libdir${#{libpathenv}:+#{pathsep}$#{libpathenv}}" EOS end @@ -504,7 +446,7 @@ $script_installer = Class.new(installer) do if trans = CONFIG["program_transform_name"] exp = [] trans.gsub!(/\$\$/, '$') - trans.scan(%r[\G[\s;]*(/(?:\\.|[^/])*/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*)\3((?:(?!\3)(?:\\.|.))*)\3([gi]*)]) do + trans.scan(%r[\G[\s;]*(/(?:\\.|[^/])*+/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*+)\3((?:(?!\3)(?:\\.|.))*+)\3([gi]*)]) do |addr, cmd, sep, pat, rep, opt| addr &&= Regexp.new(addr[/\A\/(.*)\/\z/, 1]) case cmd @@ -542,7 +484,7 @@ $script_installer = Class.new(installer) do def install(src, cmd) cmd = cmd.sub(/[^\/]*\z/m) {|n| transform(n)} - shebang, body = open(src, "rb") do |f| + shebang, body = File.open(src, "rb") do |f| next f.gets, f.read end shebang or raise "empty file - #{src}" @@ -576,126 +518,6 @@ $script_installer = Class.new(installer) do break new(ruby_shebang, ruby_bin, ruby_install_name, nil, trans) end -install?(:local, :comm, :bin, :'bin-comm') do - prepare "command scripts", bindir - - install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| - $script_installer.install(src, cmd) - end -end - -install?(:local, :comm, :lib) do - prepare "library scripts", rubylibdir - noinst = %w[*.txt *.rdoc *.gemspec] - install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) -end - -install?(:local, :comm, :hdr, :'comm-hdr') do - prepare "common headers", rubyhdrdir - - noinst = [] - unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ - noinst << "win32.h" - end - noinst = nil if noinst.empty? - install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) -end - -install?(:local, :comm, :man) do - mdocs = Dir["#{srcdir}/man/*.[1-9]"] - prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} - - case $mantype - when /\.(?:(gz)|bz2)\z/ - compress = $1 ? "gzip" : "bzip2" - suffix = $& - end - mandir = File.join(mandir, "man") - has_goruby = File.exist?(goruby_install_name+exeext) - require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ $mantype - mdocs.each do |mdoc| - next unless File.file?(mdoc) and open(mdoc){|fh| fh.read(1) == '.'} - base = File.basename(mdoc) - if base == "goruby.1" - next unless has_goruby - end - - destdir = mandir + (section = mdoc[/\d+$/]) - destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) - destfile = File.join(destdir, "#{destname}.#{section}") - - if /\Adoc\b/ =~ $mantype - if compress - w = open(mdoc) {|f| - stdin = STDIN.dup - STDIN.reopen(f) - begin - destfile << suffix - IO.popen(compress, &:read) - ensure - STDIN.reopen(stdin) - stdin.close - end - } - open_for_install(destfile, $data_mode) {w} - else - install mdoc, destfile, :mode => $data_mode - end - else - class << (w = []) - alias print push - end - if File.basename(mdoc).start_with?('bundle') || - File.basename(mdoc).start_with?('gemfile') - w = File.read(mdoc) - else - open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} - w = w.join("") - end - if compress - require 'tmpdir' - Dir.mktmpdir("man") {|d| - dest = File.join(d, File.basename(destfile)) - File.open(dest, "wb") {|f| f.write w} - if system(compress, dest) - w = File.open(dest+suffix, "rb") {|f| f.read} - destfile << suffix - end - } - end - open_for_install(destfile, $data_mode) {w} - end - end -end - -install?(:dbg, :nodefault) do - prepare "debugger commands", bindir - prepare "debugger scripts", rubylibdir - conf = RbConfig::MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) - Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| - cmd = $script_installer.transform(File.basename(src, ".in")) - open_for_install(File.join(bindir, cmd), $script_mode) { - RbConfig.expand(File.read(src), conf) - } - end - install File.join(srcdir, "misc/lldb_cruby.py"), File.join(rubylibdir, "lldb_cruby.py") - install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") - if $debug_symbols - { - ruby_install_name => bindir, - rubyw_install_name => bindir, - goruby_install_name => bindir, - dll => libdir, - }.each do |src, dest| - next if src.empty? - src += $debug_symbols - if File.directory?(src) - install_recursive src, File.join(dest, src) - end - end - end -end - module RbInstall def self.no_write(options = nil) u = File.umask(0022) @@ -705,6 +527,8 @@ module RbInstall const_set(:FileUtils, fu::NoWrite) fu end + # RubyGems 3.0.0 or later supports `dir_mode`, but it uses + # `File` method to apply it, not `FileUtils`. dir_mode = options.delete(:dir_mode) if options end yield @@ -721,76 +545,132 @@ module RbInstall module Specs class FileCollector - def initialize(gemspec) + def self.for(srcdir, type, gemspec) + relative_base = (File.dirname(gemspec) if gemspec.include?("/")) + const_get(type.capitalize).new(gemspec, srcdir, relative_base) + end + + attr_reader :gemspec, :srcdir, :relative_base + def initialize(gemspec, srcdir, relative_base) @gemspec = gemspec - @base_dir = File.dirname(gemspec) + @srcdir = srcdir + @relative_base = relative_base end def collect - (ruby_libraries + built_libraries).sort + requirable_features.sort end private - def type - /\/(ext|lib)?\/.*?\z/ =~ @base_dir - $1 + + def features_from_makefile(makefile_path) + makefile = File.read(makefile_path) + + name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + return [] if name.nil? || name.empty? + + feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = feature.sub("$(TARGET)", name) + + target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? + + Array(feature) end - def ruby_libraries - case type - when "ext" - prefix = "#{$extout}/common/" - base = "#{prefix}#{relative_base}" - when "lib" - base = @base_dir - prefix = base.sub(/lib\/.*?\z/, "") - # for lib/net/net-smtp.gemspec - if m = File.basename(@gemspec, ".gemspec").match(/.*\-(.*)\z/) - base = "#{@base_dir}/#{m[1]}" unless remove_prefix(prefix, @base_dir).include?(m[1]) + class Ext < self + def requirable_features + # install ext only when it's configured + return [] unless File.exist?(makefile_path) + + ruby_features + ext_features + end + + private + + def ruby_features + Dir.glob("**/*.rb", base: "#{makefile_dir}/lib") + end + + def ext_features + features_from_makefile(makefile_path) + end + + def makefile_path + if File.exist?("#{makefile_dir}/Makefile") + "#{makefile_dir}/Makefile" + else + # for out-of-place build + "#{$ext_build_dir}/#{relative_base}/Makefile" end end - files = if base - Dir.glob("#{base}{.rb,/**/*.rb}").collect do |ruby_source| - remove_prefix(prefix, ruby_source) - end - else - [File.basename(@gemspec, '.gemspec') + '.rb'] - end - - case File.basename(@gemspec, ".gemspec") - when "net-http" - files << "lib/net/https.rb" - when "optparse" - files << "lib/optionparser.rb" + def makefile_dir + "#{root}/#{relative_base}" end - files + def root + File.expand_path($ext_build_dir, srcdir) + end end - def built_libraries - case type - when "ext" - prefix = "#{$extout}/#{CONFIG['arch']}/" - base = "#{prefix}#{relative_base}" - dlext = CONFIG['DLEXT'] - Dir.glob("#{base}{.#{dlext},/**/*.#{dlext}}").collect do |built_library| - remove_prefix(prefix, built_library) + class Lib < self + def requirable_features + ruby_features + ext_features + end + + private + + def ruby_features + gemname = File.basename(gemspec, ".gemspec") + base = relative_base || gemname + # for lib/net/net-smtp.gemspec + if m = /.*(?=-(.*)\z)/.match(gemname) + base = File.join(base, *m.to_a.select {|n| !base.include?(n)}) end - when "lib" - [] - else - [] + files = Dir.glob("#{base}{.rb,/**/*.rb}", base: root) + if !relative_base and files.empty? # no files at the toplevel + # pseudo gem like ruby2_keywords + files << "#{gemname}.rb" + end + + case gemname + when "net-http" + files << "net/https.rb" + when "optparse" + files << "optionparser.rb" + end + + files end - end - def relative_base - /\/#{Regexp.escape(type)}\/(.*?)\z/ =~ @base_dir - $1 + def ext_features + loaded_gemspec = load_gemspec("#{root}/#{gemspec}") + extension = loaded_gemspec.extensions.first + return [] unless extension + + extconf = File.expand_path(extension, srcdir) + ext_build_dir = File.dirname(extconf) + makefile_path = "#{ext_build_dir}/Makefile" + return [] unless File.exist?(makefile_path) + + features_from_makefile(makefile_path) + end + + def root + "#{srcdir}/lib" + end end - def remove_prefix(prefix, string) - string.sub(/\A#{Regexp.escape(prefix)}/, "") + class UnpackedGem < self + def collect + base = @srcdir or return [] + Dir.glob("**/*", File::FNM_DOTMATCH, base: base).select do |n| + next if n == "." + next if File.fnmatch?("*.gemspec", n, File::FNM_DOTMATCH|File::FNM_PATHNAME) + !File.directory?(File.join(base, n)) + end + end end end end @@ -828,14 +708,79 @@ module RbInstall end end - class GemInstaller < Gem::Installer - end + class UnpackedInstaller < Gem::Installer + # This method is mostly copied from old version of Gem::Installer#install + def install_with_default_gem + verify_gem_home - class UnpackedInstaller < GemInstaller - def write_cache_file + # The name and require_paths must be verified first, since it could contain + # ruby code that would be eval'ed in #ensure_loadable_spec + verify_spec + + ensure_loadable_spec + + if options[:install_as_default] + Gem.ensure_default_gem_subdirectories gem_home + else + Gem.ensure_gem_subdirectories gem_home + end + + return true if @force + + ensure_dependencies_met unless @ignore_dependencies + + run_pre_install_hooks + + # Set loaded_from to ensure extension_dir is correct + if @options[:install_as_default] + spec.loaded_from = default_spec_file + else + spec.loaded_from = spec_file + end + + # Completely remove any previous gem files + FileUtils.rm_rf gem_dir + FileUtils.rm_rf spec.extension_dir + + dir_mode = options[:dir_mode] + FileUtils.mkdir_p gem_dir, mode: dir_mode && 0o755 + + if @options[:install_as_default] + extract_bin + write_default_spec + else + extract_files + + build_extensions + write_build_info_file + run_post_build_hooks + end + + generate_bin + generate_plugins + + unless @options[:install_as_default] + write_spec + write_cache_file + end + + File.chmod(dir_mode, gem_dir) if dir_mode + + say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? + + Gem::Specification.add_spec(spec) unless @install_dir + + load_plugin + + run_post_install_hooks + + spec + rescue Errno::EACCES => e + # Permission denied - /path/to/foo + raise Gem::FilePermissionError, e.message.split(" - ").last end - def build_extensions + def write_cache_file end def shebang(bin_file_name) @@ -858,11 +803,6 @@ module RbInstall super end - def generate_bin_script(filename, bindir) - return if same_bin_script?(filename, bindir) - super - end - def same_bin_script?(filename, bindir) path = File.join(bindir, formatted_program_filename(filename)) begin @@ -881,19 +821,25 @@ module RbInstall super unless $dryrun $installed_list.puts(without_destdir(default_spec_file)) if $installed_list end - end - class GemInstaller def install spec.post_install_message = nil - RbInstall.no_write(options) {super} + dir_creating(without_destdir(gem_dir)) + RbInstall.no_write(options) { install_with_default_gem } + end + + # Now build-ext builds all extensions including bundled gems. + def build_extensions end def generate_bin_script(filename, bindir) + return if same_bin_script?(filename, bindir) name = formatted_program_filename(filename) unless $dryrun super - File.chmod($script_mode, File.join(bindir, name)) + script = File.join(bindir, name) + File.chmod($script_mode, script) + File.unlink("#{script}.lock") rescue nil end $installed_list.puts(File.join(without_destdir(bindir), name)) if $installed_list end @@ -908,39 +854,46 @@ module RbInstall $installed_list.puts(d+"/") if $installed_list end end - end -end - -# :startdoc: -install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do - install_default_gem('lib', srcdir, bindir) -end -install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do - install_default_gem('ext', srcdir, bindir) + def load_plugin + # Suppress warnings for constant re-assignment + verbose, $VERBOSE = $VERBOSE, nil + super + ensure + $VERBOSE = verbose + end + end end -def load_gemspec(file, expanded = false) +def load_gemspec(file, base = nil, files: nil) file = File.realpath(file) code = File.read(file, encoding: "utf-8:-") - code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split\([^\)]*\)/m) do - files = [] - if expanded - base = File.dirname(file) - Dir.glob("**/*", File::FNM_DOTMATCH, base: base) do |n| - case File.basename(n); when ".", ".."; next; end - next if File.directory?(File.join(base, n)) - files << n.dump - end - end - "[" + files.join(", ") + "]" + + code.gsub!(/^ *#.*/, "") + spec_files = files ? files.map(&:dump).join(", ") : "" + code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split(\([^\)]*\))?/m) do + "[" + spec_files + "]" + end \ + or + code.gsub!(/IO\.popen\(.*git.*?\)/) do + "[" + spec_files + "] || itself" end + spec = eval(code, binding, file) + # for out-of-place build + collected_files = files ? spec.files.concat(files).uniq : spec.files + spec.files = collected_files.map do |f| + if !File.exist?(File.join(base || ".", f)) && f.end_with?(".rb") + "lib/#{f}" + else + f + end + end unless Gem::Specification === spec raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." end - spec.loaded_from = file - spec.files.reject! {|n| n.end_with?(".gemspec") or n.start_with?(".git")} + spec.loaded_from = base ? File.join(base, File.basename(file)) : file + spec.date = RUBY_RELEASE_DATE spec end @@ -950,6 +903,7 @@ def install_default_gem(dir, srcdir, bindir) install_dir = with_destdir(gem_dir) prepare "default gems from #{dir}", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_default_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end @@ -966,13 +920,13 @@ def install_default_gem(dir, srcdir, bindir) } default_spec_dir = Gem.default_specifications_dir - gems = Dir.glob("#{srcdir}/#{dir}/**/*.gemspec").map {|src| - spec = load_gemspec(src) - file_collector = RbInstall::Specs::FileCollector.new(src) - files = file_collector.collect - next if files.empty? - spec.files = files - spec + base = "#{srcdir}/#{dir}" + gems = Dir.glob("**/*.gemspec", base: base).map {|src| + files = RbInstall::Specs::FileCollector.for(srcdir, dir, src).collect + if files.empty? + next + end + load_gemspec("#{base}/#{src}", files: files) } gems.compact.sort_by(&:name).each do |gemspec| old_gemspecs = Dir[File.join(with_destdir(default_spec_dir), "#{gemspec.name}-*.gemspec")] @@ -991,15 +945,264 @@ def install_default_gem(dir, srcdir, bindir) end end +def mdoc_file?(mdoc) + /^\.Nm / =~ File.read(mdoc, 1024) +end + +# :startdoc: + +install?(:local, :arch, :bin, :'bin-arch') do + prepare "binary commands", (dest = archbindir || bindir) + + def (bins = []).add(name) + push(name) + name + end + + install bins.add(ruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + if rubyw_install_name and !rubyw_install_name.empty? + install bins.add(rubyw_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support + # to load and execute the second .wasm file. Both are required to execute ruby + if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" + install bins.add(ruby_install_name+".wasm"), dest, :mode => $prog_mode, :strip => $strip + end + if File.exist? goruby_install_name+exeext + install bins.add(goruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + if enable_shared and dll != lib + install bins.add(dll), dest, :mode => $prog_mode, :strip => $strip + end + if archbindir + prepare "binary command links", bindir + relpath = Path.relative(archbindir, bindir) + bins.each do |f| + ln_sf(File.join(relpath, f), File.join(bindir, f)) + end + end +end + +install?(:local, :arch, :lib, :'lib-arch') do + prepare "base libraries", libdir + + install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc + install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" + if dll == lib and dll != arc + for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] + ln_sf(dll, File.join(libdir, link)) + end + end + + prepare "arch files", archlibdir + install "rbconfig.rb", archlibdir, :mode => $data_mode + if CONFIG["ARCHFILE"] + for file in CONFIG["ARCHFILE"].split + install file, archlibdir, :mode => $data_mode + end + end +end + +install?(:local, :arch, :data) do + pc = CONFIG["ruby_pc"] + if pc and File.file?(pc) and File.size?(pc) + prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") + install pc, pkgconfigdir, :mode => $data_mode + if (pkgconfig_base = CONFIG["libdir", true]) != libdir + prepare "pkgconfig data link", File.join(pkgconfig_base, "pkgconfig") + ln_sf(File.join("..", Path.relative(pkgconfigdir, pkgconfig_base), pc), + File.join(pkgconfig_base, "pkgconfig", pc)) + end + end +end + +install?(:ext, :arch, :'ext-arch') do + prepare "extension objects", archlibdir + noinst = %w[-* -*/] | (CONFIG["no_install_files"] || "").split + install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) + prepare "extension objects", sitearchlibdir + prepare "extension objects", vendorarchlibdir + if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and + !(extso = extso.gsub(/\\\n/, '').split).empty? + libpathenv = CONFIG["LIBPATHENV"] + dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] + prepare "external libraries", dest + for file in extso + install file, dest, :mode => $prog_mode + end + end +end + +install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do + prepare "extension headers", archhdrdir + install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:ext, :comm, :'ext-comm') do + prepare "extension scripts", rubylibdir + install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) + prepare "extension scripts", sitelibdir + prepare "extension scripts", vendorlibdir +end + +install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do + hdrdir = rubyhdrdir + "/ruby" + prepare "extension headers", hdrdir + install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) +end + +install?(:doc, :rdoc) do + if $rdocdir + ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") + prepare "rdoc", ridatadir + install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :html) do + if $htmldir + prepare "html-docs", docdir + install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :capi) do + prepare "capi-docs", docdir + install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode +end + +install?(:local, :comm, :bin, :'bin-comm') do + prepare "command scripts", bindir + + install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| + $script_installer.install(src, cmd) + end +end + +install?(:local, :comm, :lib) do + prepare "library scripts", rubylibdir + noinst = %w[*.txt *.rdoc *.gemspec] + install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) +end + +install?(:local, :comm, :hdr, :'comm-hdr') do + prepare "common headers", rubyhdrdir + + noinst = [] + unless RUBY_PLATFORM =~ /mswin|mingw|bccwin/ + noinst << "win32.h" + end + noinst = nil if noinst.empty? + install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) +end + +install?(:local, :comm, :man) do + mdocs = Dir["#{srcdir}/man/*.[1-9]"] + prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} + + mantype, suffix, compress = Compressors.for($mantype) + has_goruby = File.exist?(goruby_install_name+exeext) + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ mantype + mdocs.each do |mdoc| + next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' + base = File.basename(mdoc) + if base == "goruby.1" + next unless has_goruby + end + + destdir = File.join(mandir, "man" + (section = mdoc[/\d+$/])) + destname = $script_installer.transform(base.chomp(".#{section}")) + destfile = File.join(destdir, "#{destname}.#{section}") + + if /\Adoc\b/ =~ mantype or !mdoc_file?(mdoc) + if compress + begin + w = IO.popen(compress, "rb", in: mdoc, &:read) + rescue + else + destfile << suffix + end + end + if w + open_for_install(destfile, $data_mode) {w} + else + install mdoc, destfile, :mode => $data_mode + end + else + class << (w = []) + alias print push + end + File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + w = w.join("") + if compress + begin + w = IO.popen(compress, "r+b") do |f| + Thread.start {f.write w; f.close_write} + f.read + end + rescue + else + destfile << suffix + end + end + open_for_install(destfile, $data_mode) {w} + end + end +end + +install?(:dbg, :nodefault) do + prepare "debugger commands", bindir + prepare "debugger scripts", rubylibdir + conf = MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) + Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| + cmd = $script_installer.transform(File.basename(src, ".in")) + open_for_install(File.join(bindir, cmd), $script_mode) { + RbConfig.expand(File.read(src), conf) + } + end + Dir.glob(File.join(srcdir, "misc/lldb_*")) do |src| + if File.directory?(src) + install_recursive src, File.join(rubylibdir, File.basename(src)) + else + install src, rubylibdir + end + end + install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") + if $debug_symbols + { + ruby_install_name => archbindir || bindir, + rubyw_install_name => archbindir || bindir, + goruby_install_name => archbindir || bindir, + dll => libdir, + }.each do |src, dest| + next if src.empty? + src += $debug_symbols + if File.directory?(src) + install_recursive src, File.join(dest, src) + end + end + end +end + +install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do + install_default_gem('lib', srcdir, bindir) +end + +install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do + install_default_gem('ext', srcdir, bindir) +end + install?(:ext, :comm, :gem, :'bundled-gems') do gem_dir = Gem.default_dir install_dir = with_destdir(gem_dir) prepare "bundled gems", gem_dir RbInstall.no_write do + # Record making directories makedirs(Gem.ensure_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) end installed_gems = {} + skipped = {} options = { :install_dir => install_dir, :bin_dir => with_destdir(bindir), @@ -1011,35 +1214,64 @@ install?(:ext, :comm, :gem, :'bundled-gems') do :wrappers => true, :format_executable => true, } - gem_ext_dir = "#$extout/gems/#{CONFIG['arch']}" + extensions_dir = Gem::StubSpecification.gemspec_stub("", gem_dir, gem_dir).extensions_dir + specifications_dir = File.join(gem_dir, "specifications") + build_dir = Gem::StubSpecification.gemspec_stub("", ".bundle", ".bundle").extensions_dir + + # We are about to build extensions, and want to configure extensions with the + # newly installed ruby. + Gem.instance_variable_set(:@ruby, with_destdir(File.join(bindir, ruby_install_name))) + # Prevent fake.rb propagation. It conflicts with the natural mkmf configs of + # the newly installed ruby. + ENV.delete('RUBYOPT') + + collector = RbInstall::Specs::FileCollector::UnpackedGem File.foreach("#{srcdir}/gems/bundled_gems") do |name| next if /^\s*(?:#|$)/ =~ name next unless /^(\S+)\s+(\S+).*/ =~ name + gem = $1 gem_name = "#$1-#$2" - path = "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec" - if File.exist?(path) - spec = load_gemspec(path) - else - path = "#{srcdir}/.bundle/gems/#{gem_name}/#$1.gemspec" - next unless File.exist?(path) - spec = load_gemspec(path, true) + path = [ + # gemspec that removed duplicated dependencies of bundled gems + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec", + # gemspec for C ext gems, It has the original dependencies + # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec", + # original gemspec generated by rubygems + "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + ].find { |gemspec| File.exist?(gemspec) } + if path.nil? + skipped[gem_name] = "gemspec not found" + next end - next unless spec.platform == Gem::Platform::RUBY - next unless spec.full_name == gem_name - spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" - if File.directory?(ext = "#{gem_ext_dir}/#{spec.full_name}") - spec.extensions[0] ||= "-" + base = "#{srcdir}/.bundle/gems/#{gem_name}" + files = collector.new(path, base, nil).collect + files.delete("#{gem}.gemspec") + files.delete("#{gem_name}.gemspec") + spec = load_gemspec(path, base, files: files) + unless spec.platform == Gem::Platform::RUBY + skipped[gem_name] = "not ruby platform (#{spec.platform})" + next + end + unless spec.full_name == gem_name + skipped[gem_name] = "full name unmatch #{spec.full_name}" + next + end + # Skip install C ext bundled gem if it is build failed or not found + if !spec.extensions.empty? && !File.exist?("#{build_dir}/#{gem_name}/gem.build_complete") + skipped[gem_name] = "extensions not found or build failed #{spec.full_name}" + next end + spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + package = RbInstall::DirPackage.new spec ins = RbInstall::UnpackedInstaller.new(package, options) puts "#{INDENT}#{spec.name} #{spec.version}" ins.install - unless $dryrun - File.chmod($data_mode, File.join(install_dir, "specifications", "#{spec.full_name}.gemspec")) - end - unless spec.extensions.empty? - install_recursive(ext, spec.extension_dir) + install_recursive("#{build_dir}/#{gem_name}", "#{extensions_dir}/#{gem_name}") do |src, dest| + # puts "#{INDENT} #{dest[extensions_dir.size+gem_name.size+2..-1]}" + install src, dest, :mode => (File.executable?(src) ? $prog_mode : $data_mode) end installed_gems[spec.full_name] = true end @@ -1048,29 +1280,22 @@ install?(:ext, :comm, :gem, :'bundled-gems') do prepare "bundled gem cache", gem_dir+"/cache" install installed_gems, gem_dir+"/cache" end - next if gems.empty? - if defined?(Zlib) - Gem.instance_variable_set(:@ruby, with_destdir(File.join(bindir, ruby_install_name))) - silent = Gem::SilentUI.new + unless gems.empty? + skipped.default = "not found in bundled_gems" + puts "skipped bundled gems:" gems.each do |gem| - package = Gem::Package.new(gem) - inst = RbInstall::GemInstaller.new(package, options) - inst.spec.extension_dir = with_destdir(inst.spec.extension_dir) - begin - Gem::DefaultUserInteraction.use_ui(silent) {inst.install} - rescue Gem::InstallError - next - end - gemname = File.basename(gem) - puts "#{INDENT}#{gemname}" + gem = File.basename(gem) + printf " %-31s %s\n", gem, skipped[gem.chomp(".gem")] end - # fix directory permissions - # TODO: Gem.install should accept :dir_mode option or something - File.chmod($dir_mode, *Dir.glob(install_dir+"/**/")) - # fix .gemspec permissions - File.chmod($data_mode, *Dir.glob(install_dir+"/specifications/*.gemspec")) - else - puts "skip installing bundled gems because of lacking zlib" + end +end + +install?('modular-gc') do + if modular_gc_dir = CONFIG['modular_gc_dir'] and !modular_gc_dir.empty? + dlext = CONFIG['DLEXT', true] + modular_gc_dir = File.expand_path(modular_gc_dir, CONFIG['prefix']) + prepare "modular GC library", modular_gc_dir + install Dir.glob("gc/*/librubygc.*.#{dlext}"), modular_gc_dir end end @@ -1098,5 +1323,9 @@ installs.each do |block| Dir.chdir(dir) end end +unless installs.empty? or $destdir.empty? + require_relative 'lib/colorize' + puts "Installed under #{Colorize.new.info($destdir)}" +end # vi:set sw=2: diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests new file mode 100644 index 0000000000..4bcb5707a5 --- /dev/null +++ b/tool/rbs_skip_tests @@ -0,0 +1,57 @@ +# Running tests of RBS gem may fail because of various reasons. +# You can skip tests of RBS gem using this file, instead of pushing a new commit to `ruby/rbs` repository. +# +# The most frequently seen reason is the incompatibilities introduced to the unreleased version, including +# +# * Strict argument type check is introduced +# * A required method parameter is added +# * A method/class is removed +# +# Feel free to skip the tests with this file for that case. +# +# Syntax: +# +# $(test-case-name) ` ` $(optional comment) # Skipping single test case +# $(test-class-name) ` ` $(optional comment) # Skipping a test class +# + +## Failed tests because of testing environment + +test_collection_install(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__stdlib_source(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__dependency_no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__rbs_dependency_and__gem_dependency(RBS::CliTest) running tests without Bundler +test_collection_install_frozen(RBS::CliTest) running tests without Bundler +test_collection_install_gemspec(RBS::CliTest) running tests without Bundler +test_collection_update(RBS::CliTest) running tests without Bundler + +NetSingletonTest depending on external resources +NetInstanceTest depending on external resources +TestHTTPRequest depending on external resources +TestSingletonNetHTTPResponse depending on external resources +TestInstanceNetHTTPResponse depending on external resources + +test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` + +# Failing because ObjectSpace.count_nodes has been removed +test_count_nodes(ObjectSpaceTest) + +## Unknown failures + +# NoMethodError: undefined method 'inspect' for an instance of RBS::UnitTest::Convertibles::ToInt +test_compile(RegexpSingletonTest) +test_linear_time?(RegexpSingletonTest) +test_new(RegexpSingletonTest) + +## Failed tests caused by unreleased version of Ruby +test_source_location(MethodInstanceTest) +test_source_location(ProcInstanceTest) +test_source_location(UnboundMethodInstanceTest) + +# Errno::ENOENT: No such file or directory - bundle +test_collection_install__pathname_set(RBS::CliTest) +test_collection_install__set_pathname__manifest(RBS::CliTest) diff --git a/tool/rbs_skip_tests_windows b/tool/rbs_skip_tests_windows new file mode 100644 index 0000000000..db12c69419 --- /dev/null +++ b/tool/rbs_skip_tests_windows @@ -0,0 +1,111 @@ +ARGFTest Failing on Windows + +RactorSingletonTest Hangs up on Windows +RactorInstanceTest Hangs up on Windows + +# NotImplementedError: fileno() function is unimplemented on this machine +test_fileno(DirInstanceTest) +test_fchdir(DirSingletonTest) +test_for_fd(DirSingletonTest) + +# ArgumentError: user root doesn't exist +test_home(DirSingletonTest) + +# NameError: uninitialized constant Etc::CS_PATH +test_confstr(EtcSingletonTest) + +# NameError: uninitialized constant Etc::SC_ARG_MAX +test_sysconf(EtcSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-udw6rx/chmod +test_chmod(FileInstanceTest) +test_chmod(FileInstanceTest) +test_truncate(FileInstanceTest) + +# Errno::EISDIR: Is a directory @ rb_sysopen - C:/a/ruby/ruby/src/gems/src/rbs/test/stdlib +test_directory?(FileSingletonTest) + +# NotImplementedError: lutime() function is unimplemented on this machine +test_lutime(FileSingletonTest) + +# NotImplementedError: mkfifo() function is unimplemented on this machine +test_mkfifo(FileSingletonTest) + +# Returns `nil` on Windows +test_getgrgid(EtcSingletonTest) +test_getgrnam(EtcSingletonTest) +test_getpwnam(EtcSingletonTest) +test_getpwuid(EtcSingletonTest) + +# Returns `false` +test_setgid?(FileSingletonTest) +test_setuid?(FileSingletonTest) +test_sticky?(FileSingletonTest) + +test_world_readable?(FileSingletonTest) # Returns `420` +test_world_readable?(FileStatInstanceTest) # Returns `420` +test_world_writable?(FileSingletonTest) # Returns `nil` +test_dev_major(FileStatInstanceTest) # Returns `nil` +test_dev_minor(FileStatInstanceTest) # Returns `nil` +test_rdev_major(FileStatInstanceTest) # Returns `nil` +test_rdev_minor(FileStatInstanceTest) # Returns `nil` + +# ArgumentError: wrong number of arguments (given -403772944, expected 0+) +test_curry(MethodInstanceTest) + +# ArgumentError: no output encoding given +test_tolocale(KconvSingletonTest) + +# Errno::EINVAL: Invalid argument - : +test_system(KernelInstanceTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_load(OpenSSLConfigSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - +test_parse(OpenSSLConfigSingletonTest) +test_parse_config(OpenSSLConfigSingletonTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_each(OpenSSLConfigTest) +test_lookup_and_set(OpenSSLConfigTest) +test_sections(OpenSSLConfigTest) + +# OpenSSL::SSL::SSLError: SSL_connect returned=1 errno=0 peeraddr=185.199.108.153:443 state=error: certificate verify failed (unable to get local issuer certificate) +test_URI_open(OpenURISingletonTest) + +# ArgumentError: both textmode and binmode specified +test_binwrite(PathnameInstanceTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-delete-test20250813-10156-mb3e9i +test_delete(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-binwrite-test20250813-10156-sh8145 +test_open(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_truncate(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_unlink(PathnameInstanceTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /etc/resolv.conf +test_parse_resolv_conf(ResolvDNSConfigSingletonTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvInstanceTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvSingletonTest) + +# ArgumentError: unsupported signal 'SIGUSR2' +test_trap(SignalSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-mgr4tx +test_create(TempfileSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-hp9nzu +test_initialize(TempfileSingletonTest) +test_new(TempfileSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-f8z9pn/test.gz +test_open(ZlibGzipReaderSingletonTest) + +# Errno::EACCES: Permission denied @ rb_file_s_rename +# D:/a/ruby/ruby/src/lib/rubygems/util/atomic_file_writer.rb:42:in 'File.rename' +test_write_binary(GemSingletonTest) diff --git a/tool/rbuninstall.rb b/tool/rbuninstall.rb index f0c286012c..60f5241a4f 100755 --- a/tool/rbuninstall.rb +++ b/tool/rbuninstall.rb @@ -21,15 +21,33 @@ BEGIN { end $dirs = [] $files = [] + COLUMNS = $tty && (ENV["COLUMNS"]&.to_i || begin require 'io/console/size'; rescue; else IO.console_size&.at(1); end)&.then do |n| + n-1 if n > 1 + end + if COLUMNS + $column = 0 + def message(str = nil) + $stdout.print "\b \b" * $column + if str + if str.size > COLUMNS + str = "..." + str[(-COLUMNS+3)..-1] + end + $stdout.print str + end + $stdout.flush + $column = str&.size || 0 + end + else + alias message puts + end } list = ($_.chomp!('/') ? $dirs : $files) list << $_ END { status = true $\ = nil - ors = (!$dryrun and $tty) ? "\e[K\r" : "\n" $files.each do |file| - print "rm #{file}#{ors}" + message "rm #{file}" unless $dryrun file = File.join($destdir, file) if $destdir begin @@ -45,9 +63,10 @@ END { $dirs.each do |dir| unlink[dir] = true end + nonempty = {} while dir = $dirs.pop dir = File.dirname(dir) while File.basename(dir) == '.' - print "rmdir #{dir}#{ors}" + message "rmdir #{dir}" unless $dryrun realdir = $destdir ? File.join($destdir, dir) : dir begin @@ -58,16 +77,23 @@ END { raise unless File.symlink?(realdir) File.unlink(realdir) end - rescue Errno::ENOENT, Errno::ENOTEMPTY + rescue Errno::ENOTEMPTY + nonempty[dir] = true + rescue Errno::ENOENT rescue status = false puts $! else + nonempty.delete(dir) parent = File.dirname(dir) $dirs.push(parent) unless parent == dir or unlink[parent] end end end - print ors.chomp + message + unless nonempty.empty? + puts "Non empty director#{nonempty.size == 1 ? 'y' : 'ies'}:" + nonempty.each_key {|dir| print " #{dir}\n"} + end exit(status) } diff --git a/tool/rdoc-srcdir b/tool/rdoc-srcdir new file mode 100755 index 0000000000..ecc49b4b2c --- /dev/null +++ b/tool/rdoc-srcdir @@ -0,0 +1,30 @@ +#!ruby -W0 + +%w[tsort rdoc].each do |lib| + path = Dir.glob("#{File.dirname(__dir__)}/.bundle/gems/#{lib}-*").first + $LOAD_PATH.unshift("#{path}/lib") +end +require 'rdoc/rdoc' + +# Make only the output directory relative to the invoked directory. +invoked = Dir.pwd + +# Load options and parse files from srcdir. +Dir.chdir(File.dirname(__dir__)) + +options = RDoc::Options.load_options +options.title = options.title.sub(/Ruby \K.*version/) { + File.read("include/ruby/version.h") + .scan(/^ *# *define +RUBY_API_VERSION_(MAJOR|MINOR) +(\d+)/) + .sort # "MAJOR" < "MINOR", fortunately + .to_h.values.join(".") +} +options.parse ARGV + ["#{invoked}/rbconfig.rb"] + +options.singleton_class.define_method(:finish) do + super() + @op_dir = File.expand_path(@op_dir, invoked) +end + +# Do not hide errors when generating documents of Ruby itself. +RDoc::RDoc.new.document options diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index df54c7eb26..95a9688cb2 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -10,13 +10,7 @@ require 'optparse' require 'abbrev' require 'pp' require 'shellwords' -begin - require 'readline' -rescue LoadError - module Readline; end -end - -VERSION = '0.0.1' +require 'reline' opts = OptionParser.new target_version = nil @@ -24,10 +18,9 @@ repo_path = nil api_key = nil ssl_verify = true opts.on('-k REDMINE_API_KEY', '--key=REDMINE_API_KEY', 'specify your REDMINE_API_KEY') {|v| api_key = v} -opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 2.1)') {|v| target_version = v} +opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 3.1)') {|v| target_version = v} opts.on('-r RUBY_REPO_PATH', '--repository=RUBY_REPO_PATH', 'specify repository path') {|v| repo_path = v} opts.on('--[no-]ssl-verify', TrueClass, 'use / not use SSL verify') {|v| ssl_verify = v} -opts.version = VERSION opts.parse!(ARGV) http_options = {use_ssl: true} @@ -35,17 +28,17 @@ http_options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify $openuri_options = {} $openuri_options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify -TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (raise 'need to specify TARGET_VERSION') +TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (puts opts.help; raise 'need to specify TARGET_VERSION') RUBY_REPO_PATH = repo_path || ENV['RUBY_REPO_PATH'] BACKPORT_CF_KEY = 'cf_5' STATUS_CLOSE = 5 -REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (raise 'need to specify REDMINE_API_KEY') +REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (puts opts.help; raise 'need to specify REDMINE_API_KEY') REDMINE_BASE = 'https://bugs.ruby-lang.org' @query = { 'f[]' => BACKPORT_CF_KEY, "op[#{BACKPORT_CF_KEY}]" => '~', - "v[#{BACKPORT_CF_KEY}][]" => "#{TARGET_VERSION}: REQUIRED", + "v[#{BACKPORT_CF_KEY}][]" => "\"#{TARGET_VERSION}: REQUIRED\"", 'limit' => 40, 'status_id' => STATUS_CLOSE, 'sort' => 'updated_on' @@ -70,28 +63,28 @@ COLORS = { } class String - def color(fore=nil, back=nil, bold: false, underscore: false) + def color(fore=nil, back=nil, opts={}, bold: false, underscore: false) seq = "" - if bold - seq << "\e[1m" + if bold || opts[:bold] + seq = seq + "\e[1m" end - if underscore - seq << "\e[2m" + if underscore || opts[:underscore] + seq = seq + "\e[2m" end if fore c = COLORS[fore] raise "unknown foreground color #{fore}" unless c - seq << "\e[#{c}m" + seq = seq + "\e[#{c}m" end if back c = COLORS[back] raise "unknown background color #{back}" unless c - seq << "\e[#{c + 10}m" + seq = seq + "\e[#{c + 10}m" end if seq.empty? self else - seq << self << "\e[0m" + seq = seq + self + "\e[0m" end end end @@ -162,84 +155,6 @@ def more(sio) end end -class << Readline - def readline(prompt = '') - console = IO.console - console.binmode - _, lx = console.winsize - if /mswin|mingw/ =~ RUBY_PLATFORM or /^(?:vt\d\d\d|xterm)/i =~ ENV["TERM"] - cls = "\r\e[2K" - else - cls = "\r" << (" " * lx) - end - cls << "\r" << prompt - console.print prompt - console.flush - line = '' - while true - case c = console.getch - when "\r", "\n" - puts - HISTORY << line - return line - when "\C-?", "\b" # DEL/BS - print "\b \b" if line.chop! - when "\C-u" - print cls - line.clear - when "\C-d" - return nil if line.empty? - line << c - when "\C-p" - HISTORY.pos -= 1 - line = HISTORY.current - print cls - print line - when "\C-n" - HISTORY.pos += 1 - line = HISTORY.current - print cls - print line - else - if c >= " " - print c - line << c - end - end - end - end - - HISTORY = [] - def HISTORY.<<(val) - HISTORY.push(val) - @pos = self.size - self - end - def HISTORY.pos - @pos ||= 0 - end - def HISTORY.pos=(val) - @pos = val - if @pos < 0 - @pos = -1 - elsif @pos >= self.size - @pos = self.size - end - end - def HISTORY.current - @pos ||= 0 - if @pos < 0 || @pos >= self.size - '' - else - self[@pos] - end - end -end unless defined?(Readline.readline) - -def find_svn_log(pattern) - `svn log --xml --stop-on-copy --search="#{pattern}" #{RUBY_REPO_PATH}` -end - def find_git_log(pattern) `git #{RUBY_REPO_PATH ? "-C #{RUBY_REPO_PATH.shellescape}" : ""} log --grep="#{pattern}"` end @@ -275,11 +190,13 @@ def backport_command_string next false if c.match(/\A\d{1,6}\z/) # skip SVN revision # check if the Git revision is included in master - has_commit(c, "master") + has_commit(c, "origin/master") + end.sort_by do |changeset| + Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) end @changesets.define_singleton_method(:validated){true} end - " #{merger_path} --ticket=#{@issue} #{@changesets.sort.join(',')}" + "#{merger_path} --ticket=#{@issue} #{@changesets.join(',')}" end def status_char(obj) @@ -294,7 +211,7 @@ end console = IO.console row, = console.winsize @query['limit'] = row - 2 -puts "Backporter #{VERSION}".color(bold: true) + " for #{TARGET_VERSION}" +puts "Redmine Backporter".color(bold: true) + " for Ruby #{TARGET_VERSION}" class CommandSyntaxError < RuntimeError; end commands = { @@ -306,10 +223,11 @@ commands = { @issues = issues = res["issues"] from = res["offset"] + 1 total = res["total_count"] + closed = issues.count { |x, _| x["status"]["name"] == "Closed" } to = from + issues.size - 1 - puts "#{from}-#{to} / #{total}" + puts "#{from}-#{to} / #{total} (closed: #{closed})" issues.each_with_index do |x, i| - id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]]) + id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]], bold: x["status"]["name"] == "Closed") puts "#{'%2d' % i} #{id} #{x["priority"]["name"][0]} #{status_char(x["status"])} #{x["subject"][0,80]}" end }, @@ -376,9 +294,6 @@ eom "rel" => proc{|args| # this feature requires custom redmine which allows add_related_issue API case args - when /\Ar?(\d+)\z/ # SVN - rev = $1 - uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/trunk/revisions/#{rev}/issues.json") when /\A\h{7,40}\z/ # Git rev = args uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/git/revisions/#{rev}/issues.json") @@ -422,7 +337,7 @@ eom }, "done" => proc{|args| - raise CommandSyntaxError unless /\A(\d+)?(?: by (\h+))?(?:\s*-- +(.*))?\z/ =~ args + raise CommandSyntaxError unless /\A(\d+)?(?: *by (\h+))?(?:\s*-- +(.*))?\z/ =~ args notes = $3 notes.strip! if notes rev = $2 @@ -436,33 +351,22 @@ eom next end - if rev - elsif system("svn info #{RUBY_REPO_PATH&.shellescape}", %i(out err) => IO::NULL) # SVN - if (log = find_svn_log("##@issue]")) && (/revision="(?<rev>\d+)/ =~ log) - rev = "r#{rev}" - end - else # Git - if log = find_git_log("##@issue]") - /^commit (?<rev>\h{40})$/ =~ log - end - end - if log && rev - str = log[/merge revision\(s\) ([^:]+)(?=:)/] - if str - str.insert(5, "d") - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev} #{str}." + if rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") + notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." + elsif rev.nil? && (log = find_git_log("##@issue]")) && !(revs = log.scan(/^commit (\h{40})$/).flatten).empty? + commits = revs.map { |rev| "commit:#{rev}" }.join(", ") + if merged_revs = log[/merge revision\(s\) ([^:]+)(?=:)/] + merged_revs.sub!(/\Amerge/, 'merged') + merged_revs.gsub!(/\h{8,40}/, 'commit:\0') + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits} #{merged_revs}." else - str = "ruby_#{TARGET_VERSION.tr('.','_')} #{rev}." + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits}." end if notes str << "\n" str << notes end notes = str - elsif rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") - # Backport commit's log doesn't have the issue number. - # Instead of that manually it's provided. - notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." else puts "no commit is found whose log include ##@issue" next @@ -571,7 +475,7 @@ list = Abbrev.abbrev(commands.keys) @changesets = nil while true begin - l = Readline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " + l = Reline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " rescue Interrupt break end diff --git a/tool/release.sh b/tool/release.sh index 0988dc2a67..d467d8f24b 100755 --- a/tool/release.sh +++ b/tool/release.sh @@ -1,7 +1,15 @@ #!/bin/bash # Bash version 3.2+ is required for regexp +# Usage: +# tool/release.sh 3.0.0 +# tool/release.sh 3.0.0-rc1 -EXTS='.tar.gz .tar.bz2 .tar.xz .zip' +EXTS='.tar.gz .tar.xz .zip' +if [[ -n $AWS_ACCESS_KEY_ID ]]; then + AWS_CLI_OPTS="" +else + AWS_CLI_OPTS="--profile ruby" +fi ver=$1 if [[ $ver =~ ^([1-9]\.[0-9])\.([0-9]|[1-9][0-9]|0-(preview[1-9]|rc[1-9]))$ ]]; then @@ -15,5 +23,5 @@ short=${BASH_REMATCH[1]} echo $ver echo $short for ext in $EXTS; do - aws --profile ruby s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext + aws $AWS_CLI_OPTS s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext done diff --git a/tool/releng/gen-mail.rb b/tool/releng/gen-mail.rb index b958a64e65..17fa499d69 100755 --- a/tool/releng/gen-mail.rb +++ b/tool/releng/gen-mail.rb @@ -4,22 +4,27 @@ require "yaml" lang = ARGV.shift unless lang - abort "usage: #$1 {en,ja} | pbcopy" + abort "usage: #$1 {en,ja,release.md} | pbcopy" end # Confirm current directory is www.ruby-lang.org's working directory def confirm_w_r_l_o_wd File.foreach('.git/config') do |line| - return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + return true if line.include?('ruby/www.ruby-lang.org.git') end abort "Run this script in www.ruby-lang.org's working directory" end confirm_w_r_l_o_wd -releases = YAML.load_file('_data/releases.yml') +releases = YAML.safe_load_file('_data/releases.yml', permitted_classes: [Date]) -url = "https://hackmd.io/@naruse/ruby-relnote-#{lang}/download" -src = URI(url).read +case lang +when "en", "ja" + url = "https://hackmd.io/@naruse/ruby-relnote-#{lang}/download" + src = URI(url).read +else # the path of the Release note in markdown is given + src = File.read(lang) +end src.gsub!(/[ \t]+$/, "") src.sub!(/(?<!\n)\z/, "\n") src.sub!(/^breaks: false\n/, '') diff --git a/tool/releng/update-www-meta.rb b/tool/releng/update-www-meta.rb index 8a5651dcd0..0dd5b25631 100755 --- a/tool/releng/update-www-meta.rb +++ b/tool/releng/update-www-meta.rb @@ -1,6 +1,7 @@ #!/usr/bin/env ruby require "open-uri" require "yaml" +require_relative "../ruby-version" class Tarball attr_reader :version, :size, :sha1, :sha256, :sha512 @@ -41,22 +42,7 @@ eom unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version raise "unexpected version string '#{version}'" end - x = $1.to_i - y = $2.to_i - z = $3.to_i - # previous tag for git diff --shortstat - # It's only for x.y.0 release - if z != 0 - prev_tag = nil - elsif y != 0 - prev_tag = "v#{x}_#{y-1}_0" - prev_ver = "#{x}.#{y-1}.0" - elsif x == 3 && y == 0 && z == 0 - prev_tag = "v2_7_0" - prev_ver = "2.7.0" - else - raise "unexpected version for prev_ver '#{version}'" - end + teeny = Integer($3) uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" info = YAML.load(URI(uri).read) @@ -74,9 +60,10 @@ eom tarballs << tarball end - if prev_tag + if teeny == 0 # show diff shortstat - tag = "v#{version.gsub(/[.\-]/, '_')}" + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) rubydir = File.expand_path(File.join(__FILE__, '../../../')) puts %`git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` @@ -155,7 +142,7 @@ eom date = Time.now.utc # use utc to use previous day in midnight entry = <<eom - version: #{ver} - tag: v#{ver.tr('-.', '_')} + tag: #{RubyVersion.tag(ver)} date: #{date.strftime("%Y-%m-%d")} post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ stats: diff --git a/tool/ruby-version.rb b/tool/ruby-version.rb new file mode 100755 index 0000000000..3bbec576e1 --- /dev/null +++ b/tool/ruby-version.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby + +module RubyVersion + def self.tag(version) + major_version = Integer(version.split('.', 2)[0]) + if major_version >= 4 + "v#{version}" + else + "v#{version.tr('.-', '_')}" + end + end + + # Return the previous version to be used for release diff links. + # For a ".0" version, it returns the previous ".0" version. + # For a non-".0" version, it returns the previous teeny version. + def self.previous(version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + major = Integer($1) + minor = Integer($2) + teeny = Integer($3) + + if teeny != 0 + "#{major}.#{minor}.#{teeny-1}" + elsif minor != 0 # && teeny == 0 + "#{major}.#{minor-1}.#{teeny}" + else # minor == 0 && teeny == 0 + case major + when 3 + "2.7.0" + when 4 + "3.4.0" + else + raise "it doesn't know what is the previous version of '#{version}'" + end + end + end +end + +if __FILE__ == $0 + case ARGV[0] + when "tag" + print RubyVersion.tag(ARGV[1]) + when "previous" + print RubyVersion.previous(ARGV[1]) + when "previous-tag" + print RubyVersion.tag(RubyVersion.previous(ARGV[1])) + else + "#{$0}: unexpected command #{ARGV[0].inspect}" + end +end diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb index 25c10947ed..f6c0e39600 100644 --- a/tool/ruby_vm/controllers/application_controller.rb +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -16,10 +15,11 @@ require_relative '../models/typemap' require_relative '../loaders/vm_opts_h' class ApplicationController - def generate i, destdir + def generate i, destdir, basedir path = Pathname.new i dst = destdir ? Pathname.new(destdir).join(i) : Pathname.new(i) - dumper = RubyVM::Dumper.new dst + base = basedir ? Pathname.new(basedir) : Pathname.pwd + dumper = RubyVM::Dumper.new dst, base.expand_path return [path, dumper] end end diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb index 34fafd1e34..628cb0428b 100644 --- a/tool/ruby_vm/helpers/c_escape.rb +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -17,7 +16,10 @@ module RubyVM::CEscape # generate comment, with escaps. def commentify str - return "/* #{str.b.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" + unless str = str.dump[/\A"\K.*(?="\z)/] + raise Encoding::CompatibilityError, "must be ASCII-compatible (#{str.encoding})" + end + return "/* #{str.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" end # Mimic gensym of CL. diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb index 98104f4b92..f0758dd44f 100644 --- a/tool/ruby_vm/helpers/dumper.rb +++ b/tool/ruby_vm/helpers/dumper.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -28,16 +27,12 @@ class RubyVM::Dumper path = Pathname.new(__FILE__) path = (path.relative_path_from(Pathname.pwd) rescue path).dirname path += '../views' - path += spec - src = path.read mode: 'rt:utf-8:utf-8' + path += Pathname.pwd.join(spec).expand_path.to_s.sub("#{@base}/", '') + src = path.expand_path.read mode: 'rt:utf-8:utf-8' rescue Errno::ENOENT raise "don't know how to generate #{path}" else - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%-') - else - erb = ERB.new(src, nil, '%-') - end + erb = ERB.new(src, trim_mode: '%-') erb.filename = path.to_path return erb end @@ -85,10 +80,11 @@ class RubyVM::Dumper . join end - def initialize dst + def initialize dst, base @erb = {} @empty = new_binding @file = cstr dst.to_path + @base = base end def render partial, opts = { :locals => {} } diff --git a/tool/ruby_vm/helpers/scanner.rb b/tool/ruby_vm/helpers/scanner.rb index ef6de8120e..8998abb2d3 100644 --- a/tool/ruby_vm/helpers/scanner.rb +++ b/tool/ruby_vm/helpers/scanner.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb index 034905f74e..d45d0ba83c 100644 --- a/tool/ruby_vm/loaders/insns_def.rb +++ b/tool/ruby_vm/loaders/insns_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb index aa6fd79e79..0750f1823a 100644 --- a/tool/ruby_vm/loaders/opt_insn_unif_def.rb +++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb index 29aef8a325..e08509a433 100644 --- a/tool/ruby_vm/loaders/opt_operand_def.rb +++ b/tool/ruby_vm/loaders/opt_operand_def.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb index 3f05c270ee..d626ea0296 100644 --- a/tool/ruby_vm/loaders/vm_opts_h.rb +++ b/tool/ruby_vm/loaders/vm_opts_h.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index de35e7234a..177b701b92 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -21,7 +20,7 @@ class RubyVM::Attribute @key = opts[:name] @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] @type = opts[:type] - @ope_decls = @insn.opes.map do |operand| + @ope_decls = @insn.operands.map do |operand| decl = operand[:decl] if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA' decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci') diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instruction.rb index 6b5f1f6cf8..f87dd74179 100755..100644 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -15,8 +14,8 @@ require_relative 'c_expr' require_relative 'typemap' require_relative 'attribute' -class RubyVM::BareInstructions - attr_reader :template, :name, :opes, :pops, :rets, :decls, :expr +class RubyVM::BareInstruction + attr_reader :template, :name, :operands, :pops, :rets, :decls, :expr def initialize opts = {} @template = opts[:template] @@ -24,7 +23,7 @@ class RubyVM::BareInstructions @loc = opts[:location] @sig = opts[:signature] @expr = RubyVM::CExpr.new opts[:expr] - @opes = typesplit @sig[:ope] + @operands = typesplit @sig[:ope] @pops = typesplit @sig[:pop].reject {|i| i == '...' } @rets = typesplit @sig[:ret].reject {|i| i == '...' } @attrs = opts[:attributes].map {|i| @@ -51,7 +50,7 @@ class RubyVM::BareInstructions def call_attribute name return sprintf 'attr_%s_%s(%s)', name, @name, \ - @opes.map {|i| i[:name] }.compact.join(', ') + @operands.map {|i| i[:name] }.compact.join(', ') end def has_attribute? k @@ -65,7 +64,7 @@ class RubyVM::BareInstructions end def width - return 1 + opes.size + return 1 + operands.size end def declarations @@ -98,7 +97,7 @@ class RubyVM::BareInstructions end def operands_info - opes.map {|o| + operands.map {|o| c, _ = RubyVM::Typemap.fetch o[:type] next c }.join @@ -108,14 +107,6 @@ class RubyVM::BareInstructions /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr end - def always_leaf? - @attrs.fetch('leaf').expr.expr == 'true;' - end - - def leaf_without_check_ints? - @attrs.fetch('leaf').expr.expr == 'leafness_of_check_ints;' - end - def handle_canary stmt # Stack canary is basically a good thing that we want to add, however: # @@ -137,7 +128,7 @@ class RubyVM::BareInstructions end def has_ope? var - return @opes.any? {|i| i[:name] == var[:name] } + return @operands.any? {|i| i[:name] == var[:name] } end def has_pop? var @@ -149,6 +140,10 @@ class RubyVM::BareInstructions @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } end + def zjit_profile? + @attrs.fetch('zjit_profile').expr.expr != 'false;' + end + private def check_attribute_consistency @@ -180,18 +175,19 @@ class RubyVM::BareInstructions # Beware: order matters here because some attribute depends another. generate_attribute 'const char*', 'name', "insn_name(#{bin})" generate_attribute 'enum ruby_vminsn_type', 'bin', bin - generate_attribute 'rb_num_t', 'open', opes.size + generate_attribute 'rb_num_t', 'open', operands.size generate_attribute 'rb_num_t', 'popn', pops.size generate_attribute 'rb_num_t', 'retn', rets.size generate_attribute 'rb_num_t', 'width', width generate_attribute 'rb_snum_t', 'sp_inc', rets.size - pops.size generate_attribute 'bool', 'handles_sp', default_definition_of_handles_sp generate_attribute 'bool', 'leaf', default_definition_of_leaf + generate_attribute 'bool', 'zjit_profile', false end def default_definition_of_handles_sp # Insn with ISEQ should yield it; can handle sp. - return opes.any? {|o| o[:type] == 'ISEQ' } + return operands.any? {|o| o[:type] == 'ISEQ' } end def default_definition_of_leaf @@ -228,13 +224,13 @@ class RubyVM::BareInstructions new h.merge(:template => h) } - def self.fetch name + def self.find(name) @instances.find do |insn| insn.name == name end or raise IndexError, "instruction not found: #{name}" end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb index 073112f545..095ff4f1d9 100644 --- a/tool/ruby_vm/models/c_expr.rb +++ b/tool/ruby_vm/models/c_expr.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -36,6 +35,10 @@ class RubyVM::CExpr end def inspect - sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + if @__LINE__ + sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + else + sprintf "#<%s %s>", @__FILE__, @expr + end end end diff --git a/tool/ruby_vm/models/instructions.rb b/tool/ruby_vm/models/instructions.rb index 1198c7a4a6..7be7064b14 100644 --- a/tool/ruby_vm/models/instructions.rb +++ b/tool/ruby_vm/models/instructions.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -10,13 +9,15 @@ # conditions mentioned in the file COPYING are met. Consult the file for # details. -require_relative 'bare_instructions' -require_relative 'operands_unifications' -require_relative 'instructions_unifications' +require_relative 'bare_instruction' +require_relative 'operands_unification' +require_relative 'instructions_unification' +require_relative 'trace_instruction' +require_relative 'zjit_instruction' -RubyVM::Instructions = RubyVM::BareInstructions.to_a + \ - RubyVM::OperandsUnifications.to_a + \ - RubyVM::InstructionsUnifications.to_a - -require_relative 'trace_instructions' +RubyVM::Instructions = RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all + + RubyVM::TraceInstruction.all + + RubyVM::ZJITInstruction.all RubyVM::Instructions.freeze diff --git a/tool/ruby_vm/models/instructions_unifications.rb b/tool/ruby_vm/models/instructions_unification.rb index 214ba5fcc2..5c798e6d54 100644 --- a/tool/ruby_vm/models/instructions_unifications.rb +++ b/tool/ruby_vm/models/instructions_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,9 +11,9 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_insn_unif_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::InstructionsUnifications +class RubyVM::InstructionsUnification include RubyVM::CEscape attr_reader :name @@ -23,7 +22,7 @@ class RubyVM::InstructionsUnifications @location = opts[:location] @name = namegen opts[:signature] @series = opts[:signature].map do |i| - RubyVM::BareInstructions.fetch i # Misshit is fatal + RubyVM::BareInstruction.find(i) # Misshit is fatal end end @@ -37,7 +36,7 @@ class RubyVM::InstructionsUnifications new h end - def self.to_a + def self.all @instances end end diff --git a/tool/ruby_vm/models/operands_unifications.rb b/tool/ruby_vm/models/operands_unification.rb index ee4e3a695d..ce118648ca 100644 --- a/tool/ruby_vm/models/operands_unifications.rb +++ b/tool/ruby_vm/models/operands_unification.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -12,16 +11,16 @@ require_relative '../helpers/c_escape' require_relative '../loaders/opt_operand_def' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::OperandsUnifications < RubyVM::BareInstructions +class RubyVM::OperandsUnification < RubyVM::BareInstruction include RubyVM::CEscape attr_reader :preamble, :original, :spec def initialize opts = {} name = opts[:signature][0] - @original = RubyVM::BareInstructions.fetch name + @original = RubyVM::BareInstruction.find(name) template = @original.template parts = compose opts[:location], opts[:signature], template[:signature] json = template.dup @@ -38,8 +37,8 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions end def operand_shift_of var - before = @original.opes.find_index var - after = @opes.find_index var + before = @original.operands.find_index var + after = @operands.find_index var raise "no #{var} for #{@name}" unless before and after return before - after end @@ -50,7 +49,7 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions case val when '*' then next nil else - type = @original.opes[i][:type] + type = @original.operands[i][:type] expr = RubyVM::Typemap.typecast_to_VALUE type, val next "#{ptr}[#{i}] == #{expr}" end @@ -85,7 +84,7 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions def compose location, spec, template name = namegen spec *, argv = *spec - opes = @original.opes + opes = @original.operands if opes.size != argv.size raise sprintf("operand size mismatch for %s (%s's: %d, given: %d)", name, template[:name], opes.size, argv.size) @@ -130,12 +129,12 @@ class RubyVM::OperandsUnifications < RubyVM::BareInstructions new h end - def self.to_a + def self.all @instances end def self.each_group - to_a.group_by(&:original).each_pair do |k, v| + all.group_by(&:original).each_pair do |k, v| yield k, v end end diff --git a/tool/ruby_vm/models/trace_instructions.rb b/tool/ruby_vm/models/trace_instruction.rb index 4ed4c8cb42..6a3ad53c44 100644 --- a/tool/ruby_vm/models/trace_instructions.rb +++ b/tool/ruby_vm/models/trace_instruction.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -11,9 +10,9 @@ # details. require_relative '../helpers/c_escape' -require_relative 'bare_instructions' +require_relative 'bare_instruction' -class RubyVM::TraceInstructions +class RubyVM::TraceInstruction include RubyVM::CEscape attr_reader :name @@ -61,11 +60,11 @@ class RubyVM::TraceInstructions private - @instances = RubyVM::Instructions.map {|i| new i } + @instances = (RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all).map {|i| new(i) } - def self.to_a + def self.all @instances end - - RubyVM::Instructions.push(*to_a) end diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb index ed3aea7d2e..68ef5a41a5 100644 --- a/tool/ruby_vm/models/typemap.rb +++ b/tool/ruby_vm/models/typemap.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -16,6 +15,7 @@ RubyVM::Typemap = { "CDHASH" => %w[H TS_CDHASH], "IC" => %w[K TS_IC], "IVC" => %w[A TS_IVC], + "ICVARC" => %w[J TS_ICVARC], "ID" => %w[I TS_ID], "ISE" => %w[T TS_ISE], "ISEQ" => %w[S TS_ISEQ], diff --git a/tool/ruby_vm/models/zjit_instruction.rb b/tool/ruby_vm/models/zjit_instruction.rb new file mode 100644 index 0000000000..04764e4c61 --- /dev/null +++ b/tool/ruby_vm/models/zjit_instruction.rb @@ -0,0 +1,56 @@ +require_relative '../helpers/c_escape' +require_relative 'bare_instruction' + +# Profile YARV instructions to optimize code generated by ZJIT +class RubyVM::ZJITInstruction + include RubyVM::CEscape + + attr_reader :name + + def initialize(orig) + @orig = orig + @name = as_tr_cpp "zjit @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute?(*) + return false + end + + @instances = RubyVM::BareInstruction.all.filter(&:zjit_profile?).map {|i| new(i) } + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb index 8325dd364f..ad8603b1a8 100644 --- a/tool/ruby_vm/scripts/insns2vm.rb +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -1,4 +1,3 @@ -#! /your/favourite/path/to/ruby # -*- Ruby -*- # -*- frozen_string_literal: true; -*- # -*- warn_indent: true; -*- @@ -15,10 +14,10 @@ require_relative '../controllers/application_controller.rb' module RubyVM::Insns2VM def self.router argv - options = { destdir: nil } + options = { destdir: nil, basedir: nil } targets = generate_parser(options).parse argv return targets.map do |i| - next ApplicationController.new.generate i, options[:destdir] + next ApplicationController.new.generate i, options[:destdir], options[:basedir] end end @@ -84,6 +83,14 @@ module RubyVM::Insns2VM options[:destdir] = dir end + this.on "--basedir=DIR", <<-'begin' do |dir| + Change the base directory from the current working directory + to the given path. Used for searching the source template. + begin + raise "directory was not found in '#{dir}'" unless Dir.exist?(dir) + options[:basedir] = dir + end + this.on "-V", "--[no-]verbose", <<-'end' Please let us ignore this and be modest. end diff --git a/tool/ruby_vm/tests/.gitkeep b/tool/ruby_vm/tests/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 --- a/tool/ruby_vm/tests/.gitkeep +++ /dev/null diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb index b633ab4d32..8bb28db1c1 100644 --- a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -6,6 +6,16 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. %# +% +% stack_increase = proc do |i| +% if i.has_attribute?('sp_inc') +% '-127' +% else +% sprintf("%4d", i.rets.size - i.pops.size) +% end +% end +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); @@ -13,15 +23,14 @@ rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) { static const signed char t[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map { |i| - if i.has_attribute?('sp_inc') - '-127' - else - sprintf("%4d", i.rets.size - i.pops.size) - end - }.join(', ') -%>, +% insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, % end +#endif }; signed char c = t[insn]; @@ -42,7 +51,7 @@ comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *o % end case <%= i.bin %>: return <%= attr_function %>(<%= - i.opes.map.with_index do |v, j| + i.operands.map.with_index do |v, j| if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc') v = v.dup v[:type] = 'CALL_INFO' diff --git a/tool/ruby_vm/views/_insn_entry.erb b/tool/ruby_vm/views/_insn_entry.erb index bdd0fa3c7c..6ec33461c4 100644 --- a/tool/ruby_vm/views/_insn_entry.erb +++ b/tool/ruby_vm/views/_insn_entry.erb @@ -20,11 +20,11 @@ INSN_ENTRY(<%= insn.name %>) <%= render_c_expr konst -%> % end % -% insn.opes.each_with_index do |ope, i| +% insn.operands.each_with_index do |ope, i| <%= ope[:decl] %> = (<%= ope[:type] %>)GET_OPERAND(<%= i + 1 %>); % end # define INSN_ATTR(x) <%= insn.call_attribute(' ## x ## ') %> - bool leaf = INSN_ATTR(leaf); + const bool MAYBE_UNUSED(leaf) = INSN_ATTR(leaf); % insn.pops.reverse_each.with_index.reverse_each do |pop, i| <%= pop[:decl] %> = <%= insn.cast_from_VALUE pop, "TOPN(#{i})"%>; % end @@ -35,13 +35,13 @@ INSN_ENTRY(<%= insn.name %>) % end /* ### Instruction preambles. ### */ - if (! leaf) ADD_PC(INSN_ATTR(width)); + ADD_PC(INSN_ATTR(width)); % if insn.handles_sp? POPN(INSN_ATTR(popn)); % end <%= insn.handle_canary "SETUP_CANARY(leaf)" -%> COLLECT_USAGE_INSN(INSN_ATTR(bin)); -% insn.opes.each_with_index do |ope, i| +% insn.operands.each_with_index do |ope, i| COLLECT_USAGE_OPERAND(INSN_ATTR(bin), <%= i %>, <%= ope[:name] %>); % end % unless body.empty? @@ -68,7 +68,6 @@ INSN_ENTRY(<%= insn.name %>) VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_MOVED)); % end % end - if (leaf) ADD_PC(INSN_ATTR(width)); # undef INSN_ATTR /* ### Leave the instruction. ### */ diff --git a/tool/ruby_vm/views/_insn_leaf_info.erb b/tool/ruby_vm/views/_insn_leaf_info.erb new file mode 100644 index 0000000000..f30366ffda --- /dev/null +++ b/tool/ruby_vm/views/_insn_leaf_info.erb @@ -0,0 +1,18 @@ +MAYBE_UNUSED(static bool insn_leaf(int insn, const VALUE *opes)); +static bool +insn_leaf(int insn, const VALUE *opes) +{ + switch (insn) { +% RubyVM::Instructions.each do |insn| +% next if insn.is_a?(RubyVM::TraceInstruction) || insn.is_a?(RubyVM::ZJITInstruction) + case <%= insn.bin %>: + return attr_leaf_<%= insn.name %>(<%= + insn.operands.map.with_index do |ope, i| + "(#{ope[:type]})opes[#{i}]" + end.join(', ') + %>); +% end + default: + return false; + } +} diff --git a/tool/ruby_vm/views/_insn_len_info.erb b/tool/ruby_vm/views/_insn_len_info.erb index b292f42fb5..b29a405918 100644 --- a/tool/ruby_vm/views/_insn_len_info.erb +++ b/tool/ruby_vm/views/_insn_len_info.erb @@ -5,17 +5,32 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% CONSTFUNC(MAYBE_UNUSED(static int insn_len(VALUE insn))); +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const uint8_t rb_vm_insn_len_info[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const uint8_t rb_vm_insn_len_info[] = { +% insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#endif +}; + +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_len_info); +#endif + int insn_len(VALUE i) { - static const char t[] = { -% RubyVM::Instructions.each_slice 23 do |a| - <%= a.map(&:width).join(', ') -%>, -% end - }; - - ASSERT_VM_INSTRUCTION_SIZE(t); - return t[i]; + return rb_vm_insn_len_info[i]; } diff --git a/tool/ruby_vm/views/_insn_name_info.erb b/tool/ruby_vm/views/_insn_name_info.erb index 767346093e..2862908631 100644 --- a/tool/ruby_vm/views/_insn_name_info.erb +++ b/tool/ruby_vm/views/_insn_name_info.erb @@ -6,33 +6,54 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.name } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% next_offset = 0 +% name_offset = proc do |i| +% offset = sprintf("%4d", next_offset) +% next_offset += i.name.length + 1 # insn.name + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_name(VALUE insn))); +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const int rb_vm_max_insn_name_size; +extern const char rb_vm_insn_name_base[]; +extern const unsigned short rb_vm_insn_name_offset[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + #ifdef RUBY_VM_INSNS_INFO -const int rb_vm_max_insn_name_size = <%= a.map(&:size).max %>; -#endif +%# "trace_" is longer than "zjit_", so USE_ZJIT doesn't impact the max name size. +const int rb_vm_max_insn_name_size = <%= RubyVM::Instructions.map { |i| i.name.size }.max %>; -const char * -insn_name(VALUE i) -{ - static const char x[] = -% a.each do |i| - <%=cstr i%> "\0" +const char rb_vm_insn_name_base[] = +% insns.each do |i| + <%= cstr i.name %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each do |i| + <%= cstr i.name %> "\0" % end - ; +#endif + ; - static const unsigned short y[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%4d", i) }.join(', ') %>, +const unsigned short rb_vm_insn_name_offset[] = { +% insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, % end - }; +#endif +}; - ASSERT_VM_INSTRUCTION_SIZE(y); +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_name_offset); +#endif - return &x[y[i]]; +const char * +insn_name(VALUE i) +{ + return &rb_vm_insn_name_base[rb_vm_insn_name_offset[i]]; } diff --git a/tool/ruby_vm/views/_insn_operand_info.erb b/tool/ruby_vm/views/_insn_operand_info.erb index 69d361521a..410869fcd3 100644 --- a/tool/ruby_vm/views/_insn_operand_info.erb +++ b/tool/ruby_vm/views/_insn_operand_info.erb @@ -6,32 +6,55 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % -% a = RubyVM::Instructions.map {|i| i.operands_info } -% b = (0...a.size) -% c = a.inject([0]) {|r, i| r << (r[-1] + i.length + 1) } -% c.pop +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% operands_info = proc { |i| sprintf("%-6s", cstr(i.operands_info)) } +% +% next_offset = 0 +% op_offset = proc do |i| +% offset = sprintf("%3d", next_offset) +% next_offset += i.operands_info.length + 1 # insn.operands_info + \0 +% offset +% end % CONSTFUNC(MAYBE_UNUSED(static const char *insn_op_types(VALUE insn))); CONSTFUNC(MAYBE_UNUSED(static int insn_op_type(VALUE insn, long pos))); -const char * -insn_op_types(VALUE i) -{ - static const char x[] = -% a.each_slice 5 do |d| - <%= d.map {|i| sprintf("%-6s", cstr(i)) }.join(' "\0" ') %> "\0" +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const char rb_vm_insn_op_base[]; +extern const unsigned short rb_vm_insn_op_offset[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const char rb_vm_insn_op_base[] = +% insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" % end - ; +#endif + ; - static const unsigned short y[] = { -% c.each_slice 12 do |d| - <%= d.map {|i| sprintf("%3d", i) }.join(', ') %>, +const unsigned short rb_vm_insn_op_offset[] = { +% insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, % end - }; +#endif +}; - ASSERT_VM_INSTRUCTION_SIZE(y); +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_op_offset); +#endif - return &x[y[i]]; +const char * +insn_op_types(VALUE i) +{ + return &rb_vm_insn_op_base[rb_vm_insn_op_offset[i]]; } int diff --git a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb b/tool/ruby_vm/views/_insn_sp_pc_dependency.erb deleted file mode 100644 index 95528fbbf4..0000000000 --- a/tool/ruby_vm/views/_insn_sp_pc_dependency.erb +++ /dev/null @@ -1,27 +0,0 @@ -%# -*- C -*- -%# Copyright (c) 2019 Takashi Kokubun. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -%# -PUREFUNC(MAYBE_UNUSED(static bool insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes))); - -static bool -insn_may_depend_on_sp_or_pc(int insn, const VALUE *opes) -{ - switch (insn) { -% RubyVM::Instructions.each do |insn| -% # handles_sp?: If true, it requires to move sp in JIT -% # always_leaf?: If false, it may call an arbitrary method. pc should be moved -% # before the call, and the method may refer to caller's pc (lineno). -% unless !insn.is_a?(RubyVM::TraceInstructions) && !insn.handles_sp? && insn.always_leaf? - case <%= insn.bin %>: -% end -% end - return true; - default: - return false; - } -} diff --git a/tool/ruby_vm/views/_insn_type_chars.erb b/tool/ruby_vm/views/_insn_type_chars.erb index 4e1f63e660..27daec6c6d 100644 --- a/tool/ruby_vm/views/_insn_type_chars.erb +++ b/tool/ruby_vm/views/_insn_type_chars.erb @@ -11,3 +11,22 @@ enum ruby_insn_type_chars { <%= t %> = '<%= c %>', % end }; + +static inline union iseq_inline_storage_entry * +ISEQ_IS_ENTRY_START(const struct rb_iseq_constant_body *body, char op_type) +{ + unsigned int relative_ic_offset = 0; + switch (op_type) { + case TS_IC: + relative_ic_offset += body->ise_size; + case TS_ISE: + relative_ic_offset += body->icvarc_size; + case TS_ICVARC: + relative_ic_offset += body->ivc_size; + case TS_IVC: + break; + default: + rb_bug("Wrong op type"); + } + return &body->is_entries[relative_ic_offset]; +} diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb index 1735db2196..2756fa2dec 100644 --- a/tool/ruby_vm/views/_leaf_helpers.erb +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -10,10 +10,6 @@ #include "iseq.h" -// This is used to tell MJIT that this insn would be leaf if CHECK_INTS didn't exist. -// It should be used only when RUBY_VM_CHECK_INTS is directly written in insns.def. -static bool leafness_of_check_ints = false; - static bool leafness_of_defined(rb_num_t op_type) { @@ -25,7 +21,7 @@ leafness_of_defined(rb_num_t op_type) case DEFINED_YIELD: case DEFINED_REF: case DEFINED_ZSUPER: - return false; + return true; case DEFINED_CONST: case DEFINED_CONST_FROM: /* has rb_autoload_load(); */ diff --git a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb b/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb deleted file mode 100644 index 1acfdb7f0b..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb +++ /dev/null @@ -1,31 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2020 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # compiler: Declare dst and ic -% insn.opes.each_with_index do |ope, i| - <%= ope.fetch(:decl) %> = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end - -% # compiler: Capture IC values, locking getinlinecache - struct iseq_inline_constant_cache_entry *ice = ic->entry; - if (ice != NULL && ice->ic_serial && !status->compile_info->disable_const_cache) { -% # JIT: Inline everything in IC, and cancel the slow path - fprintf(f, " if (vm_inlined_ic_hit_p(0x%"PRIxVALUE", 0x%"PRIxVALUE", (const rb_cref_t *)0x%"PRIxVALUE", %"PRI_SERIALT_PREFIX"u, reg_cfp->ep)) {", ice->flags, ice->value, (VALUE)ice->ic_cref, ice->ic_serial); - fprintf(f, " stack[%d] = 0x%"PRIxVALUE";\n", b->stack_size, ice->value); - fprintf(f, " goto label_%d;\n", pos + insn_len(insn) + (int)dst); - fprintf(f, " }"); - fprintf(f, " else {"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " goto const_cancel;\n"); - fprintf(f, " }"); - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - break; - } diff --git a/tool/ruby_vm/views/_mjit_compile_insn.erb b/tool/ruby_vm/views/_mjit_compile_insn.erb deleted file mode 100644 index f54d1b0e0e..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_insn.erb +++ /dev/null @@ -1,92 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. - fprintf(f, "{\n"); - { -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% -% # JIT: Declare stack_size to be used in some macro of _mjit_compile_insn_body.erb - if (status->local_stack_p) { - fprintf(f, " MAYBE_UNUSED(unsigned int) stack_size = %u;\n", b->stack_size); - } -% -% # JIT: Declare variables for operands, popped values and return values -% insn.declarations.each do |decl| - fprintf(f, " <%= decl %>;\n"); -% end - -% # JIT: Set const expressions for `RubyVM::OperandsUnifications` insn -% insn.preamble.each do |amble| - fprintf(f, "<%= amble.expr.sub(/const \S+\s+/, '') %>\n"); -% end -% -% # JIT: Initialize operands -% insn.opes.each_with_index do |ope, i| - fprintf(f, " <%= ope.fetch(:name) %> = (<%= ope.fetch(:type) %>)0x%"PRIxVALUE";", operands[<%= i %>]); -% case ope.fetch(:type) -% when 'ID' - comment_id(f, (ID)operands[<%= i %>]); -% when 'CALL_DATA' - comment_id(f, vm_ci_mid(((CALL_DATA)operands[<%= i %>])->ci)); -% when 'VALUE' - if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>])); -% end - fprintf(f, "\n"); -% end -% -% # JIT: Initialize popped values -% insn.pops.reverse_each.with_index.reverse_each do |pop, i| - fprintf(f, " <%= pop.fetch(:name) %> = stack[%d];\n", b->stack_size - <%= i + 1 %>); -% end -% -% # JIT: move sp and pc if necessary -<%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> -% -% # JIT: Print insn body in insns.def -<%= render 'mjit_compile_insn_body', locals: { insn: insn } -%> -% -% # JIT: Set return values -% insn.rets.reverse_each.with_index do |ret, i| -% # TOPN(n) = ... - fprintf(f, " stack[%d] = <%= ret.fetch(:name) %>;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %> - <%= i + 1 %>); -% end -% -% # JIT: We should evaluate ISeq modified for TracePoint if it's enabled. Note: This is slow. -% # leaf insn may not cancel JIT. leaf_without_check_ints is covered in RUBY_VM_CHECK_INTS of _mjit_compile_insn_body.erb. -% unless insn.always_leaf? || insn.leaf_without_check_ints? - fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %>); - if (!pc_moved_p) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); - } - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); - fprintf(f, " goto cancel;\n"); - fprintf(f, " }\n"); -% end -% -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - } - fprintf(f, "}\n"); -% -% # compiler: If insn has conditional JUMP, the code should go to the branch not targeted by JUMP next. -% if insn.expr.expr =~ /if\s+\([^{}]+\)\s+\{[^{}]+JUMP\([^)]+\);[^{}]+\}/ - if (ALREADY_COMPILED_P(status, pos + insn_len(insn))) { - fprintf(f, "goto label_%d;\n", pos + insn_len(insn)); - } - else { - compile_insns(f, body, b->stack_size, pos + insn_len(insn), status); - } -% end -% -% # compiler: If insn returns (leave) or does longjmp (throw), the branch should no longer be compiled. TODO: create attr for it? -% if insn.expr.expr =~ /\sTHROW_EXCEPTION\([^)]+\);/ || insn.expr.expr =~ /\bvm_pop_frame\(/ - b->finish_p = TRUE; -% end diff --git a/tool/ruby_vm/views/_mjit_compile_insn_body.erb b/tool/ruby_vm/views/_mjit_compile_insn_body.erb deleted file mode 100644 index 187e043837..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_insn_body.erb +++ /dev/null @@ -1,129 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% to_cstr = lambda do |line| -% normalized = line.gsub(/\t/, ' ' * 8) -% indented = normalized.sub(/\A(?!#)/, ' ') # avoid indenting preprocessor -% rstring2cstr(indented.rstrip).sub(/"\z/, '\\n"') -% end -% -% # -% # Expand simple macro, which doesn't require dynamic C code. -% # -% expand_simple_macros = lambda do |arg_expr| -% arg_expr.dup.tap do |expr| -% # For `leave`. We can't proceed next ISeq in the same JIT function. -% expr.gsub!(/^(?<indent>\s*)RESTORE_REGS\(\);\n/) do -% indent = Regexp.last_match[:indent] -% <<-end.gsub(/^ +/, '') -% #if OPT_CALL_THREADED_CODE -% #{indent}rb_ec_thread_ptr(ec)->retval = val; -% #{indent}return 0; -% #else -% #{indent}return val; -% #endif -% end -% end -% expr.gsub!(/^(?<indent>\s*)NEXT_INSN\(\);\n/) do -% indent = Regexp.last_match[:indent] -% <<-end.gsub(/^ +/, '') -% #{indent}UNREACHABLE_RETURN(Qundef); -% end -% end -% end -% end -% -% # -% # Print a body of insn, but with macro expansion. -% # -% expand_simple_macros.call(insn.expr.expr).each_line do |line| -% # -% # Expand dynamic macro here (only JUMP for now) -% # -% # TODO: support combination of following macros in the same line -% case line -% when /\A\s+RUBY_VM_CHECK_INTS\(ec\);\s+\z/ -% if insn.leaf_without_check_ints? # lazily move PC and optionalize mjit_call_p here - fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ - fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); - fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); - fprintf(f, " goto cancel;\n"); - fprintf(f, " }\n"); - fprintf(f, " }\n"); -% else - fprintf(f, <%= to_cstr.call(line) %>); -% end -% when /\A\s+JUMP\((?<dest>[^)]+)\);\s+\z/ -% dest = Regexp.last_match[:dest] -% -% if insn.name == 'opt_case_dispatch' # special case... TODO: use another macro to avoid checking name - { - struct case_dispatch_var arg; - arg.f = f; - arg.base_pos = pos + insn_len(insn); - arg.last_value = Qundef; - - fprintf(f, " switch (<%= dest %>) {\n"); - st_foreach(RHASH_TBL_RAW(hash), compile_case_dispatch_each, (VALUE)&arg); - fprintf(f, " case %lu:\n", else_offset); - fprintf(f, " goto label_%lu;\n", arg.base_pos + else_offset); - fprintf(f, " }\n"); - } -% else -% # Before we `goto` next insn, we need to set return values, especially for getinlinecache -% insn.rets.reverse_each.with_index do |ret, i| -% # TOPN(n) = ... - fprintf(f, " stack[%d] = <%= ret.fetch(:name) %>;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %> - <%= i + 1 %>); -% end -% - next_pos = pos + insn_len(insn) + (unsigned int)<%= dest %>; - fprintf(f, " goto label_%d;\n", next_pos); -% end -% when /\A\s+CALL_SIMPLE_METHOD\(\);\s+\z/ -% # For `opt_xxx`'s fallbacks. - if (status->local_stack_p) { - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - } - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_opt_insn);\n"); - fprintf(f, " goto cancel;\n"); -% when /\A(?<prefix>.+\b)INSN_LABEL\((?<name>[^)]+)\)(?<suffix>.+)\z/m -% prefix, name, suffix = Regexp.last_match[:prefix], Regexp.last_match[:name], Regexp.last_match[:suffix] - fprintf(f, " <%= prefix.gsub(/\t/, ' ' * 8) %>INSN_LABEL(<%= name %>_%d)<%= suffix.sub(/\n/, '\n') %>", pos); -% else -% if insn.handles_sp? -% # If insn.handles_sp? is true, cfp->sp might be changed inside insns (like vm_caller_setup_arg_block) -% # and thus we need to use cfp->sp, even when local_stack_p is TRUE. When insn.handles_sp? is true, -% # cfp->sp should be available too because _mjit_compile_pc_and_sp.erb sets it. - fprintf(f, <%= to_cstr.call(line) %>); -% else -% # If local_stack_p is TRUE and insn.handles_sp? is false, stack values are only available in local variables -% # for stack. So we need to replace those macros if local_stack_p is TRUE here. -% case line -% when /\bGET_SP\(\)/ -% # reg_cfp->sp - fprintf(f, <%= to_cstr.call(line.sub(/\bGET_SP\(\)/, '%s')) %>, (status->local_stack_p ? "(stack + stack_size)" : "GET_SP()")); -% when /\bSTACK_ADDR_FROM_TOP\((?<num>[^)]+)\)/ -% # #define STACK_ADDR_FROM_TOP(n) (GET_SP()-(n)) -% num = Regexp.last_match[:num] - fprintf(f, <%= to_cstr.call(line.sub(/\bSTACK_ADDR_FROM_TOP\(([^)]+)\)/, '%s')) %>, - (status->local_stack_p ? "(stack + (stack_size - (<%= num %>)))" : "STACK_ADDR_FROM_TOP(<%= num %>)")); -% when /\bTOPN\((?<num>[^)]+)\)/ -% # #define TOPN(n) (*(GET_SP()-(n)-1)) -% num = Regexp.last_match[:num] - fprintf(f, <%= to_cstr.call(line.sub(/\bTOPN\(([^)]+)\)/, '%s')) %>, - (status->local_stack_p ? "*(stack + (stack_size - (<%= num %>) - 1))" : "TOPN(<%= num %>)")); -% else - fprintf(f, <%= to_cstr.call(line) %>); -% end -% end -% end -% end diff --git a/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb b/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb deleted file mode 100644 index a3796ffc5e..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_invokebuiltin.erb +++ /dev/null @@ -1,29 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2020 Urabe, Shyouhei. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% insn.opes.each_with_index do |ope, i| - <%= ope.fetch(:decl) %> = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end - rb_snum_t sp_inc = <%= insn.call_attribute('sp_inc') %>; - unsigned sp = b->stack_size + (unsigned)sp_inc; - VM_ASSERT(b->stack_size > -sp_inc); - VM_ASSERT(sp_inc < UINT_MAX - b->stack_size); - - if (bf->compiler) { - fprintf(f, "{\n"); - fprintf(f, " VALUE val;\n"); - bf->compiler(f, <%= - insn.name == 'invokebuiltin' ? '-1' : '(rb_num_t)operands[1]' - %>, b->stack_size, body->builtin_inline_p); - fprintf(f, " stack[%u] = val;\n", sp - 1); - fprintf(f, "}\n"); -% if insn.name != 'opt_invokebuiltin_delegate_leave' - b->stack_size = sp; - break; -% end - } diff --git a/tool/ruby_vm/views/_mjit_compile_ivar.erb b/tool/ruby_vm/views/_mjit_compile_ivar.erb deleted file mode 100644 index 5105584ba3..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_ivar.erb +++ /dev/null @@ -1,101 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # Optimized case of get_instancevariable instruction. -#if OPT_IC_FOR_IVAR -{ -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% # compiler: Use copied IVC to avoid race condition - IVC ic_copy = &(status->is_entries + ((union iseq_inline_storage_entry *)ic - body->is_entries))->iv_cache; -% - if (!status->compile_info->disable_ivar_cache && ic_copy->entry) { // Only ic_copy is enabled. -% # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. -% # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> -% -% # JIT: prepare vm_getivar/vm_setivar arguments and variables - fprintf(f, "{\n"); - fprintf(f, " VALUE obj = GET_SELF();\n"); - fprintf(f, " const uint32_t index = %u;\n", (ic_copy->entry->index)); - if (status->merge_ivar_guards_p) { -% # JIT: Access ivar without checking these VM_ASSERTed prerequisites as we checked them in the beginning of `mjit_compile_body` - fprintf(f, " VM_ASSERT(RB_TYPE_P(obj, T_OBJECT));\n"); - fprintf(f, " VM_ASSERT((rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(obj)->klass));\n", ic_copy->entry->class_serial); - fprintf(f, " VM_ASSERT(index < ROBJECT_NUMIV(obj));\n"); -% if insn.name == 'setinstancevariable' - fprintf(f, " if (LIKELY(!RB_OBJ_FROZEN_RAW(obj) && %s)) {\n", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)"); - fprintf(f, " RB_OBJ_WRITE(obj, &ROBJECT(obj)->as.%s, stack[%d]);\n", - status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]", b->stack_size - 1); - fprintf(f, " }\n"); -% else - fprintf(f, " VALUE val;\n"); - fprintf(f, " if (LIKELY(%s && (val = ROBJECT(obj)->as.%s) != Qundef)) {\n", - status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)", - status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]"); - fprintf(f, " stack[%d] = val;\n", b->stack_size); - fprintf(f, " }\n"); -%end - } - else { - fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); -% # JIT: cache hit path of vm_getivar/vm_setivar, or cancel JIT (recompile it with exivar) -% if insn.name == 'setinstancevariable' - fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && !RB_OBJ_FROZEN_RAW(obj))) {\n"); - fprintf(f, " VALUE *ptr = ROBJECT_IVPTR(obj);\n"); - fprintf(f, " RB_OBJ_WRITE(obj, &ptr[index], stack[%d]);\n", b->stack_size - 1); - fprintf(f, " }\n"); -% else - fprintf(f, " VALUE val;\n"); - fprintf(f, " if (LIKELY(RB_TYPE_P(obj, T_OBJECT) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n"); - fprintf(f, " stack[%d] = val;\n", b->stack_size); - fprintf(f, " }\n"); -% end - } - fprintf(f, " else {\n"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " goto ivar_cancel;\n"); - fprintf(f, " }\n"); - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - fprintf(f, "}\n"); - break; - } -% if insn.name == 'getinstancevariable' - else if (!status->compile_info->disable_exivar_cache && ic_copy->entry) { -% # JIT: optimize away motion of sp and pc. This path does not call rb_warning() and so it's always leaf and not `handles_sp`. -% # <%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> -% -% # JIT: prepare vm_getivar's arguments and variables - fprintf(f, "{\n"); - fprintf(f, " VALUE obj = GET_SELF();\n"); - fprintf(f, " const rb_serial_t ic_serial = (rb_serial_t)%"PRI_SERIALT_PREFIX"u;\n", ic_copy->entry->class_serial); - fprintf(f, " const uint32_t index = %u;\n", ic_copy->entry->index); -% # JIT: cache hit path of vm_getivar, or cancel JIT (recompile it without any ivar optimization) - fprintf(f, " struct gen_ivtbl *ivtbl;\n"); - fprintf(f, " VALUE val;\n"); - fprintf(f, " if (LIKELY(FL_TEST_RAW(obj, FL_EXIVAR) && ic_serial == RCLASS_SERIAL(RBASIC(obj)->klass) && rb_ivar_generic_ivtbl_lookup(obj, &ivtbl) && index < ivtbl->numiv && (val = ivtbl->ivptr[index]) != Qundef)) {\n"); - fprintf(f, " stack[%d] = val;\n", b->stack_size); - fprintf(f, " }\n"); - fprintf(f, " else {\n"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " goto exivar_cancel;\n"); - fprintf(f, " }\n"); - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - fprintf(f, "}\n"); - break; - } -% end -} -#endif // OPT_IC_FOR_IVAR diff --git a/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb b/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb deleted file mode 100644 index 390b3ce525..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_pc_and_sp.erb +++ /dev/null @@ -1,38 +0,0 @@ -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # JIT: When an insn is leaf, we don't need to Move pc for a catch table on catch_except_p, #caller_locations, -% # and rb_profile_frames. For check_ints, we lazily move PC when we have interruptions. - MAYBE_UNUSED(bool pc_moved_p) = false; - if (<%= !(insn.always_leaf? || insn.leaf_without_check_ints?) %>) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); /* ADD_PC(INSN_ATTR(width)); */ - pc_moved_p = true; - } -% -% # JIT: move sp to use or preserve stack variables - if (status->local_stack_p) { -% # sp motion is optimized away for `handles_sp? #=> false` case. -% # Thus sp should be set properly before `goto cancel`. -% if insn.handles_sp? -% # JIT-only behavior (pushing JIT's local variables to VM's stack): - { - rb_snum_t i, push_size; - push_size = -<%= insn.call_attribute('sp_inc') %> + <%= insn.rets.size %> - <%= insn.pops.size %>; - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %ld;\n", push_size); /* POPN(INSN_ATTR(popn)); */ - for (i = 0; i < push_size; i++) { - fprintf(f, " *(reg_cfp->sp + %ld) = stack[%ld];\n", i - push_size, (rb_snum_t)b->stack_size - push_size + i); - } - } -% end - } - else { -% if insn.handles_sp? - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size - <%= insn.pops.size %>); /* POPN(INSN_ATTR(popn)); */ -% else - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); -% end - } diff --git a/tool/ruby_vm/views/_mjit_compile_send.erb b/tool/ruby_vm/views/_mjit_compile_send.erb deleted file mode 100644 index 28e316a1ef..0000000000 --- a/tool/ruby_vm/views/_mjit_compile_send.erb +++ /dev/null @@ -1,119 +0,0 @@ -% # -*- C -*- -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -% -% # Optimized case of send / opt_send_without_block instructions. -{ -% # compiler: Prepare operands which may be used by `insn.call_attribute` -% insn.opes.each_with_index do |ope, i| - MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; -% end -% # compiler: Use captured cc to avoid race condition - size_t cd_index = call_data_index(cd, body); - const struct rb_callcache **cc_entries = captured_cc_entries(status); - const struct rb_callcache *captured_cc = cc_entries[cd_index]; -% -% # compiler: Inline send insn where some supported fastpath is used. - const rb_iseq_t *iseq = NULL; - const CALL_INFO ci = cd->ci; - int kw_splat = IS_ARGS_KW_SPLAT(ci) > 0; - extern bool rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci); - if (!status->compile_info->disable_send_cache && has_valid_method_type(captured_cc) && ( -% # `CC_SET_FASTPATH(cd->cc, vm_call_cfunc_with_frame, ...)` in `vm_call_cfunc` - (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC - && !rb_splat_or_kwargs_p(ci) && !kw_splat) -% # `CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(...), vm_call_iseq_optimizable_p(...))` in `vm_callee_setup_arg`, -% # and support only non-VM_CALL_TAILCALL path inside it - || (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_ISEQ - && fastpath_applied_iseq_p(ci, captured_cc, iseq = def_iseq_ptr(vm_cc_cme(captured_cc)->def)) - && !(vm_ci_flag(ci) & VM_CALL_TAILCALL)) - )) { - const bool cfunc_debug = false; // Set true when you want to see inlined cfunc - if (cfunc_debug && vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) - fprintf(stderr, " * %s\n", rb_id2name(vm_ci_mid(ci))); - - int sp_inc = (int)sp_inc_of_sendish(ci); - fprintf(f, "{\n"); - -% # JIT: Invalidate call cache if it requires vm_search_method. This allows to inline some of following things. - bool opt_class_of = !maybe_special_const_class_p(captured_cc->klass); // If true, use RBASIC_CLASS instead of CLASS_OF to reduce code size - fprintf(f, " const struct rb_callcache *cc = (const struct rb_callcache *)0x%"PRIxVALUE";\n", (VALUE)captured_cc); - fprintf(f, " const rb_callable_method_entry_t *cc_cme = (const rb_callable_method_entry_t *)0x%"PRIxVALUE";\n", (VALUE)vm_cc_cme(captured_cc)); - fprintf(f, " const VALUE recv = stack[%d];\n", b->stack_size + sp_inc - 1); - fprintf(f, " if (UNLIKELY(%s || !vm_cc_valid_p(cc, cc_cme, %s(recv)))) {\n", opt_class_of ? "RB_SPECIAL_CONST_P(recv)" : "false", opt_class_of ? "RBASIC_CLASS" : "CLASS_OF"); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " goto send_cancel;\n"); - fprintf(f, " }\n"); - -% # JIT: move sp and pc if necessary -<%= render 'mjit_compile_pc_and_sp', locals: { insn: insn } -%> - -% # JIT: If ISeq is inlinable, call the inlined method without pushing a frame. - if (iseq && status->inlined_iseqs != NULL && iseq->body == status->inlined_iseqs[pos]) { - fprintf(f, " {\n"); - fprintf(f, " VALUE orig_self = reg_cfp->self;\n"); - fprintf(f, " reg_cfp->self = stack[%d];\n", b->stack_size + sp_inc - 1); - fprintf(f, " stack[%d] = _mjit%d_inlined_%d(ec, reg_cfp, orig_self, original_iseq);\n", b->stack_size + sp_inc - 1, status->compiled_id, pos); - fprintf(f, " reg_cfp->self = orig_self;\n"); - fprintf(f, " }\n"); - } - else { -% # JIT: Forked `vm_sendish` (except method_explorer = vm_search_method_wrap) to inline various things - fprintf(f, " {\n"); - fprintf(f, " VALUE val;\n"); - fprintf(f, " struct rb_calling_info calling;\n"); -% if insn.name == 'send' - fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (const struct rb_callinfo *)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq); -% else - fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); -% end - fprintf(f, " calling.kw_splat = %d;\n", kw_splat); - fprintf(f, " calling.recv = stack[%d];\n", b->stack_size + sp_inc - 1); - fprintf(f, " calling.argc = %d;\n", vm_ci_argc(ci)); - - if (vm_cc_cme(captured_cc)->def->type == VM_METHOD_TYPE_CFUNC) { -% # TODO: optimize this more - fprintf(f, " calling.ci = (CALL_INFO)0x%"PRIxVALUE";\n", (VALUE)ci); // creating local cd here because operand's cd->cc may not be the same as inlined cc. - fprintf(f, " calling.cc = cc;"); - fprintf(f, " val = vm_call_cfunc_with_frame(ec, reg_cfp, &calling);\n"); - } - else { // VM_METHOD_TYPE_ISEQ -% # fastpath_applied_iseq_p checks rb_simple_iseq_p, which ensures has_opt == FALSE - fprintf(f, " vm_call_iseq_setup_normal(ec, reg_cfp, &calling, cc_cme, 0, %d, %d);\n", iseq->body->param.size, iseq->body->local_table_size); - if (iseq->body->catch_except_p) { - fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); - fprintf(f, " val = vm_exec(ec, true);\n"); - } - else { - fprintf(f, " if ((val = mjit_exec(ec)) == Qundef) {\n"); - fprintf(f, " VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH);\n"); // This is vm_call0_body's code after vm_call_iseq_setup - fprintf(f, " val = vm_exec(ec, false);\n"); - fprintf(f, " }\n"); - } - } - fprintf(f, " stack[%d] = val;\n", b->stack_size + sp_inc - 1); - fprintf(f, " }\n"); - -% # JIT: We should evaluate ISeq modified for TracePoint if it's enabled. Note: This is slow. - fprintf(f, " if (UNLIKELY(!mjit_call_p)) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size + (int)<%= insn.call_attribute('sp_inc') %>); - if (!pc_moved_p) { - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", next_pos); - } - fprintf(f, " RB_DEBUG_COUNTER_INC(mjit_cancel_invalidate_all);\n"); - fprintf(f, " goto cancel;\n"); - fprintf(f, " }\n"); - } - -% # compiler: Move JIT compiler's internal stack pointer - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - - fprintf(f, "}\n"); - break; - } -} diff --git a/tool/ruby_vm/views/_sp_inc_helpers.erb b/tool/ruby_vm/views/_sp_inc_helpers.erb index d0b0bd79ef..740fe10142 100644 --- a/tool/ruby_vm/views/_sp_inc_helpers.erb +++ b/tool/ruby_vm/views/_sp_inc_helpers.erb @@ -18,7 +18,7 @@ sp_inc_of_sendish(const struct rb_callinfo *ci) * 3. Pop receiver. * 4. Push return value. */ - const int argb = (vm_ci_flag(ci) & VM_CALL_ARGS_BLOCKARG) ? 1 : 0; + const int argb = (vm_ci_flag(ci) & (VM_CALL_ARGS_BLOCKARG | VM_CALL_FORWARDING)) ? 1 : 0; const int argc = vm_ci_argc(ci); const int recv = 1; const int retn = 1; diff --git a/tool/ruby_vm/views/_zjit_helpers.erb b/tool/ruby_vm/views/_zjit_helpers.erb new file mode 100644 index 0000000000..1185dbd9d8 --- /dev/null +++ b/tool/ruby_vm/views/_zjit_helpers.erb @@ -0,0 +1,31 @@ +#if USE_ZJIT + +MAYBE_UNUSED(static int vm_bare_insn_to_zjit_insn(int insn)); +static int +vm_bare_insn_to_zjit_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case BIN(<%= insn.jump_destination %>): + return <%= insn.bin %>; +% end + default: + return insn; + } +} + +MAYBE_UNUSED(static int vm_zjit_insn_to_bare_insn(int insn)); +static int +vm_zjit_insn_to_bare_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case <%= insn.bin %>: + return BIN(<%= insn.jump_destination %>); +% end + default: + return insn; + } +} + +#endif diff --git a/tool/ruby_vm/views/_zjit_instruction.erb b/tool/ruby_vm/views/_zjit_instruction.erb new file mode 100644 index 0000000000..7fd657697c --- /dev/null +++ b/tool/ruby_vm/views/_zjit_instruction.erb @@ -0,0 +1,12 @@ +#if USE_ZJIT + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + rb_zjit_profile_insn(BIN(<%= insn.jump_destination %>), ec); + DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} + +#endif diff --git a/tool/ruby_vm/views/insns.inc.erb b/tool/ruby_vm/views/insns.inc.erb index 29981a8a2d..6521a89b8a 100644 --- a/tool/ruby_vm/views/insns.inc.erb +++ b/tool/ruby_vm/views/insns.inc.erb @@ -6,21 +6,36 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'contains YARV instruction list', edit: __FILE__, } -%> +#ifndef INSNS_INC +#define INSNS_INC 1 + /* BIN : Basic Instruction Name */ #define BIN(n) YARVINSN_##n enum ruby_vminsn_type { -% RubyVM::Instructions.each do |i| +% insns.each do |i| + <%= i.bin %>, +% end +#if USE_ZJIT +% zjit_insns.each do |i| <%= i.bin %>, % end +#endif VM_INSTRUCTION_SIZE }; +#define VM_BARE_INSTRUCTION_SIZE <%= RubyVM::Instructions.count { |i| i.name !~ /\A(trace|zjit)_/ } %> + #define ASSERT_VM_INSTRUCTION_SIZE(array) \ STATIC_ASSERT(numberof_##array, numberof(array) == VM_INSTRUCTION_SIZE) + +#endif diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb index 2ca5aca7cf..48dd0e8832 100644 --- a/tool/ruby_vm/views/insns_info.inc.erb +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -11,12 +11,16 @@ this_file: 'contains instruction information for yarv instruction sequence.', edit: __FILE__, } %> +#ifndef INSNS_INFO_INC +#define INSNS_INFO_INC 1 <%= render 'insn_type_chars' %> <%= render 'insn_name_info' %> <%= render 'insn_len_info' %> <%= render 'insn_operand_info' %> <%= render 'leaf_helpers' %> <%= render 'sp_inc_helpers' %> +<%= render 'zjit_helpers' %> <%= render 'attributes' %> +<%= render 'insn_leaf_info' %> <%= render 'comptime_insn_stack_increase' %> -<%= render 'insn_sp_pc_dependency' %> +#endif diff --git a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb new file mode 100644 index 0000000000..793528af5d --- /dev/null +++ b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb @@ -0,0 +1,14 @@ +module RubyVM::RJIT # :nodoc: all + Instruction = Data.define(:name, :bin, :len, :operands) + + INSNS = { +% RubyVM::Instructions.each_with_index do |insn, i| + <%= i %> => Instruction.new( + name: :<%= insn.name %>, + bin: <%= i %>, # BIN(<%= insn.name %>) + len: <%= insn.width %>, # insn_len + operands: <%= (insn.operands unless insn.name.start_with?(/trace_|zjit_/)).inspect %>, + ), +% end + } +end diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb deleted file mode 100644 index 5820f81770..0000000000 --- a/tool/ruby_vm/views/mjit_compile.inc.erb +++ /dev/null @@ -1,110 +0,0 @@ -/* -*- C -*- */ - -% # Copyright (c) 2018 Takashi Kokubun. All rights reserved. -% # -% # This file is a part of the programming language Ruby. Permission is hereby -% # granted, to either redistribute and/or modify this file, provided that the -% # conditions mentioned in the file COPYING are met. Consult the file for -% # details. -<%= render 'copyright' %> -% -% # This is an ERB template that generates Ruby code that generates C code that -% # generates JIT-ed C code. -<%= render 'notice', locals: { - this_file: 'is the main part of compile_insn() in mjit_compile.c', - edit: __FILE__, -} -%> -% -% unsupported_insns = [ -% 'defineclass', # low priority -% ] -% -% opt_send_without_block = RubyVM::Instructions.find { |i| i.name == 'opt_send_without_block' } -% if opt_send_without_block.nil? -% raise 'opt_send_without_block not found' -% end -% -% send_compatible_opt_insns = RubyVM::BareInstructions.to_a.select do |insn| -% insn.name.start_with?('opt_') && opt_send_without_block.opes == insn.opes && -% insn.expr.expr.lines.any? { |l| l.match(/\A\s+CALL_SIMPLE_METHOD\(\);\s+\z/) } -% end.map(&:name) -% -% # Available variables and macros in JIT-ed function: -% # ec: the first argument of _mjitXXX -% # reg_cfp: the second argument of _mjitXXX -% # GET_CFP(): refers to `reg_cfp` -% # GET_EP(): refers to `reg_cfp->ep` -% # GET_SP(): refers to `reg_cfp->sp`, or `(stack + stack_size)` if local_stack_p -% # GET_SELF(): refers to `cfp_self` -% # GET_LEP(): refers to `VM_EP_LEP(reg_cfp->ep)` -% # EXEC_EC_CFP(): refers to `val = vm_exec(ec, true)` with frame setup -% # CALL_METHOD(): using `GET_CFP()` and `EXEC_EC_CFP()` -% # TOPN(): refers to `reg_cfp->sp`, or `*(stack + (stack_size - num - 1))` if local_stack_p -% # STACK_ADDR_FROM_TOP(): refers to `reg_cfp->sp`, or `stack + (stack_size - num)` if local_stack_p -% # DISPATCH_ORIGINAL_INSN(): expanded in _mjit_compile_insn.erb -% # THROW_EXCEPTION(): specially defined for JIT -% # RESTORE_REGS(): specially defined for `leave` - -switch (insn) { -% (RubyVM::BareInstructions.to_a + RubyVM::OperandsUnifications.to_a).each do |insn| -% next if unsupported_insns.include?(insn.name) - case BIN(<%= insn.name %>): { -% # Instruction-specific behavior in JIT -% case insn.name -% when 'opt_send_without_block', 'send' -<%= render 'mjit_compile_send', locals: { insn: insn } -%> -% when *send_compatible_opt_insns -% # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated. -% cd_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_DATA' } - if (has_cache_for_send(captured_cc_entries(status)[call_data_index((CALL_DATA)operands[<%= cd_index %>], body)], BIN(<%= insn.name %>))) { -<%= render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%> -<%= render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%> - break; - } -% when 'getinstancevariable', 'setinstancevariable' -<%= render 'mjit_compile_ivar', locals: { insn: insn } -%> -% when 'invokebuiltin', 'opt_invokebuiltin_delegate' -<%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%> -% when 'opt_getinlinecache' -<%= render 'mjit_compile_getinlinecache', locals: { insn: insn } -%> -% when 'leave', 'opt_invokebuiltin_delegate_leave' -% # opt_invokebuiltin_delegate_leave also implements leave insn. We need to handle it here for inlining. -% if insn.name == 'opt_invokebuiltin_delegate_leave' -<%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%> -% else - if (b->stack_size != 1) { - if (mjit_opts.warnings || mjit_opts.verbose) - fprintf(stderr, "MJIT warning: Unexpected JIT stack_size on leave: %d\n", b->stack_size); - status->success = false; - } -% end -% # Skip vm_pop_frame for inlined call - if (status->inlined_iseqs != NULL) { // the current ISeq is NOT being inlined -% # Cancel on interrupts to make leave insn leaf - fprintf(f, " if (UNLIKELY(RUBY_VM_INTERRUPTED_ANY(ec))) {\n"); - fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); - fprintf(f, " reg_cfp->pc = original_body_iseq + %d;\n", pos); - fprintf(f, " rb_threadptr_execute_interrupts(rb_ec_thread_ptr(ec), 0);\n"); - fprintf(f, " }\n"); - fprintf(f, " ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);\n"); // vm_pop_frame - } - fprintf(f, " return stack[0];\n"); - b->stack_size += <%= insn.call_attribute('sp_inc') %>; - b->finish_p = TRUE; - break; -% end -% -% # Main insn implementation generated by insns.def -<%= render 'mjit_compile_insn', locals: { insn: insn } -%> - break; - } -% end -% -% # We don't support InstructionsUnifications yet because it's not used for now. -% # We don't support TraceInstructions yet. There is no blocker for it but it's just not implemented. - default: - if (mjit_opts.warnings || mjit_opts.verbose) - fprintf(stderr, "MJIT warning: Skipped to compile unsupported instruction: %s\n", insn_name(insn)); - status->success = false; - break; -} diff --git a/tool/ruby_vm/views/opt_sc.inc.erb b/tool/ruby_vm/views/opt_sc.inc.erb deleted file mode 100644 index e58c81989f..0000000000 --- a/tool/ruby_vm/views/opt_sc.inc.erb +++ /dev/null @@ -1,40 +0,0 @@ -/* -*- C -*- */ - -%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. -%# -%# This file is a part of the programming language Ruby. Permission is hereby -%# granted, to either redistribute and/or modify this file, provided that the -%# conditions mentioned in the file COPYING are met. Consult the file for -%# details. -% raise ':FIXME:TBW' if RubyVM::VmOptsH['STACK_CACHING'] -<%= render 'copyright' %> -<%= render 'notice', locals: { - this_file: 'is for threaded code', - edit: __FILE__, -} -%> - -#define SC_STATE_SIZE 6 - -#define SCS_XX 1 -#define SCS_AX 2 -#define SCS_BX 3 -#define SCS_AB 4 -#define SCS_BA 5 - -#define SC_ERROR 0xffffffff - -static const VALUE sc_insn_info[][SC_STATE_SIZE] = { -#define NO_SC { SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR, SC_ERROR } -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map{|i| 'NO_SC' }.join(', ') %>, -% end -#undef NO_SC -}; - -static const VALUE sc_insn_next[] = { -% RubyVM::Instructions.each_slice 8 do |a| - <%= a.map{|i| 'SCS_XX' }.join(', ') %>, -% end -}; - -ASSERT_VM_INSTRUCTION_SIZE(sc_insn_next); diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb index 676f1edaba..9d9cf0a43a 100644 --- a/tool/ruby_vm/views/optinsn.inc.erb +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -23,20 +23,20 @@ insn_operands_unification(INSN *iobj) /* do nothing */; break; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| +% RubyVM::OperandsUnification.each_group do |orig, unifs| case <%= orig.bin %>: % unifs.each do |insn| /* <%= insn.pretty_name %> */ if ( <%= insn.condition('op') %> ) { -% insn.opes.each_with_index do |o, x| +% insn.operands.each_with_index do |o, x| % n = insn.operand_shift_of(o) % if n != 0 then op[<%= x %>] = op[<%= x + n %>]; % end % end iobj->insn_id = <%= insn.bin %>; - iobj->operand_size = <%= insn.opes.size %>; + iobj->operand_size = <%= insn.operands.size %>; break; } % end @@ -55,12 +55,12 @@ rb_insn_unified_local_var_level(VALUE insn) /* optimize rule */ switch (insn) { default: - return -1; /* do nothing */; -% RubyVM::OperandsUnifications.each_group do |orig, unifs| + return -1; /* do nothing */; +% RubyVM::OperandsUnification.each_group do |orig, unifs| % unifs.each do|insn| case <%= insn.bin %>: % insn.spec.map{|(var,val)|val}.reject{|i| i == '*' }.each do |val| - return <%= val %>; + return <%= val %>; % break % end % end diff --git a/tool/ruby_vm/views/optunifs.inc.erb b/tool/ruby_vm/views/optunifs.inc.erb index e92a95beff..c096712936 100644 --- a/tool/ruby_vm/views/optunifs.inc.erb +++ b/tool/ruby_vm/views/optunifs.inc.erb @@ -7,7 +7,6 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. % raise ':FIXME:TBW' if RubyVM::VmOptsH['INSTRUCTIONS_UNIFICATION'] -% n = RubyVM::Instructions.size <%= render 'copyright' %> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -16,6 +15,4 @@ /* Let .bss section automatically initialize this variable */ /* cf. Section 6.7.8 of ISO/IEC 9899:1999 */ -static const int *const *const unified_insns_data[<%= n %>]; - -ASSERT_VM_INSTRUCTION_SIZE(unified_insns_data); +static const int *const *const unified_insns_data[VM_INSTRUCTION_SIZE]; diff --git a/tool/ruby_vm/views/vm.inc.erb b/tool/ruby_vm/views/vm.inc.erb index c1a3faf60a..38bf5f05ae 100644 --- a/tool/ruby_vm/views/vm.inc.erb +++ b/tool/ruby_vm/views/vm.inc.erb @@ -13,18 +13,22 @@ } -%> #include "vm_insnhelper.h" -% RubyVM::BareInstructions.to_a.each do |insn| +% RubyVM::BareInstruction.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::OperandsUnifications.to_a.each do |insn| +% RubyVM::OperandsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::InstructionsUnifications.to_a.each do |insn| +% RubyVM::InstructionsUnification.all.each do |insn| <%= render 'insn_entry', locals: { insn: insn } -%> % end % -% RubyVM::TraceInstructions.to_a.each do |insn| +% RubyVM::ZJITInstruction.all.each do |insn| +<%= render 'zjit_instruction', locals: { insn: insn } -%> +% end +% +% RubyVM::TraceInstruction.all.each do |insn| <%= render 'trace_instruction', locals: { insn: insn } -%> % end diff --git a/tool/ruby_vm/views/vmtc.inc.erb b/tool/ruby_vm/views/vmtc.inc.erb index 99cbd92614..39dc8bfa6b 100644 --- a/tool/ruby_vm/views/vmtc.inc.erb +++ b/tool/ruby_vm/views/vmtc.inc.erb @@ -6,6 +6,9 @@ %# granted, to either redistribute and/or modify this file, provided that the %# conditions mentioned in the file COPYING are met. Consult the file for %# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% <%= render 'copyright' -%> <%= render 'notice', locals: { this_file: 'is for threaded code', @@ -13,9 +16,14 @@ } -%> static const void *const insns_address_table[] = { -% RubyVM::Instructions.each do |i| +% insns.each do |i| LABEL_PTR(<%= i.name %>), % end +#if USE_ZJIT +% zjit_insns.each do |i| + LABEL_PTR(<%= i.name %>), +% end +#endif }; ASSERT_VM_INSTRUCTION_SIZE(insns_address_table); diff --git a/tool/run-gcov.rb b/tool/run-gcov.rb index 5df7622aa3..46626e4703 100644 --- a/tool/run-gcov.rb +++ b/tool/run-gcov.rb @@ -47,7 +47,8 @@ Pathname.glob("**/*.gcda").sort.each do |gcda| )? Creating\ .*\n \n - )+\z + )+ + (Lines\ executed:.*\n)?\z )x raise "Unexpected gcov output" end diff --git a/tool/run-lcov.rb b/tool/run-lcov.rb index f27578200a..bdccc29a11 100644 --- a/tool/run-lcov.rb +++ b/tool/run-lcov.rb @@ -20,7 +20,7 @@ def backup_gcda_files(gcda_files) end def run_lcov(*args) - system("lcov", "--rc", "lcov_branch_coverage=1", *args) + system("lcov", "--rc", "geninfo_unexecuted_blocks=1", "--rc", "lcov_branch_coverage=1", *args, exception: true) end $info_files = [] @@ -41,11 +41,19 @@ def run_lcov_remove(info_src, info_out) ext/-test-/* ext/nkf/nkf-utf8/nkf.c ).each {|f| dirs << File.join(File.dirname(__dir__), f) } - run_lcov("--remove", info_src, *dirs, "-o", info_out) + run_lcov("--ignore-errors", "unused", "--remove", info_src, *dirs, "-o", info_out) end def run_genhtml(info, out) - system("genhtml", "--branch-coverage", "--ignore-errors", "source", info, "-o", out) + base_dir = File.dirname(File.dirname(__dir__)) + ignore_errors = %w(source unmapped category).reject do |a| + Open3.capture3("genhtml", "--ignore-errors", a)[1].include?("unknown argument for --ignore-errors") + end + system("genhtml", + "--branch-coverage", + "--prefix", base_dir, + *ignore_errors.flat_map {|a| ["--ignore-errors", a] }, + info, "-o", out, exception: true) end def gen_rb_lcov(file) diff --git a/tool/runruby.rb b/tool/runruby.rb index 9562efc00d..ec63d1008a 100755 --- a/tool/runruby.rb +++ b/tool/runruby.rb @@ -11,6 +11,10 @@ when ENV['RUNRUBY_USE_GDB'] == 'true' debugger = :gdb when ENV['RUNRUBY_USE_LLDB'] == 'true' debugger = :lldb +when ENV['RUNRUBY_USE_RR'] == 'true' + debugger = :rr +when ENV['RUNRUBY_YJIT_STATS'] + use_yjit_stat = true end while arg = ARGV[0] break ARGV.shift if arg == '--' @@ -120,27 +124,27 @@ if e = ENV["RUBYLIB"] end env["RUBYLIB"] = $:.replace(libs).join(File::PATH_SEPARATOR) +gem_path = [abs_archdir, srcdir].map {|d| File.realdirpath(".bundle", d)} +if e = ENV["GEM_PATH"] + gem_path |= e.split(File::PATH_SEPARATOR) +end +env["GEM_PATH"] = gem_path.join(File::PATH_SEPARATOR) + libruby_so = File.join(abs_archdir, config['LIBRUBY_SO']) if File.file?(libruby_so) if e = config['LIBPATHENV'] and !e.empty? env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) end - unless runner - if e = config['PRELOADENV'] - e = nil if e.empty? - e ||= "LD_PRELOAD" if /linux/ =~ RUBY_PLATFORM - end - if e - env[e] = [libruby_so, ENV[e]].compact.join(File::PATH_SEPARATOR) - end - end end +# Work around a bug in FreeBSD 13.2 which can cause fork(2) to hang +# See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271490 +env['LD_BIND_NOW'] = 'yes' if /freebsd/ =~ RUBY_PLATFORM ENV.update env if debugger case debugger - when :gdb, nil + when :gdb debugger = %W'gdb -x #{srcdir}/.gdbinit' if File.exist?(gdb = 'run.gdb') or File.exist?(gdb = File.join(abs_archdir, 'run.gdb')) @@ -154,6 +158,8 @@ if debugger debugger.push('-s', lldb) end debugger << '--' + when :rr + debugger = ['rr', 'record'] end if idx = precommand.index(:debugger) @@ -164,6 +170,9 @@ if debugger end cmd = [runner || ruby] +if use_yjit_stat + cmd << '--yjit-stats' +end cmd.concat(ARGV) cmd.unshift(*precommand) unless precommand.empty? diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 5c2ffc649e..14d7a3893d 100644..100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -1,552 +1,900 @@ -# sync upstream github repositories to ruby repository +#!/usr/bin/env ruby +# Sync upstream github repositories to ruby repository. +# See `tool/sync_default_gems.rb --help` for how to use this. require 'fileutils' -include FileUtils - -REPOSITORIES = { - rubygems: 'rubygems/rubygems', - rdoc: 'ruby/rdoc', - reline: 'ruby/reline', - json: 'flori/json', - psych: 'ruby/psych', - fileutils: 'ruby/fileutils', - fiddle: 'ruby/fiddle', - stringio: 'ruby/stringio', - "io-console": 'ruby/io-console', - "io-nonblock": 'ruby/io-nonblock', - "io-wait": 'ruby/io-wait', - csv: 'ruby/csv', - etc: 'ruby/etc', - date: 'ruby/date', - zlib: 'ruby/zlib', - fcntl: 'ruby/fcntl', - strscan: 'ruby/strscan', - ipaddr: 'ruby/ipaddr', - logger: 'ruby/logger', - ostruct: 'ruby/ostruct', - irb: 'ruby/irb', - forwardable: "ruby/forwardable", - mutex_m: "ruby/mutex_m", - racc: "ruby/racc", - singleton: "ruby/singleton", - open3: "ruby/open3", - getoptlong: "ruby/getoptlong", - pstore: "ruby/pstore", - delegate: "ruby/delegate", - benchmark: "ruby/benchmark", - cgi: "ruby/cgi", - readline: "ruby/readline", - "readline-ext": "ruby/readline-ext", - observer: "ruby/observer", - timeout: "ruby/timeout", - yaml: "ruby/yaml", - uri: "ruby/uri", - openssl: "ruby/openssl", - did_you_mean: "ruby/did_you_mean", - weakref: "ruby/weakref", - tempfile: "ruby/tempfile", - tmpdir: "ruby/tmpdir", - English: "ruby/English", - "net-protocol": "ruby/net-protocol", - "net-http": "ruby/net-http", - bigdecimal: "ruby/bigdecimal", - optparse: "ruby/optparse", - set: "ruby/set", - find: "ruby/find", - rinda: "ruby/rinda", - erb: "ruby/erb", - nkf: "ruby/nkf", - tsort: "ruby/tsort", - abbrev: "ruby/abbrev", - shellwords: "ruby/shellwords", - base64: "ruby/base64", - syslog: "ruby/syslog", - "open-uri": "ruby/open-uri", - securerandom: "ruby/securerandom", - resolv: "ruby/resolv", - "resolv-replace": "ruby/resolv-replace", - time: "ruby/time", - pp: "ruby/pp", - prettyprint: "ruby/prettyprint", - drb: "ruby/drb", - pathname: "ruby/pathname", - digest: "ruby/digest", - error_highlight: "ruby/error_highlight", - un: "ruby/un", -} - -def sync_default_gems(gem) - repo = REPOSITORIES[gem.to_sym] - puts "Sync #{repo}" - - upstream = File.join("..", "..", repo) - - case gem - when "rubygems" - rm_rf(%w[lib/rubygems lib/rubygems.rb test/rubygems]) - cp_r(Dir.glob("#{upstream}/lib/rubygems*"), "lib") - cp_r("#{upstream}/test/rubygems", "test") - rm_rf(%w[lib/bundler lib/bundler.rb libexec/bundler libexec/bundle spec/bundler tool/bundler/*]) - cp_r(Dir.glob("#{upstream}/bundler/lib/bundler*"), "lib") - cp_r(Dir.glob("#{upstream}/bundler/exe/bundle*"), "libexec") - - gemspec_content = File.readlines("#{upstream}/bundler/bundler.gemspec").map do |line| +require "rbconfig" +require "find" +require "tempfile" + +module SyncDefaultGems + include FileUtils + extend FileUtils + + module_function + + # upstream: "owner/repo" + # branch: "branch_name" + # mappings: [ ["path_in_upstream", "path_in_ruby"], ... ] + # NOTE: path_in_ruby is assumed to be "owned" by this gem, and the contents + # will be removed before sync + # exclude: [ "fnmatch_pattern_after_mapping", ... ] + Repository = Data.define(:upstream, :branch, :mappings, :exclude) do + def excluded?(newpath) + p = newpath + until p == "." + return true if exclude.any? {|pat| File.fnmatch?(pat, p, File::FNM_PATHNAME|File::FNM_EXTGLOB)} + p = File.dirname(p) + end + false + end + + def rewrite_for_ruby(path) + newpath = mappings.find do |src, dst| + if path == src || path.start_with?(src + "/") + break path.sub(src, dst) + end + end + return nil unless newpath + return nil if excluded?(newpath) + newpath + end + end + + CLASSICAL_DEFAULT_BRANCH = "master" + + def repo((upstream, branch), mappings, exclude: []) + branch ||= CLASSICAL_DEFAULT_BRANCH + exclude += ["ext/**/depend"] + Repository.new(upstream:, branch:, mappings:, exclude:) + end + + def lib((upstream, branch), gemspec_in_subdir: false) + _org, name = upstream.split("/") + gemspec_dst = gemspec_in_subdir ? "lib/#{name}/#{name}.gemspec" : "lib/#{name}.gemspec" + repo([upstream, branch], [ + ["lib/#{name}.rb", "lib/#{name}.rb"], + ["lib/#{name}", "lib/#{name}"], + ["test/test_#{name}.rb", "test/test_#{name}.rb"], + ["test/#{name}", "test/#{name}"], + ["#{name}.gemspec", gemspec_dst], + ]) + end + + # Note: tool/auto_review_pr.rb also depends on these constants. + NO_UPSTREAM = [ + "lib/unicode_normalize", # not to match with "lib/un" + ] + REPOSITORIES = { + Onigmo: repo("k-takata/Onigmo", [ + ["regcomp.c", "regcomp.c"], + ["regenc.c", "regenc.c"], + ["regenc.h", "regenc.h"], + ["regerror.c", "regerror.c"], + ["regexec.c", "regexec.c"], + ["regint.h", "regint.h"], + ["regparse.c", "regparse.c"], + ["regparse.h", "regparse.h"], + ["regsyntax.c", "regsyntax.c"], + ["onigmo.h", "include/ruby/onigmo.h"], + ["enc", "enc"], + ]), + "io-console": repo("ruby/io-console", [ + ["ext/io/console", "ext/io/console"], + ["test/io/console", "test/io/console"], + ["lib/io/console", "ext/io/console/lib/console"], + ["io-console.gemspec", "ext/io/console/io-console.gemspec"], + ]), + "io-nonblock": repo("ruby/io-nonblock", [ + ["ext/io/nonblock", "ext/io/nonblock"], + ["test/io/nonblock", "test/io/nonblock"], + ["io-nonblock.gemspec", "ext/io/nonblock/io-nonblock.gemspec"], + ]), + "io-wait": repo("ruby/io-wait", [ + ["ext/io/wait", "ext/io/wait"], + ["test/io/wait", "test/io/wait"], + ["io-wait.gemspec", "ext/io/wait/io-wait.gemspec"], + ]), + "net-http": repo("ruby/net-http", [ + ["lib/net/http.rb", "lib/net/http.rb"], + ["lib/net/http", "lib/net/http"], + ["test/net/http", "test/net/http"], + ["net-http.gemspec", "lib/net/http/net-http.gemspec"], + ]), + "net-protocol": repo("ruby/net-protocol", [ + ["lib/net/protocol.rb", "lib/net/protocol.rb"], + ["test/net/protocol", "test/net/protocol"], + ["net-protocol.gemspec", "lib/net/net-protocol.gemspec"], + ]), + "open-uri": lib("ruby/open-uri"), + "win32-registry": repo("ruby/win32-registry", [ + ["lib/win32/registry.rb", "ext/win32/lib/win32/registry.rb"], + ["test/win32/test_registry.rb", "test/win32/test_registry.rb"], + ["win32-registry.gemspec", "ext/win32/win32-registry.gemspec"], + ]), + English: lib("ruby/English"), + cgi: repo("ruby/cgi", [ + ["ext/cgi", "ext/cgi"], + ["lib/cgi/escape.rb", "lib/cgi/escape.rb"], + ["test/cgi/test_cgi_escape.rb", "test/cgi/test_cgi_escape.rb"], + ["test/cgi/update_env.rb", "test/cgi/update_env.rb"], + ]), + date: repo("ruby/date", [ + ["doc/date", "doc/date"], + ["ext/date", "ext/date"], + ["lib", "ext/date/lib"], + ["test/date", "test/date"], + ["date.gemspec", "ext/date/date.gemspec"], + ], exclude: [ + "ext/date/lib/date_core.bundle", + ]), + delegate: lib("ruby/delegate"), + did_you_mean: repo("ruby/did_you_mean", [ + ["lib/did_you_mean.rb", "lib/did_you_mean.rb"], + ["lib/did_you_mean", "lib/did_you_mean"], + ["test", "test/did_you_mean"], + ["did_you_mean.gemspec", "lib/did_you_mean/did_you_mean.gemspec"], + ], exclude: [ + "test/did_you_mean/lib", + "test/did_you_mean/tree_spell/test_explore.rb", + ]), + digest: repo("ruby/digest", [ + ["ext/digest/lib/digest/sha2", "ext/digest/sha2/lib/sha2"], + ["ext/digest", "ext/digest"], + ["lib/digest.rb", "ext/digest/lib/digest.rb"], + ["lib/digest/version.rb", "ext/digest/lib/digest/version.rb"], + ["lib/digest/sha2.rb", "ext/digest/sha2/lib/sha2.rb"], + ["test/digest", "test/digest"], + ["digest.gemspec", "ext/digest/digest.gemspec"], + ]), + erb: repo("ruby/erb", [ + ["ext/erb", "ext/erb"], + ["lib/erb", "lib/erb"], + ["lib/erb.rb", "lib/erb.rb"], + ["test/erb", "test/erb"], + ["erb.gemspec", "lib/erb/erb.gemspec"], + ["libexec/erb", "libexec/erb"], + ]), + error_highlight: repo("ruby/error_highlight", [ + ["lib/error_highlight.rb", "lib/error_highlight.rb"], + ["lib/error_highlight", "lib/error_highlight"], + ["test", "test/error_highlight"], + ["error_highlight.gemspec", "lib/error_highlight/error_highlight.gemspec"], + ]), + etc: repo("ruby/etc", [ + ["ext/etc", "ext/etc"], + ["test/etc", "test/etc"], + ["etc.gemspec", "ext/etc/etc.gemspec"], + ]), + fcntl: repo("ruby/fcntl", [ + ["ext/fcntl", "ext/fcntl"], + ["fcntl.gemspec", "ext/fcntl/fcntl.gemspec"], + ]), + fileutils: lib("ruby/fileutils"), + find: lib("ruby/find"), + forwardable: lib("ruby/forwardable", gemspec_in_subdir: true), + ipaddr: lib("ruby/ipaddr"), + json: repo("ruby/json", [ + ["ext/json/ext", "ext/json"], + ["test/json", "test/json"], + ["lib", "ext/json/lib"], + ["json.gemspec", "ext/json/json.gemspec"], + ], exclude: [ + "ext/json/lib/json/ext/.keep", + "ext/json/lib/json/pure.rb", + "ext/json/lib/json/pure", + "ext/json/lib/json/truffle_ruby", + "test/json/lib", + "ext/json/extconf.rb", + ]), + mmtk: repo(["ruby/mmtk", "main"], [ + ["gc/mmtk", "gc/mmtk"], + ]), + open3: lib("ruby/open3", gemspec_in_subdir: true).tap { + it.exclude << "lib/open3/jruby_windows.rb" + }, + openssl: repo("ruby/openssl", [ + ["ext/openssl", "ext/openssl"], + ["lib", "ext/openssl/lib"], + ["test/openssl", "test/openssl"], + ["sample", "sample/openssl"], + ["openssl.gemspec", "ext/openssl/openssl.gemspec"], + ["History.md", "ext/openssl/History.md"], + ], exclude: [ + "test/openssl/envutil.rb", + "ext/openssl/depend", + ]), + optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap { + it.mappings << ["doc/optparse", "doc/optparse"] + }, + pathname: repo("ruby/pathname", [ + ["ext/pathname/pathname.c", "pathname.c"], + ["lib/pathname_builtin.rb", "pathname_builtin.rb"], + ["lib/pathname.rb", "lib/pathname.rb"], + ["test/pathname", "test/pathname"], + ]), + pp: lib("ruby/pp"), + prettyprint: lib("ruby/prettyprint"), + prism: repo(["ruby/prism", "main"], [ + ["ext/prism", "prism"], + ["lib/prism.rb", "lib/prism.rb"], + ["lib/prism", "lib/prism"], + ["test/prism", "test/prism"], + ["src", "prism"], + ["prism.gemspec", "lib/prism/prism.gemspec"], + ["include/prism", "prism"], + ["include/prism.h", "prism/prism.h"], + ["config.yml", "prism/config.yml"], + ["templates", "prism/templates"], + ], exclude: [ + "prism/templates/{javascript,java,rbi,sig}", + "test/prism/snapshots_test.rb", + "test/prism/snapshots", + "prism/extconf.rb", + "prism/srcs.mk*", + ]), + psych: repo("ruby/psych", [ + ["ext/psych", "ext/psych"], + ["lib", "ext/psych/lib"], + ["test/psych", "test/psych"], + ["psych.gemspec", "ext/psych/psych.gemspec"], + ], exclude: [ + "ext/psych/lib/org", + "ext/psych/lib/psych.jar", + "ext/psych/lib/psych_jars.rb", + "ext/psych/lib/psych.{bundle,so}", + "ext/psych/lib/2.*", + "ext/psych/yaml/LICENSE", + "ext/psych/.gitignore", + ]), + resolv: repo("ruby/resolv", [ + ["lib/resolv.rb", "lib/resolv.rb"], + ["test/resolv", "test/resolv"], + ["resolv.gemspec", "lib/resolv.gemspec"], + ["ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32/resolv.rb"], + ["ext/win32/resolv", "ext/win32/resolv"], + ]), + rubygems: repo("ruby/rubygems", [ + ["lib/rubygems.rb", "lib/rubygems.rb"], + ["lib/rubygems", "lib/rubygems"], + ["test/rubygems", "test/rubygems"], + ["bundler/lib/bundler.rb", "lib/bundler.rb"], + ["bundler/lib/bundler", "lib/bundler"], + ["bundler/exe/bundle", "libexec/bundle"], + ["bundler/exe/bundler", "libexec/bundler"], + ["bundler/bundler.gemspec", "lib/bundler/bundler.gemspec"], + ["bundler/spec", "spec/bundler"], + *["bundle", "parallel_rspec", "rspec"].map {|binstub| + ["bundler/bin/#{binstub}", "spec/bin/#{binstub}"] + }, + *%w[dev_gems test_gems rubocop_gems standard_gems].flat_map {|gemfile| + ["rb.lock", "rb"].map do |ext| + ["tool/bundler/#{gemfile}.#{ext}", "tool/bundler/#{gemfile}.#{ext}"] + end + }, + ], exclude: [ + "spec/bundler/bin", + "spec/bundler/support/artifice/vcr_cassettes", + "spec/bundler/support/artifice/used_cassettes.txt", + "lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}", + ]), + securerandom: lib("ruby/securerandom"), + shellwords: lib("ruby/shellwords"), + singleton: lib("ruby/singleton"), + stringio: repo("ruby/stringio", [ + ["ext/stringio", "ext/stringio"], + ["test/stringio", "test/stringio"], + ["stringio.gemspec", "ext/stringio/stringio.gemspec"], + ["doc/stringio", "doc/stringio"], + ], exclude: [ + "ext/stringio/README.md", + ]), + strscan: repo("ruby/strscan", [ + ["ext/strscan", "ext/strscan"], + ["lib", "ext/strscan/lib"], + ["test/strscan", "test/strscan"], + ["strscan.gemspec", "ext/strscan/strscan.gemspec"], + ["doc/strscan", "doc/strscan"], + ], exclude: [ + "ext/strscan/regenc.h", + "ext/strscan/regint.h", + ]), + syntax_suggest: lib(["ruby/syntax_suggest", "main"], gemspec_in_subdir: true), + tempfile: lib("ruby/tempfile"), + time: lib("ruby/time"), + timeout: lib("ruby/timeout"), + tmpdir: lib("ruby/tmpdir"), + un: lib("ruby/un"), + uri: lib("ruby/uri", gemspec_in_subdir: true), + weakref: lib("ruby/weakref"), + yaml: lib("ruby/yaml", gemspec_in_subdir: true), + zlib: repo("ruby/zlib", [ + ["ext/zlib", "ext/zlib"], + ["test/zlib", "test/zlib"], + ["zlib.gemspec", "ext/zlib/zlib.gemspec"], + ]), + }.transform_keys(&:to_s) + + class << Repository + def find_upstream(file) + return if NO_UPSTREAM.any? {|dst| file.start_with?(dst) } + REPOSITORIES.find do |repo_name, repository| + if repository.mappings.any? {|_src, dst| file.start_with?(dst) } + break repo_name + end + end + end + + def group(files) + files.group_by {|file| find_upstream(file)} + end + end + + # Allow synchronizing commits up to this FETCH_DEPTH. We've historically merged PRs + # with about 250 commits to ruby/ruby, so we use this depth for ruby/ruby in general. + FETCH_DEPTH = 500 + + def pipe_readlines(args, rs: "\0", chomp: true) + IO.popen(args) do |f| + f.readlines(rs, chomp: chomp) + end + end + + def porcelain_status(*pattern) + pipe_readlines(%W"git status --porcelain --no-renames -z --" + pattern) + end + + def replace_rdoc_ref(file) + src = File.binread(file) + changed = false + changed |= src.gsub!(%r[\[\Khttps://docs\.ruby-lang\.org/en/master(?:/doc)?/(([A-Z]\w+(?:/[A-Z]\w+)*)|\w+_rdoc)\.html(\#\S+)?(?=\])]) do + name, mod, label = $1, $2, $3 + mod &&= mod.gsub('/', '::') + if label && (m = label.match(/\A\#(?:method-([ci])|(?:(?:class|module)-#{mod}-)?label)-([-+\w]+)\z/)) + scope, label = m[1], m[2] + scope = scope ? scope.tr('ci', '.#') : '@' + end + "rdoc-ref:#{mod || name.chomp("_rdoc") + ".rdoc"}#{scope}#{label}" + end + changed or return false + File.binwrite(file, src) + return true + end + + def replace_rdoc_ref_all + result = porcelain_status("*.c", "*.rb", "*.rdoc") + result.map! {|line| line[/\A.M (.*)/, 1]} + result.compact! + return if result.empty? + result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + result) + result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def replace_rdoc_ref_all_full + Dir.glob("**/*.{c,rb,rdoc}").inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def rubygems_do_fixup + gemspec_content = File.readlines("lib/bundler/bundler.gemspec").map do |line| next if line =~ /LICENSE\.md/ - line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec").gsub('"exe"', '"libexec"') + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") end.compact.join File.write("lib/bundler/bundler.gemspec", gemspec_content) - cp_r("#{upstream}/bundler/spec", "spec/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/test_gems*"), "tool/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/rubocop_gems*"), "tool/bundler") - cp_r(Dir.glob("#{upstream}/bundler/tool/bundler/standard_gems*"), "tool/bundler") - rm_rf(%w[spec/bundler/support/artifice/vcr_cassettes]) - when "rdoc" - rm_rf(%w[lib/rdoc lib/rdoc.rb test/rdoc libexec/rdoc libexec/ri]) - cp_r(Dir.glob("#{upstream}/lib/rdoc*"), "lib") - cp_r("#{upstream}/test/rdoc", "test") - cp_r("#{upstream}/rdoc.gemspec", "lib/rdoc") - cp_r("#{upstream}/Gemfile", "lib/rdoc") - cp_r("#{upstream}/Rakefile", "lib/rdoc") - cp_r("#{upstream}/exe/rdoc", "libexec") - cp_r("#{upstream}/exe/ri", "libexec") - parser_files = { - 'lib/rdoc/markdown.kpeg' => 'lib/rdoc/markdown.rb', - 'lib/rdoc/markdown/literals.kpeg' => 'lib/rdoc/markdown/literals.rb', - 'lib/rdoc/rd/block_parser.ry' => 'lib/rdoc/rd/block_parser.rb', - 'lib/rdoc/rd/inline_parser.ry' => 'lib/rdoc/rd/inline_parser.rb' - } - Dir.chdir(upstream) do - `bundle install` - parser_files.each_value do |dst| - `bundle exec rake #{dst}` - end + ["bundle", "parallel_rspec", "rspec"].each do |binstub| + path = "spec/bin/#{binstub}" + next unless File.exist?(path) + content = File.read(path).gsub("../spec", "../bundler") + File.write(path, content) + chmod("+x", path) end - parser_files.each_pair do |src, dst| - rm_rf(src) - cp_r("#{upstream}/#{dst}", dst) - end - `git checkout lib/rdoc/.document` - when "reline" - rm_rf(%w[lib/reline lib/reline.rb test/reline]) - cp_r(Dir.glob("#{upstream}/lib/reline*"), "lib") - cp_r("#{upstream}/test/reline", "test") - cp_r("#{upstream}/reline.gemspec", "lib/reline") - when "irb" - rm_rf(%w[lib/irb lib/irb.rb test/irb]) - cp_r(Dir.glob("#{upstream}/lib/irb*"), "lib") - cp_r("#{upstream}/test/irb", "test") - cp_r("#{upstream}/irb.gemspec", "lib/irb") - cp_r("#{upstream}/man/irb.1", "man/irb.1") - cp_r("#{upstream}/doc/irb", "doc") - when "json" - rm_rf(%w[ext/json test/json]) - cp_r("#{upstream}/ext/json/ext", "ext/json") - cp_r("#{upstream}/tests", "test/json") - rm_rf("test/json/lib") - cp_r("#{upstream}/lib", "ext/json") - cp_r("#{upstream}/json.gemspec", "ext/json") - cp_r("#{upstream}/VERSION", "ext/json") - rm_rf(%w[ext/json/lib/json/ext ext/json/lib/json/pure.rb ext/json/lib/json/pure]) - `git checkout ext/json/extconf.rb ext/json/parser/prereq.mk ext/json/generator/depend ext/json/parser/depend ext/json/depend` - when "psych" - rm_rf(%w[ext/psych test/psych]) - cp_r("#{upstream}/ext/psych", "ext") - cp_r("#{upstream}/lib", "ext/psych") - cp_r("#{upstream}/test/psych", "test") - rm_rf(%w[ext/psych/lib/org ext/psych/lib/psych.jar ext/psych/lib/psych_jars.rb]) - rm_rf(%w[ext/psych/lib/psych.{bundle,so} ext/psych/lib/2.*]) - rm_rf(["ext/psych/yaml/LICENSE"]) - cp_r("#{upstream}/psych.gemspec", "ext/psych") - `git checkout ext/psych/depend` - when "fiddle" - rm_rf(%w[ext/fiddle test/fiddle]) - cp_r("#{upstream}/ext/fiddle", "ext") - cp_r("#{upstream}/lib", "ext/fiddle") - cp_r("#{upstream}/test/fiddle", "test") - cp_r("#{upstream}/fiddle.gemspec", "ext/fiddle") - `git checkout ext/fiddle/depend` - rm_rf(%w[ext/fiddle/lib/fiddle.{bundle,so}]) - when "stringio" - rm_rf(%w[ext/stringio test/stringio]) - cp_r("#{upstream}/ext/stringio", "ext") - cp_r("#{upstream}/test/stringio", "test") - cp_r("#{upstream}/stringio.gemspec", "ext/stringio") - `git checkout ext/stringio/depend ext/stringio/README.md` - when "io-console" - rm_rf(%w[ext/io/console test/io/console]) - cp_r("#{upstream}/ext/io/console", "ext/io") - cp_r("#{upstream}/test/io/console", "test/io") - mkdir_p("ext/io/console/lib") - cp_r("#{upstream}/lib/io/console", "ext/io/console/lib") - rm_rf("ext/io/console/lib/console/ffi") - cp_r("#{upstream}/io-console.gemspec", "ext/io/console") - `git checkout ext/io/console/depend` - when "io-nonblock" - rm_rf(%w[ext/io/nonblock test/io/nonblock]) - cp_r("#{upstream}/ext/io/nonblock", "ext/io") - cp_r("#{upstream}/test/io/nonblock", "test/io") - cp_r("#{upstream}/io-nonblock.gemspec", "ext/io/nonblock") - `git checkout ext/io/nonblock/depend` - when "io-wait" - rm_rf(%w[ext/io/wait test/io/wait]) - cp_r("#{upstream}/ext/io/wait", "ext/io") - cp_r("#{upstream}/test/io/wait", "test/io") - cp_r("#{upstream}/io-wait.gemspec", "ext/io/wait") - `git checkout ext/io/wait/depend` - when "etc" - rm_rf(%w[ext/etc test/etc]) - cp_r("#{upstream}/ext/etc", "ext") - cp_r("#{upstream}/test/etc", "test") - cp_r("#{upstream}/etc.gemspec", "ext/etc") - `git checkout ext/etc/depend` - when "date" - rm_rf(%w[ext/date test/date]) - cp_r("#{upstream}/ext/date", "ext") - cp_r("#{upstream}/lib", "ext/date") - cp_r("#{upstream}/test/date", "test") - cp_r("#{upstream}/date.gemspec", "ext/date") - `git checkout ext/date/depend` - rm_rf(["ext/date/lib/date_core.bundle"]) - when "zlib" - rm_rf(%w[ext/zlib test/zlib]) - cp_r("#{upstream}/ext/zlib", "ext") - cp_r("#{upstream}/test/zlib", "test") - cp_r("#{upstream}/zlib.gemspec", "ext/zlib") - `git checkout ext/zlib/depend` - when "fcntl" - rm_rf(%w[ext/fcntl]) - cp_r("#{upstream}/ext/fcntl", "ext") - cp_r("#{upstream}/fcntl.gemspec", "ext/fcntl") - `git checkout ext/fcntl/depend` - when "strscan" - rm_rf(%w[ext/strscan test/strscan]) - cp_r("#{upstream}/ext/strscan", "ext") - cp_r("#{upstream}/test/strscan", "test") - cp_r("#{upstream}/strscan.gemspec", "ext/strscan") - rm_rf(%w["ext/strscan/regenc.h ext/strscan/regint.h"]) - `git checkout ext/strscan/depend` - when "racc" - rm_rf(%w[lib/racc lib/racc.rb ext/racc test/racc]) - cp_r(Dir.glob("#{upstream}/lib/racc*"), "lib") - mkdir_p("ext/racc/cparse") - cp_r(Dir.glob("#{upstream}/ext/racc/cparse/*"), "ext/racc/cparse") - cp_r("#{upstream}/test", "test/racc") - cp_r("#{upstream}/racc.gemspec", "lib/racc") - rm_rf("test/racc/lib") - rm_rf("lib/racc/cparse-jruby.jar") - `git checkout ext/racc/cparse/README ext/racc/cparse/depend` - when "cgi" - rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) - cp_r("#{upstream}/ext/cgi", "ext") - cp_r("#{upstream}/lib", ".") - cp_r("#{upstream}/test/cgi", "test") - cp_r("#{upstream}/cgi.gemspec", "lib/cgi") - `git checkout ext/cgi/escape/depend` - when "openssl" - rm_rf(%w[ext/openssl test/openssl]) - cp_r("#{upstream}/ext/openssl", "ext") - cp_r("#{upstream}/lib", "ext/openssl") - cp_r("#{upstream}/test/openssl", "test") - rm_rf("test/openssl/envutil.rb") - cp_r("#{upstream}/openssl.gemspec", "ext/openssl") - cp_r("#{upstream}/History.md", "ext/openssl") - `git checkout ext/openssl/depend` - when "net-protocol" - rm_rf(%w[lib/net/protocol.rb lib/net/net-protocol.gemspec test/net/protocol]) - cp_r("#{upstream}/lib/net/protocol.rb", "lib/net") - cp_r("#{upstream}/test/net/protocol", "test/net") - cp_r("#{upstream}/net-protocol.gemspec", "lib/net") - when "net-http" - rm_rf(%w[lib/net/http.rb lib/net/http test/net/http]) - cp_r("#{upstream}/lib/net/http.rb", "lib/net") - cp_r("#{upstream}/lib/net/http", "lib/net") - cp_r("#{upstream}/test/net/http", "test/net") - cp_r("#{upstream}/net-http.gemspec", "lib/net/http") - when "readline-ext" - rm_rf(%w[ext/readline test/readline]) - cp_r("#{upstream}/ext/readline", "ext") - cp_r("#{upstream}/test/readline", "test") - cp_r("#{upstream}/readline-ext.gemspec", "ext/readline") - `git checkout ext/readline/depend` - when "did_you_mean" - rm_rf(%w[lib/did_you_mean lib/did_you_mean.rb test/did_you_mean]) - cp_r(Dir.glob("#{upstream}/lib/did_you_mean*"), "lib") - cp_r("#{upstream}/did_you_mean.gemspec", "lib/did_you_mean") - cp_r("#{upstream}/test", "test/did_you_mean") - rm_rf(%w[test/did_you_mean/tree_spell/test_explore.rb]) - when "erb" - rm_rf(%w[lib/erb* test/erb libexec/erb]) - cp_r("#{upstream}/lib/erb.rb", "lib") - cp_r("#{upstream}/test/erb", "test") - cp_r("#{upstream}/erb.gemspec", "lib") - cp_r("#{upstream}/libexec/erb", "libexec") - when "nkf" - rm_rf(%w[ext/nkf test/nkf]) - cp_r("#{upstream}/ext/nkf", "ext") - cp_r("#{upstream}/lib", "ext/nkf") - cp_r("#{upstream}/test/nkf", "test") - cp_r("#{upstream}/nkf.gemspec", "ext/nkf") - `git checkout ext/nkf/depend` - when "syslog" - rm_rf(%w[ext/syslog test/syslog test/test_syslog.rb]) - cp_r("#{upstream}/ext/syslog", "ext") - cp_r("#{upstream}/lib", "ext/syslog") - cp_r("#{upstream}/test/syslog", "test") - cp_r("#{upstream}/test/test_syslog.rb", "test") - cp_r("#{upstream}/syslog.gemspec", "ext/syslog") - `git checkout ext/syslog/depend` - when "bigdecimal" - rm_rf(%w[ext/bigdecimal test/bigdecimal]) - cp_r("#{upstream}/ext/bigdecimal", "ext") - cp_r("#{upstream}/sample", "ext/bigdecimal") - cp_r("#{upstream}/lib", "ext/bigdecimal") - cp_r("#{upstream}/test/bigdecimal", "test") - cp_r("#{upstream}/bigdecimal.gemspec", "ext/bigdecimal") - `git checkout ext/bigdecimal/depend` - when "pathname" - rm_rf(%w[ext/pathname test/pathname]) - cp_r("#{upstream}/ext/pathname", "ext") - cp_r("#{upstream}/test/pathname", "test") - cp_r("#{upstream}/lib", "ext/pathname") - cp_r("#{upstream}/pathname.gemspec", "ext/pathname") - `git checkout ext/pathname/depend` - when "digest" - rm_rf(%w[ext/digest test/digest]) - cp_r("#{upstream}/ext/digest", "ext") - mkdir_p("ext/digest/lib") - cp_r("#{upstream}/lib/digest.rb", "ext/digest/lib/") - cp_r("#{upstream}/test/digest", "test") - cp_r("#{upstream}/digest.gemspec", "ext/digest") - `git checkout ext/digest/depend ext/digest/*/depend` - when "set" - sync_lib gem, upstream - cp_r("#{upstream}/test", ".") - when "optparse" - sync_lib gem, upstream - rm_rf(%w[doc/optparse]) - mkdir_p("doc/optparse") - cp_r("#{upstream}/doc/optparse", "doc") - when "error_highlight" - rm_rf(%w[lib/error_highlight lib/error_highlight.rb test/error_highlight]) - cp_r(Dir.glob("#{upstream}/lib/error_highlight*"), "lib") - cp_r("#{upstream}/error_highlight.gemspec", "lib/error_highlight") - cp_r("#{upstream}/test", "test/error_highlight") - else - sync_lib gem, upstream end -end -IGNORE_FILE_PATTERN = - /\A(?:[A-Z]\w*\.(?:md|txt) - |[^\/]+\.yml - |\.git.* - |[A-Z]\w+file - |COPYING - |rakelib\/ - )\z/x - -def message_filter(repo, sha) - log = STDIN.read - log.delete!("\r") - url = "https://github.com/#{repo}" - print "[#{repo}] ", log.gsub(/fix +#\d+|\(#\d+\)/i) { - $&.sub(/#/) {"#{url}/pull/"} - }.sub(/\s*(?=(?i:\nCo-authored-by:.*)*\Z)/) { - "\n\n" "#{url}/commit/#{sha[0,10]}\n" - } -end + # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. + def sync_default_gems(gem) + config = REPOSITORIES[gem] + puts "Sync #{config.upstream}" + + upstream = File.join("..", "..", config.upstream) -def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo = REPOSITORIES[gem.to_sym] - puts "Sync #{repo} with commit history." + config.mappings.each do |src, dst| + rm_rf(dst) + end - IO.popen(%W"git remote") do |f| - unless f.read.split.include?(gem) - `git remote add #{gem} git@github.com:#{repo}.git` + copied = Set.new + config.mappings.each do |src, dst| + prefix = File.join(upstream, src) + # Maybe mapping needs to be updated? + next unless File.exist?(prefix) + Find.find(prefix) do |path| + next if File.directory?(path) + if copied.add?(path) + newpath = config.rewrite_for_ruby(path.sub(%r{\A#{Regexp.escape(upstream)}/}, "")) + next unless newpath + mkdir_p(File.dirname(newpath)) + cp(path, newpath) + end + end end + + porcelain_status().each do |line| + /\A(?:.)(?:.) (?<path>.*)\z/ =~ line or raise + if config.excluded?(path) + puts "Restoring excluded file: #{path}" + IO.popen(%W"git checkout --" + [path], "rb", &:read) + end + end + + # RubyGems/Bundler needs special care + if gem == "rubygems" + rubygems_do_fixup + end + + check_prerelease_version(gem) + + # Architecture-dependent files must not pollute libdir. + rm_rf(Dir["lib/**/*.#{RbConfig::CONFIG['DLEXT']}"]) + replace_rdoc_ref_all end - system(*%W"git fetch --no-tags #{gem}") - if ranges == true - pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" - log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", &:read) - ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/master"] + def check_prerelease_version(gem) + return if ["rubygems", "mmtk", "cgi", "pathname", "Onigmo"].include?(gem) + + require "net/https" + require "json" + require "uri" + + uri = URI("https://rubygems.org/api/v1/versions/#{gem.downcase}/latest.json") + response = Net::HTTP.get(uri) + latest_version = JSON.parse(response)["version"] + + gemspec = [ + "lib/#{gem}/#{gem}.gemspec", + "lib/#{gem}.gemspec", + "ext/#{gem}/#{gem}.gemspec", + "ext/#{gem.split("-").join("/")}/#{gem}.gemspec", + "lib/#{gem.split("-").first}/#{gem}.gemspec", + "ext/#{gem.split("-").first}/#{gem}.gemspec", + "lib/#{gem.split("-").join("/")}/#{gem}.gemspec", + ].find{|gemspec| File.exist?(gemspec)} + spec = Gem::Specification.load(gemspec) + puts "#{gem}-#{spec.version} is not latest version of rubygems.org" if spec.version.to_s != latest_version end - commits = ranges.flat_map do |range| - unless range.include?("..") - range = "#{range}~1..#{range}" + def message_filter(repo, sha, log, context: nil) + unless repo.count("/") == 1 and /\A\S+\z/ =~ repo + raise ArgumentError, "invalid repository: #{repo}" + end + unless /\A\h{10,40}\z/ =~ sha + raise ArgumentError, "invalid commit-hash: #{sha}" + end + repo_url = "https://github.com/#{repo}" + + # Log messages generated by GitHub web UI have inconsistent line endings + log = log.delete("\r") + log << "\n" if !log.end_with?("\n") + + # Split the subject from the log message according to git conventions. + # SPECIAL TREAT: when the first line ends with a dot `.` (which is not + # obeying the conventions too), takes only that line. + subject, log = log.split(/\A.+\.\K\n(?=\S)|\n(?:[ \t]*(?:\n|\z))/, 2) + conv = proc do |s| + mod = true if s.gsub!(/\b(?:(?i:fix(?:e[sd])?|close[sd]?|resolve[sd]?) +)\K#(?=\d+\b)|\bGH-#?(?=\d+\b)|\(\K#(?=\d+\))/) { + "#{repo_url}/pull/" + } + mod |= true if s.gsub!(%r{(?<![-\[\](){}\w@/])(?:(\w+(?:-\w+)*/\w+(?:-\w+)*)@)?(\h{10,40})\b}) {|c| + "https://github.com/#{$1 || repo}/commit/#{$2[0,12]}" + } + mod + end + subject = "[#{repo}] #{subject}" + subject.gsub!(/\s*\n\s*/, " ") + if conv[subject] + if subject.size > 68 + subject.gsub!(/\G.{,67}[^\s.,][.,]*\K\s+/, "\n") + end end + commit_url = "#{repo_url}/commit/#{sha[0,10]}\n" + sync_note = context ? "#{commit_url}\n#{context}" : commit_url + if log and !log.empty? + log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last + conv[log] + log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) { + ($~.begin(1) ? "" : "\n\n") + sync_note + ($~.begin(2) ? "\n" : "") + } + else + log = sync_note + end + "#{subject}\n\n#{log}" + end + + def log_format(format, args, &block) + IO.popen(%W[git -c core.autocrlf=false -c core.eol=lf + log --no-show-signature --format=#{format}] + args, "rb", &block) + end + + def commits_in_range(upto, exclude, toplevel:) + args = [upto, *exclude.map {|s|"^#{s}"}] + log_format('%H,%P,%s', %W"--first-parent" + args) do |f| + f.read.split("\n").reverse.flat_map {|commit| + hash, parents, subject = commit.split(',', 3) + parents = parents.split + + # Non-merge commit + if parents.size <= 1 + puts "#{hash} #{subject}" + next [[hash, subject]] + end + + # Clean 2-parent merge commit: follow the other parent as long as it + # contains no potentially-non-clean merges + if parents.size == 2 && + IO.popen(%W"git diff-tree --remerge-diff #{hash}", "rb", &:read).empty? + puts "\e[2mChecking the other parent of #{hash} #{subject}\e[0m" + ret = catch(:quit) { + commits_in_range(parents[1], exclude + [parents[0]], toplevel: false) + } + next ret if ret + end - IO.popen(%W"git log --format=%H,%s #{range} --") do |f| - f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} + unless toplevel + puts "\e[1mMerge commit with possible conflict resolution #{hash} #{subject}\e[0m" + throw :quit + end + + puts "#{hash} #{subject} " \ + "\e[1m[merge commit with possible conflicts, will do a squash merge]\e[0m" + [[hash, subject]] + } end end - # Ignore Merge commit and insufficiency commit for ruby core repository. - commits.delete_if do |sha, subject| - files = IO.popen(%W"git diff-tree --no-commit-id --name-only -r #{sha}", &:readlines) - subject =~ /^Merge/ || subject =~ /^Auto Merge/ || files.all?{|file| file =~ IGNORE_FILE_PATTERN} + # Returns commit list as array of [commit_hash, subject, sync_note]. + def commits_in_ranges(ranges) + ranges.flat_map do |range| + exclude, upto = range.include?("..") ? range.split("..", 2) : ["#{range}~1", range] + puts "Looking for commits in range #{exclude}..#{upto}" + commits_in_range(upto, exclude.empty? ? [] : [exclude], toplevel: true) + end.uniq end - if commits.empty? - puts "No commits to pick" + #-- + # Following methods used by sync_default_gems_with_commits return + # true: success + # false: skipped + # nil: failed + #++ + + def resolve_conflicts(gem, sha, edit) + # Discover unmerged files: any unstaged changes + changes = porcelain_status() + conflict = changes.grep(/\A(?:.[^ ?]) /) {$'} + # If -e option is given, open each conflicted file with an editor + unless conflict.empty? + if edit + case + when (editor = ENV["GIT_EDITOR"] and !editor.empty?) + when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) + end + if editor + system([editor, conflict].join(' ')) + conflict.delete_if {|f| !File.exist?(f)} + return true if conflict.empty? + return system(*%w"git add --", *conflict) + end + end + return false + end + return true end - puts "Try to pick these commits:" - puts commits.map{|commit| commit.join(": ")} - puts "----" + def collect_cacheinfo(tree) + pipe_readlines(%W"git ls-tree -r -t -z #{tree}").filter_map do |line| + fields, path = line.split("\t", 2) + mode, type, object = fields.split(" ", 3) + next unless type == "blob" + [mode, type, object, path] + end + end - failed_commits = [] + def rewrite_cacheinfo(gem, blobs) + config = REPOSITORIES[gem] + rewritten = [] + ignored = blobs.dup + ignored.delete_if do |mode, type, object, path| + newpath = config.rewrite_for_ruby(path) + next unless newpath + rewritten << [mode, type, object, newpath] + end + [rewritten, ignored] + end + + def make_commit_info(gem, sha) + config = REPOSITORIES[gem] + headers, orig = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + /^author (?<author_name>.+?) <(?<author_email>.*?)> (?<author_date>.+?)$/ =~ headers or + raise "unable to parse author info for commit #{sha}" + author = { + "GIT_AUTHOR_NAME" => author_name, + "GIT_AUTHOR_EMAIL" => author_email, + "GIT_AUTHOR_DATE" => author_date, + } + context = nil + if /^parent (?<first_parent>.{40})\nparent .{40}$/ =~ headers + # Squashing a merge commit: keep authorship information + context = IO.popen(%W"git shortlog #{first_parent}..#{sha} --", "rb", &:read) + end + message = message_filter(config.upstream, sha, orig, context: context) + [author, message] + end - ENV["FILTER_BRANCH_SQUELCH_WARNING"] = "1" + def fixup_commit(gem, commit) + wt = File.join("tmp", "sync_default_gems-fixup-worktree") + if File.directory?(wt) + IO.popen(%W"git -C #{wt} clean -xdf", "rb", &:read) + IO.popen(%W"git -C #{wt} reset --hard #{commit}", "rb", &:read) + else + IO.popen(%W"git worktree remove --force #{wt}", "rb", err: File::NULL, &:read) + IO.popen(%W"git worktree add --detach #{wt} #{commit}", "rb", &:read) + end + raise "git worktree prepare failed for commit #{commit}" unless $?.success? - require 'shellwords' - filter = [ - ENV.fetch('RUBY', 'ruby').shellescape, - File.realpath(__FILE__).shellescape, - "--message-filter", - ] - commits.each do |sha, subject| - puts "Pick #{sha} from #{repo}." - - skipped = false - result = IO.popen(%W"git cherry-pick #{sha}", &:read) - if result =~ /nothing\ to\ commit/ - `git reset` - skipped = true - puts "Skip empty commit #{sha}" - end - next if skipped - - if result.empty? - skipped = true - elsif /^CONFLICT/ =~ result - result = IO.popen(%W"git status --porcelain", &:readlines).each(&:chomp!) - result.map! {|line| line[/^.U (.*)/, 1]} - result.compact! - ignore, conflict = result.partition {|name| IGNORE_FILE_PATTERN =~ name} - unless ignore.empty? - system(*%W"git reset HEAD --", *ignore) - File.unlink(*ignore) - ignore = IO.popen(%W"git status --porcelain" + ignore, &:readlines).map! {|line| line[/^.. (.*)/, 1]} - system(*%W"git checkout HEAD --", *ignore) unless ignore.empty? + Dir.chdir(wt) do + if gem == "rubygems" + rubygems_do_fixup end - unless conflict.empty? - if edit - case - when (editor = ENV["GIT_EDITOR"] and !editor.empty?) - when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) - end - if editor - system([editor, conflict].join(' ')) - end + replace_rdoc_ref_all_full + end + + IO.popen(%W"git -C #{wt} add -u", "rb", &:read) + IO.popen(%W"git -C #{wt} commit --amend --no-edit", "rb", &:read) + IO.popen(%W"git -C #{wt} rev-parse HEAD", "rb", &:read).chomp + end + + def make_and_fixup_commit(gem, original_commit, cacheinfo, parent: nil, message: nil, author: nil) + tree = Tempfile.create("sync_default_gems-#{gem}-index") do |f| + File.unlink(f.path) + IO.popen({"GIT_INDEX_FILE" => f.path}, + %W"git update-index --index-info", "wb", out: IO::NULL) do |io| + cacheinfo.each do |mode, type, object, path| + io.puts("#{mode} #{type} #{object}\t#{path}") end end - skipped = !system({"GIT_EDITOR"=>"true"}, *%W"git cherry-pick --no-edit --continue") + raise "git update-index failed" unless $?.success? + + IO.popen({"GIT_INDEX_FILE" => f.path}, %W"git write-tree --missing-ok", "rb", &:read).chomp end - if skipped - failed_commits << sha - `git reset` && `git checkout .` && `git clean -fd` - puts "Failed to pick #{sha}" - next + args = ["-m", message || "Rewriten commit for #{original_commit}"] + args += ["-p", parent] if parent + commit = IO.popen({**author}, %W"git commit-tree #{tree}" + args, "rb", &:read).chomp + + # Apply changes that require a working tree + commit = fixup_commit(gem, commit) + + commit + end + + def rewrite_commit(gem, sha) + author, message = make_commit_info(gem, sha) + new_blobs = collect_cacheinfo("#{sha}") + new_rewritten, new_ignored = rewrite_cacheinfo(gem, new_blobs) + + headers, _ = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + first_parent = headers[/^parent (.{40})$/, 1] + unless first_parent + # Root commit, first time to sync this repo + return make_and_fixup_commit(gem, sha, new_rewritten, message: message, author: author) end - puts "Update commit message: #{sha}" + old_blobs = collect_cacheinfo(first_parent) + old_rewritten, old_ignored = rewrite_cacheinfo(gem, old_blobs) + if old_ignored != new_ignored + paths = (old_ignored + new_ignored - (old_ignored & new_ignored)) + .map {|*_, path| path}.uniq + puts "\e\[1mIgnoring file changes not in mappings: #{paths.join(" ")}\e\[0m" + end + changed_paths = (old_rewritten + new_rewritten - (old_rewritten & new_rewritten)) + .map {|*_, path| path}.uniq + if changed_paths.empty? + puts "Skip commit only for tools or toplevel" + return false + end + + # Build commit objects from "cacheinfo" + new_parent = make_and_fixup_commit(gem, first_parent, old_rewritten) + new_commit = make_and_fixup_commit(gem, sha, new_rewritten, parent: new_parent, message: message, author: author) + puts "Created a temporary commit for cherry-pick: #{new_commit}" + new_commit + end + + def pickup_commit(gem, sha, edit) + rewritten = rewrite_commit(gem, sha) - IO.popen(%W[git filter-branch -f --msg-filter #{[filter, repo, sha].join(' ')} -- HEAD~1..HEAD], &:read) + # No changes remaining after rewriting + return false unless rewritten + + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{rewritten}", "rb", err: [:child, :out], &:read) unless $?.success? - puts "Failed to modify commit message of #{sha}" - break + if result =~ /The previous cherry-pick is now empty/ + system(*%w"git cherry-pick --skip") + puts "Skip empty commit #{sha}" + return false + end + + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless resolve_conflicts(gem, sha, edit) + system(*%w"git --no-pager diff") if !edit # If failed, show `git diff` unless editing + `git reset` && `git checkout .` && `git clean -fd` # Clean up un-committed diffs + return nil # Fail unless cherry-picked + end + + # Commit cherry-picked commit + if porcelain_status().empty? + system(*%w"git cherry-pick --skip") + return false + else + system(*%w"git cherry-pick --continue --no-edit") + return nil unless $?.success? + end end - end - unless failed_commits.empty? - puts "---- failed commits ----" - puts failed_commits - return false + new_head = IO.popen(%W"git rev-parse HEAD", "rb", &:read).chomp + puts "Committed cherry-pick as #{new_head}" + return true end - return true -end -def sync_lib(repo, upstream = nil) - unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") - abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] + # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. + # @param ranges [Array<String>, true] "commit", "before..after", or true. Note that it will NOT sync "before" (but commits after that). + # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. + def sync_default_gems_with_commits(gem, ranges, edit: nil) + config = REPOSITORIES[gem] + repo, default_branch = config.upstream, config.branch + puts "Sync #{repo} with commit history." + + # Fetch the repository to be synchronized + IO.popen(%W"git remote") do |f| + unless f.read.split.include?(gem) + `git remote add #{gem} https://github.com/#{repo}.git` + end + end + system(*%W"git fetch --no-tags --depth=#{FETCH_DEPTH} #{gem} #{default_branch}") + + # If -a is given, discover all commits since the last picked commit + if ranges == true + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" + log = log_format('%B', %W"-E --grep=#{pattern} -n1 --", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] + end + commits = commits_in_ranges(ranges) + if commits.empty? + puts "No commits to pick" + return true + end + + failed_commits = [] + commits.each do |sha, subject| + puts "----" + puts "Pick #{sha} #{subject}" + case pickup_commit(gem, sha, edit) + when false + # skipped + when nil + failed_commits << [sha, subject] + end + end + + unless failed_commits.empty? + puts "---- failed commits ----" + failed_commits.each do |sha, subject| + puts "#{sha} #{subject}" + end + return false + end + return true end - rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) - cp_r(Dir.glob("#{upstream}/lib/*"), "lib") - tests = if File.directory?("test/#{repo}") - "test/#{repo}" - else - "test/test_#{repo}.rb" - end - cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") - gemspec = if File.directory?("lib/#{repo}") - "lib/#{repo}/#{repo}.gemspec" + + def sync_lib(repo, upstream = nil) + unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") + abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] + end + rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) + cp_r(Dir.glob("#{upstream}/lib/*"), "lib") + tests = if File.directory?("test/#{repo}") + "test/#{repo}" else - "lib/#{repo}.gemspec" + "test/test_#{repo}.rb" end - cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") -end + cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") + gemspec = if File.directory?("lib/#{repo}") + "lib/#{repo}/#{repo}.gemspec" + else + "lib/#{repo}.gemspec" + end + cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") + end -def update_default_gems(gem) + def update_default_gems(gem, release: false) + config = REPOSITORIES[gem] + author, repository = config.upstream.split('/') + default_branch = config.branch - author, repository = REPOSITORIES[gem.to_sym].split('/') + puts "Update #{author}/#{repository}" - puts "Update #{author}/#{repository}" + unless File.exist?("../../#{author}/#{repository}") + mkdir_p("../../#{author}") + `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` + end - unless File.exist?("../../#{author}/#{repository}") - mkdir_p("../../#{author}") - `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` + Dir.chdir("../../#{author}/#{repository}") do + unless `git remote`.match(/ruby\-core/) + `git remote add ruby-core git@github.com:ruby/ruby.git` + end + `git fetch ruby-core master --no-tags` + unless `git branch`.match(/ruby\-core/) + `git checkout ruby-core/master` + `git branch ruby-core` + end + `git checkout ruby-core` + `git rebase ruby-core/master` + `git fetch origin --tags` + + if release + last_release = `git tag | sort -V`.chomp.split.delete_if{|v| v =~ /pre|beta/ }.last + `git checkout #{last_release}` + else + `git checkout #{default_branch}` + `git rebase origin/#{default_branch}` + end + end end - Dir.chdir("../../#{author}/#{repository}") do - unless `git remote`.match(/ruby\-core/) - `git remote add ruby-core git@github.com:ruby/ruby.git` + case ARGV[0] + when "up" + if ARGV[1] + update_default_gems(ARGV[1]) + else + REPOSITORIES.each_key {|gem| update_default_gems(gem)} end - `git fetch ruby-core master --no-tags` - unless `git branch`.match(/ruby\-core/) - `git checkout ruby-core/master` - `git branch ruby-core` + when "all" + REPOSITORIES.each_key do |gem| + next if ["Onigmo"].include?(gem) + update_default_gems(gem, release: true) if ARGV[1] == "release" + sync_default_gems(gem) end - `git checkout ruby-core` - `git rebase ruby-core/master` - `git checkout master` - `git fetch origin master` - `git rebase origin/master` - end -end - -case ARGV[0] -when "up" - if ARGV[1] - update_default_gems(ARGV[1]) - else - REPOSITORIES.keys.each{|gem| update_default_gems(gem.to_s)} - end -when "all" - REPOSITORIES.keys.each{|gem| sync_default_gems(gem.to_s)} -when "list" - ARGV.shift - pattern = Regexp.new(ARGV.join('|')) - REPOSITORIES.each_pair do |name, gem| - next unless pattern =~ name or pattern =~ gem - printf "%-15s https://github.com/%s\n", name, gem - end -when "--message-filter" - ARGV.shift - abort unless ARGV.size == 2 - message_filter(*ARGV) - exit -when nil, "-h", "--help" + when "list" + ARGV.shift + pattern = Regexp.new(ARGV.join('|')) + REPOSITORIES.each do |gem, config| + next unless pattern =~ gem or pattern =~ config.upstream + printf "%-15s https://github.com/%s\n", gem, config.upstream + end + when "rdoc-ref" + ARGV.shift + pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV + result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + pattern) + result.inject(false) do |changed, file| + if replace_rdoc_ref(file) + puts "replaced rdoc-ref in #{file}" + changed = true + end + changed + end + when nil, "-h", "--help" puts <<-HELP \e[1mSync with upstream code of default libraries\e[0m -\e[1mImport a default library through `git clone` and `cp -rf` (git commits are lost)\e[0m +\e[1mImport all default gems through `git clone` and `cp -rf` (git commits are lost)\e[0m + ruby #$0 all + +\e[1mImport all released version of default gems\e[0m + ruby #$0 all release + +\e[1mImport a default gem with specific gem same as all command\e[0m ruby #$0 rubygems \e[1mPick a single commit from the upstream repository\e[0m @@ -555,6 +903,12 @@ when nil, "-h", "--help" \e[1mPick a commit range from the upstream repository\e[0m ruby #$0 rubygems 97e9768612..9e53702832 +\e[1mPick all commits since the last picked commit\e[0m + ruby #$0 -a rubygems + +\e[1mUpdate repositories of default gems\e[0m + ruby #$0 up + \e[1mList known libraries\e[0m ruby #$0 list @@ -562,27 +916,28 @@ when nil, "-h", "--help" ruby #$0 list read HELP - exit -else - while /\A-/ =~ ARGV[0] - case ARGV[0] - when "-e" - edit = true - ARGV.shift - when "-a" - auto = true - ARGV.shift + exit + else + while /\A-/ =~ ARGV[0] + case ARGV[0] + when "-e" + edit = true + ARGV.shift + when "-a" + auto = true + ARGV.shift + else + $stderr.puts "Unknown command line option: #{ARGV[0]}" + exit 1 + end + end + gem = ARGV.shift + if ARGV[0] + exit sync_default_gems_with_commits(gem, ARGV, edit: edit) + elsif auto + exit sync_default_gems_with_commits(gem, true, edit: edit) else - $stderr.puts "Unknown command line option: #{ARGV[0]}" - exit 1 + sync_default_gems(gem) end - end - gem = ARGV.shift - if ARGV[0] - exit sync_default_gems_with_commits(gem, ARGV, edit: edit) - elsif auto - exit sync_default_gems_with_commits(gem, true, edit: edit) - else - sync_default_gems(gem) - end + end if $0 == __FILE__ end diff --git a/tool/test-annocheck.sh b/tool/test-annocheck.sh new file mode 100755 index 0000000000..6896869e07 --- /dev/null +++ b/tool/test-annocheck.sh @@ -0,0 +1,40 @@ +#!/bin/sh -eu +# Run the `tool/test-annocheck.sh [binary files]` to check security issues +# by annocheck <https://sourceware.org/annobin/>. +# +# E.g. `tool/test-annocheck.sh ruby libruby.so.3.2.0`. +# +# Note that as the annocheck binary package is not available on Ubuntu, and it +# is working in progress in Debian, this script uses Fedora container for now. +# It requires docker or podman. +# https://www.debian.org/devel/wnpp/itp.en.html +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=926470 + +set -x + +DOCKER="$(command -v docker || command -v podman)" +TAG=ruby-fedora-annocheck +TOOL_DIR="$(dirname "${0}")" +TMP_DIR="tmp/annocheck" +DOCKER_RUN_VOLUME_OPTS= + +if [ -z "${CI-}" ]; then + # Use a volume option on local (non-CI). + DOCKER_RUN_VOLUME_OPTS="-v $(pwd):/work" + "${DOCKER}" build --rm -t "${TAG}" ${TOOL_DIR}/annocheck/ +else + # TODO: A temporary workaround on CI to build by copying binary files from + # host to container without volume option, as I couldn't find a way to use + # volume in container in container on GitHub Actions + # <.github/workflows/compilers.yml>. + TAG="${TAG}-copy" + rm -rf "${TMP_DIR}" + mkdir -p "${TMP_DIR}" + for file in "${@}"; do + cp -p "${file}" "${TMP_DIR}" + done + "${DOCKER}" build --rm -t "${TAG}" --build-arg=IN_DIR="${TMP_DIR}" -f ${TOOL_DIR}/annocheck/Dockerfile-copy . + rm -rf "${TMP_DIR}" +fi + +"${DOCKER}" run --rm -t ${DOCKER_RUN_VOLUME_OPTS} "${TAG}" annocheck --verbose ${TEST_ANNOCHECK_OPTS-} "${@}" diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb index 6419ffaa50..778fe3311a 100644 --- a/tool/test-bundled-gems.rb +++ b/tool/test-bundled-gems.rb @@ -1,69 +1,139 @@ require 'rbconfig' require 'timeout' +require 'fileutils' +require 'shellwords' +require_relative 'lib/colorize' +require_relative 'lib/gem_env' + +ENV.delete("GNUMAKEFLAGS") github_actions = ENV["GITHUB_ACTIONS"] == "true" +DEFAULT_ALLOWED_FAILURES = RUBY_PLATFORM =~ /mswin|mingw/ ? [ + 'debug', + 'irb', + 'csv', +] : [] allowed_failures = ENV['TEST_BUNDLED_GEMS_ALLOW_FAILURES'] || '' -allowed_failures = allowed_failures.split(',').reject(&:empty?) +allowed_failures = allowed_failures.split(',').concat(DEFAULT_ALLOWED_FAILURES).uniq.reject(&:empty?) + +# make test-bundled-gems BUNDLED_GEMS=gem1,gem2,gem3 +bundled_gems = nil if (bundled_gems = ARGV.first&.split(","))&.empty? +colorize = Colorize.new rake = File.realpath("../../.bundle/bin/rake", __FILE__) gem_dir = File.realpath('../../gems', __FILE__) +rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +run_opts = ENV["RUN_OPTS"]&.shellsplit exit_code = 0 ruby = ENV['RUBY'] || RbConfig.ruby failed = [] File.foreach("#{gem_dir}/bundled_gems") do |line| - next if /^\s*(?:#|$)/ =~ line - gem = line.split.first - next if ARGV.any? {|pat| !File.fnmatch?(pat, gem)} - puts "#{github_actions ? "##[group]" : "\n"}Testing the #{gem} gem" + next unless gem = line[/^[^\s\#]+/] + next if bundled_gems&.none? {|pat| File.fnmatch?(pat, gem)} + next unless File.directory?("#{gem_dir}/src/#{gem}/test") - test_command = "#{ruby} -C #{gem_dir}/src/#{gem} -Ilib #{rake} test" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", rake, "test"] first_timeout = 600 # 10min - if gem == "typeprof" - raise "need to run rbs test suite before typeprof" unless File.readable?("#{gem_dir}/src/rbs/lib/rbs/parser.rb") - ENV["RUBYLIB"] = ["#{gem_dir}/src/rbs/lib", ENV.fetch("RUBYLIB", nil)].compact.join(":") + toplib = gem + unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + toplib = gem.tr("-", "/") + next unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") end - if gem == "rbs" - racc = File.realpath("../../libexec/racc", __FILE__) - pid = Process.spawn("#{ruby} -C #{gem_dir}/src/#{gem} -Ilib #{racc} -v -o lib/rbs/parser.rb lib/rbs/parser.y") - Process.waitpid(pid) - test_command << " stdlib_test validate" + case gem + when "rbs" + # TODO: We should skip test file instead of test class/methods + skip_test_files = %w[ + ] + + skip_test_files.each do |file| + path = "#{gem_dir}/src/#{gem}/#{file}" + File.unlink(path) if File.exist?(path) + end + + rbs_skip_tests = [ + File.join(__dir__, "/rbs_skip_tests") + ] + + if /mswin|mingw/ =~ RUBY_PLATFORM + rbs_skip_tests << File.join(__dir__, "/rbs_skip_tests_windows") + end + test_command.concat %W[stdlib_test validate RBS_SKIP_TESTS=#{rbs_skip_tests.join(File::PATH_SEPARATOR)} SKIP_RBS_VALIDATION=true] first_timeout *= 3 + + when "debug" + # Since debug gem requires debug.so in child processes without + # activating the gem, we preset necessary paths in RUBYLIB + # environment variable. + load_path = true + + when "test-unit" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", "test/run.rb"] + + when "csv" + first_timeout = 30 + + when "win32ole" + next unless /mswin|mingw/ =~ RUBY_PLATFORM + end - if gem == "minitest" - # Tentatively exclude some tests that conflict with error_highlight - # https://github.com/seattlerb/minitest/pull/880 - test_command << " 'TESTOPTS=-e /test_stub_value_block_args_5__break_if_not_passed|test_no_method_error_on_unexpected_methods/'" + if load_path + libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) + next unless $?.success? + ENV["RUBYLIB"] = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) + else + ENV["RUBYLIB"] = rubylib end + # 93(bright yellow) is copied from .github/workflows/mingw.yml + puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" print "[command]" if github_actions - puts test_command - pid = Process.spawn(test_command, "#{/mingw|mswin/ =~ RUBY_PLATFORM ? 'new_' : ''}pgroup": true) - {nil => first_timeout, INT: 30, TERM: 10, KILL: nil}.each do |sig, sec| + p test_command + timeouts = {nil => first_timeout, INT: 30, TERM: 10, KILL: nil} + if /mingw|mswin/ =~ RUBY_PLATFORM + timeouts.delete(:TERM) # Inner process signal on Windows + group = :new_pgroup + pg = "" + else + group = :pgroup + pg = "-" + end + pid = Process.spawn(*test_command, group => true) + timeouts.each do |sig, sec| if sig puts "Sending #{sig} signal" - Process.kill("-#{sig}", pid) + Process.kill("#{pg}#{sig}", pid) end begin break Timeout.timeout(sec) {Process.wait(pid)} rescue Timeout::Error end + rescue Interrupt + exit_code = Signal.list["INT"] + Process.kill("#{pg}KILL", pid) + Process.wait(pid) + break end + print "::endgroup::\n" if github_actions unless $?.success? - puts "Tests failed with exit code #{$?.exitstatus}" + + mesg = "Tests failed " + + ($?.signaled? ? "by SIG#{Signal.signame($?.termsig)}" : + "with exit code #{$?.exitstatus}") + puts colorize.decorate(mesg, "fail") if allowed_failures.include?(gem) - puts "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES" + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" + puts colorize.decorate(mesg, "skip") else failed << gem - exit_code = $?.exitstatus + exit_code = 1 end end - print "##[endgroup]\n" if github_actions end puts "Failed gems: #{failed.join(', ')}" unless failed.empty? diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb index 4950bc65de..28ef0bf7f8 100644 --- a/tool/test-coverage.rb +++ b/tool/test-coverage.rb @@ -4,6 +4,16 @@ Coverage.start(lines: true, branches: true, methods: true) TEST_COVERAGE_DATA_FILE = "test-coverage.dat" +FILTER_PATHS = %w[ + lib/bundler/vendor + lib/rubygems/resolver/molinillo + lib/rubygems/tsort + lib/rubygems/optparse + tool + test + spec +] + def merge_coverage_data(res1, res2) res1.each do |path, cov1| cov2 = res2[path] @@ -62,8 +72,11 @@ def save_coverage_data(res1) end def invoke_simplecov_formatter - %w[doclie simplecov-html simplecov].each do |f| - $LOAD_PATH.unshift "#{__dir__}/../coverage/#{f}/lib" + # XXX docile-x.y.z and simplecov-x.y.z, simplecov-html-x.y.z, simplecov_json_formatter-x.y.z + %w[simplecov simplecov-html simplecov_json_formatter docile].each do |f| + Dir.glob("#{__dir__}/../.bundle/gems/#{f}-*/lib").each do |d| + $LOAD_PATH.unshift d + end end require "simplecov" @@ -74,8 +87,8 @@ def invoke_simplecov_formatter res.each do |path, cov| next unless path.start_with?(base_dir) || path.start_with?(cur_dir) - next if path.start_with?(File.join(base_dir, "test")) - simplecov_result[path] = cov[:lines] + next if FILTER_PATHS.any? {|dir| path.start_with?(File.join(base_dir, dir))} + simplecov_result[path] = cov end a, b = base_dir, cur_dir @@ -101,6 +114,10 @@ pid = $$ pwd = Dir.pwd at_exit do + # Some tests leave GC.stress enabled, causing slow coverage processing. + # Reset it here to avoid performance issues. + GC.stress = false + exit_exc = $! Dir.chdir(pwd) do diff --git a/tool/test/init.rb b/tool/test/init.rb new file mode 100644 index 0000000000..3fd1419a9c --- /dev/null +++ b/tool/test/init.rb @@ -0,0 +1,26 @@ +# This file includes the settings for "make test-all" and "make test-tool". +# Note that this file is loaded not only by test/runner.rb but also by tool/lib/test/unit/parallel.rb. + +# Prevent test-all from using bundled gems +["GEM_HOME", "GEM_PATH"].each do |gem_env| + # Preserve the gem environment prepared by tool/runruby.rb for test-tool, which uses bundled gems. + ENV["BUNDLED_#{gem_env}"] = ENV[gem_env] + + ENV[gem_env] = "".freeze +end +ENV["GEM_SKIP"] = "".freeze + +ENV.delete("RUBY_CODESIGN") + +Warning[:experimental] = false + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) + +require 'test/unit' + +require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') +require "tracepointchecker" +require "zombie_hunter" +require "iseq_loader_checker" +require "gc_checker" +require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') diff --git a/tool/test/runner.rb b/tool/test/runner.rb index c629943090..9001fc2d06 100644 --- a/tool/test/runner.rb +++ b/tool/test/runner.rb @@ -1,16 +1,7 @@ # frozen_string_literal: true require 'rbconfig' -$LOAD_PATH.unshift File.expand_path("../lib", __dir__) - -require 'test/unit' - -require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') -require "tracepointchecker" -require "zombie_hunter" -require "iseq_loader_checker" -require "gc_checker" -require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') +require_relative "init" case $0 when __FILE__ @@ -18,6 +9,6 @@ when __FILE__ when "-e" # No default directory else - dir = File.expand_path("..", $0) + dir = File.realdirpath("..", $0) end exit Test::Unit::AutoRunner.run(true, dir) diff --git a/tool/test/test_commit_email.rb b/tool/test/test_commit_email.rb new file mode 100644 index 0000000000..db441584fd --- /dev/null +++ b/tool/test/test_commit_email.rb @@ -0,0 +1,102 @@ +require 'test/unit' +require 'shellwords' +require 'tmpdir' +require 'fileutils' +require 'open3' + +class TestCommitEmail < Test::Unit::TestCase + STDIN_DELIMITER = "---\n" + + def setup + omit 'git command is not available' unless system('git', '--version', out: File::NULL, err: File::NULL) + + @ruby = Dir.mktmpdir + Dir.chdir(@ruby) do + git('init', '--initial-branch=master') + git('config', 'user.name', 'Jóhän Grübél') + git('config', 'user.email', 'johan@example.com') + env = { + 'GIT_AUTHOR_DATE' => '2025-10-08T12:00:00Z', + 'GIT_CONFIG_GLOBAL' => @ruby + "/gitconfig", + 'TZ' => 'UTC', + } + git('commit', '--allow-empty', '-m', 'New repository initialized by cvs2svn.', env:) + git('commit', '--allow-empty', '-m', 'Initial revision', env:) + git('commit', '--allow-empty', '-m', 'version 1.0.0', env:) + end + + @sendmail = File.join(Dir.mktmpdir, 'sendmail') + File.write(@sendmail, <<~SENDMAIL, mode: "wx", perm: 0755) + #!/bin/sh + echo #{STDIN_DELIMITER.chomp.dump} + exec cat + SENDMAIL + + @commit_email = File.expand_path('../../tool/commit-email.rb', __dir__) + end + + def teardown + # Clean up temporary files if #setup was not omitted + if @sendmail + File.unlink(@sendmail) + Dir.rmdir(File.dirname(@sendmail)) + end + if @ruby + FileUtils.rm_rf(@ruby) + end + end + + def test_sendmail_encoding + omit 'the sendmail script does not work on windows' if windows? + + Dir.chdir(@ruby) do + before_rev = git('rev-parse', 'HEAD^').chomp + after_rev = git('rev-parse', 'HEAD').chomp + short_rev = after_rev[0...10] + + out, _, status = EnvUtil.invoke_ruby([ + { 'SENDMAIL' => @sendmail, 'TZ' => 'UTC' }.merge!(gem_env), + @commit_email, './', 'cvs-admin@ruby-lang.org', + before_rev, after_rev, 'refs/heads/master', + '--viewer-uri', 'https://github.com/ruby/ruby/commit/', + '--error-to', 'cvs-admin@ruby-lang.org', + ], '', true) + stdin = out.b.split(STDIN_DELIMITER.b, 2).last.force_encoding('UTF-8') + + assert_true(status.success?) + assert_equal(stdin, <<~EOS) + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: =?UTF-8?B?SsOzaMOkbiBHcsO8YsOpbA==?= <noreply@ruby-lang.org> + To: cvs-admin@ruby-lang.org + Subject: #{short_rev} (master): =?UTF-8?B?dmVyc2lvbuOAgDEuMC4w?= + J=C3=B3h=C3=A4n Gr=C3=BCb=C3=A9l\t2025-10-08 12:00:00 +0000 (Wed, 08 Oct 2= + 025) + + New Revision: #{short_rev} + + https://github.com/ruby/ruby/commit/#{short_rev} + + Log: + version=E3=80=801.0.0= + EOS + end + end + + private + + # Resurrect the gem environment preserved by tool/test/init.rb. + # This should work as long as you have run `make up` or `make install`. + def gem_env + { 'GEM_PATH' => ENV['BUNDLED_GEM_PATH'], 'GEM_HOME' => ENV['BUNDLED_GEM_HOME'] } + end + + def git(*cmd, env: {}) + out, status = Open3.capture2(env, 'git', *cmd) + unless status.success? + raise "git #{cmd.shelljoin}\n#{out}" + end + out + end +end diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb new file mode 100755 index 0000000000..252687f3f3 --- /dev/null +++ b/tool/test/test_sync_default_gems.rb @@ -0,0 +1,373 @@ +#!/usr/bin/ruby +require 'test/unit' +require 'stringio' +require 'tmpdir' +require 'rubygems/version' +require_relative '../sync_default_gems' + +module Test_SyncDefaultGems + class TestMessageFilter < Test::Unit::TestCase + def assert_message_filter(expected, trailers, input, repo = "ruby/test", sha = "0123456789") + subject, *expected = expected + expected = [ + "[#{repo}] #{subject}\n", + *expected.map {_1+"\n"}, + "\n", + "https://github.com/#{repo}/commit/#{sha[0, 10]}\n", + ] + if trailers + expected << "\n" + expected.concat(trailers.map {_1+"\n"}) + end + + out = SyncDefaultGems.message_filter(repo, sha, input) + assert_pattern_list(expected, out) + end + + def test_subject_only + expected = [ + "initial commit", + ] + assert_message_filter(expected, nil, "initial commit") + end + + def test_link_in_parenthesis + expected = [ + "fix (https://github.com/ruby/test/pull/1)", + ] + assert_message_filter(expected, nil, "fix (#1)") + end + + def test_co_authored_by + expected = [ + "commit something", + ] + trailers = [ + "Co-Authored-By: git <git@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers, ""].join("\n")) + end + + def test_multiple_co_authored_by + expected = [ + "many commits", + ] + trailers = [ + "Co-authored-by: git <git@ruby-lang.org>", + "Co-authored-by: svn <svn@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers, ""].join("\n")) + end + + def test_co_authored_by_no_newline + expected = [ + "commit something", + ] + trailers = [ + "Co-Authored-By: git <git@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers].join("\n")) + end + + def test_dot_ending_subject + expected = [ + "subject with a dot.", + "", + "- next body line", + ] + assert_message_filter(expected, nil, [expected[0], expected[2], ""].join("\n")) + end + end + + class TestSyncWithCommits < Test::Unit::TestCase + def setup + super + @target = nil + pend "No git" unless system("git --version", out: IO::NULL) + @testdir = Dir.mktmpdir("sync") + user, email = "Ruby", "test@ruby-lang.org" + @git_config = %W"HOME USER GIT_CONFIG_GLOBAL GNUPGHOME".each_with_object({}) {|k, c| c[k] = ENV[k]} + ENV["HOME"] = @testdir + ENV["USER"] = user + ENV["GNUPGHOME"] = @testdir + '/.gnupg' + expire = EnvUtil.apply_timeout_scale(30).to_i + # Generate a new unprotected key with default parameters that + # expires after 30 seconds. + if @gpgsign = system(*%w"gpg --quiet --batch --passphrase", "", + "--quick-generate-key", email, *%W"default default seconds=#{expire}", + err: IO::NULL) + # Fetch the generated public key. + signingkey = IO.popen(%W"gpg --quiet --list-public-key #{email}", &:read)[/^pub .*\n +\K\h+/] + end + ENV["GIT_CONFIG_GLOBAL"] = @testdir + "/gitconfig" + git(*%W"config --global user.email", email) + git(*%W"config --global user.name", user) + git(*%W"config --global init.defaultBranch default") + if signingkey + git(*%W"config --global user.signingkey", signingkey) + git(*%W"config --global commit.gpgsign true") + git(*%W"config --global gpg.program gpg") + git(*%W"config --global log.showSignature true") + end + @target = "sync-test" + SyncDefaultGems::REPOSITORIES[@target] = SyncDefaultGems.repo( + ["ruby/#{@target}", "default"], + [ + ["lib", "lib"], + ["test", "test"], + ], + exclude: [ + "test/fixtures/*", + ], + ) + @sha = {} + @origdir = Dir.pwd + Dir.chdir(@testdir) + ["src", @target].each do |dir| + git(*%W"init -q #{dir}") + File.write("#{dir}/.gitignore", "*~\n") + Dir.mkdir("#{dir}/lib") + File.write("#{dir}/lib/common.rb", ":ok\n") + Dir.mkdir("#{dir}/.github") + Dir.mkdir("#{dir}/.github/workflows") + File.write("#{dir}/.github/workflows/default.yml", "default:\n") + git(*%W"add .gitignore lib/common.rb .github", chdir: dir) + git(*%W"commit -q -m", "Initialize", chdir: dir) + if dir == "src" + File.write("#{dir}/lib/fine.rb", "return\n") + Dir.mkdir("#{dir}/test") + File.write("#{dir}/test/test_fine.rb", "return\n") + git(*%W"add lib/fine.rb test/test_fine.rb", chdir: dir) + git(*%W"commit -q -m", "Looks fine", chdir: dir) + end + Dir.mkdir("#{dir}/tool") + File.write("#{dir}/tool/ok", "#!/bin/sh\n""echo ok\n") + git(*%W"add tool/ok", chdir: dir) + git(*%W"commit -q -m", "Add tool #{dir}", chdir: dir) + @sha[dir] = top_commit(dir) + end + git(*%W"remote add #{@target} ../#{@target}", chdir: "src") + end + + def teardown + if @target + if @gpgsign + system(*%W"gpgconf --kill all") + end + Dir.chdir(@origdir) + SyncDefaultGems::REPOSITORIES.delete(@target) + ENV.update(@git_config) + FileUtils.rm_rf(@testdir) + end + super + end + + def capture_process_output_to(outputs) + return yield unless outputs&.empty? == false + IO.pipe do |r, w| + orig = outputs.map {|out| out.dup} + outputs.each {|out| out.reopen(w)} + w.close + reader = Thread.start {r.read} + yield + ensure + outputs.each {|out| o = orig.shift; out.reopen(o); o.close} + return reader.value + end + end + + def capture_process_outputs + out = err = nil + synchronize do + out = capture_process_output_to(STDOUT) do + err = capture_process_output_to(STDERR) do + yield + end + end + end + return out, err + end + + def git(*commands, **opts) + system("git", *commands, exception: true, **opts) + end + + def top_commit(dir, format: "%H") + IO.popen(%W[git log --no-show-signature --format=#{format} -1], chdir: dir, &:read)&.chomp + end + + def assert_sync(commits = true, success: true, editor: nil) + result = nil + out = capture_process_output_to([STDOUT, STDERR]) do + Dir.chdir("src") do + orig_editor = ENV["GIT_EDITOR"] + ENV["GIT_EDITOR"] = editor || 'false' + edit = true if editor + + result = SyncDefaultGems.sync_default_gems_with_commits(@target, commits, edit: edit) + ensure + ENV["GIT_EDITOR"] = orig_editor + end + end + assert_equal(success, result, out) + out + end + + def test_sync + File.write("#@target/lib/common.rb", "# OK!\n") + git(*%W"commit -q -m", "OK", "lib/common.rb", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# OK!\n", File.read("src/lib/common.rb")) + log = top_commit("src", format: "%B").lines + assert_equal("[ruby/#@target] OK\n", log.first, out) + assert_match(%r[/ruby/#{@target}/commit/\h+$], log.last, out) + assert_operator(top_commit(@target), :start_with?, log.last[/\h+$/], out) + end + + def test_skip_tool + git(*%W"rm -q tool/ok", chdir: @target) + git(*%W"commit -q -m", "Remove tool", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_test_fixtures + Dir.mkdir("#@target/test") + Dir.mkdir("#@target/test/fixtures") + File.write("#@target/test/fixtures/fixme.rb", "") + git(*%W"add test/fixtures/fixme.rb", chdir: @target) + git(*%W"commit -q -m", "Add fixtures", chdir: @target) + out = assert_sync(["#{@sha[@target]}..#{@target}/default"]) + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= NEWS!!!\n") + git(*%W"add --", "docs/NEWS.md", chdir: @target) + File.write("#@target/docs/hello.md", "Hello\n") + git(*%W"add --", "docs/hello.md", chdir: @target) + git(*%W"commit -q -m", "It's a news", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_adding_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= New library\n") + File.write("#@target/lib/news.rb", "return\n") + git(*%W"add --", "docs/NEWS.md", "lib/news.rb", chdir: @target) + git(*%W"commit -q -m", "New lib", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal "return\n", File.read("src/lib/news.rb") + assert_include top_commit("src", format: "oneline"), "[ruby/#{@target}] New lib" + assert_not_operator File, :exist?, "src/docs" + end + + def test_gitignore + File.write("#@target/.gitignore", "*.bak\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add .github", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", + *%W".gitignore lib/common.rb .github", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("*~\n", File.read("src/.gitignore"), out) + assert_equal("#!/bin/sh\n""echo ok\n", File.read("src/tool/ok"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_gitignore_after_conflict + File.write("src/Gemfile", "# main\n") + git(*%W"add Gemfile", chdir: "src") + git(*%W"commit -q -m", "Add Gemfile", chdir: "src") + File.write("#@target/Gemfile", "# conflict\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add Gemfile .github lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# main\n", File.read("src/Gemfile"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_delete_after_conflict + File.write("#@target/lib/bad.rb", "raise\n") + git(*%W"add lib/bad.rb", chdir: @target) + git(*%W"commit -q -m", "Add bad.rb", chdir: @target) + out = assert_sync + assert_equal("raise\n", File.read("src/lib/bad.rb")) + + git(*%W"rm lib/bad.rb", chdir: "src", out: IO::NULL) + git(*%W"commit -q -m", "Remove bad.rb", chdir: "src") + + File.write("#@target/lib/bad.rb", "raise 'bar'\n") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + git(*%W"add lib/bad.rb lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict", chdir: @target) + + head = top_commit("src") + out = assert_sync(editor: "git rm -f lib/bad.rb") + assert_not_equal(head, top_commit("src")) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/lib/bad.rb", out) + end + + def test_squash_merge + # This test is known to fail with git 2.43.0, which is used by Ubuntu 24.04. + # We don't know which exact version fixed it, but we know git 2.52.0 works. + stdout, status = Open3.capture2('git', '--version', err: File::NULL) + omit 'git version check failed' unless status.success? + git_version = stdout[/\Agit version \K\S+/] + omit "git #{git_version} is too old" if Gem::Version.new(git_version) < Gem::Version.new('2.44.0') + + # 2---. <- branch + # / \ + # 1---3---3'<- merge commit with conflict resolution + File.write("#@target/lib/conflict.rb", "# 1\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict.rb", chdir: @target) + + git(*%W"checkout -q -b branch", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 2\n") + File.write("#@target/lib/new.rb", "# new\n") + git(*%W"add lib/conflict.rb lib/new.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in branch", chdir: @target) + + git(*%W"checkout -q default", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 3\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in default", chdir: @target) + + # How can I suppress "Auto-merging ..." message from git merge? + git(*%W"merge -X ours -m", "Merge commit", "branch", chdir: @target, out: IO::NULL) + + out = assert_sync() + assert_equal("# 3\n", File.read("src/lib/conflict.rb"), out) + subject, body = top_commit("src", format: "%B").split("\n\n", 2) + assert_equal("[ruby/#@target] Merge commit", subject, out) + assert_includes(body, "Commit in branch", out) + end + + def test_no_upstream_file + group = SyncDefaultGems::Repository.group(%w[ + lib/un.rb + lib/unicode_normalize/normalize.rb + lib/unicode_normalize/tables.rb + lib/net/https.rb + ]) + expected = { + "un" => %w[lib/un.rb], + "net-http" => %w[lib/net/https.rb], + nil => %w[lib/unicode_normalize/normalize.rb lib/unicode_normalize/tables.rb], + } + assert_equal(expected, group) + end + end if /darwin|linux/ =~ RUBY_PLATFORM +end diff --git a/tool/test/testunit/test4test_hideskip.rb b/tool/test/testunit/test4test_hideskip.rb index 410bffc13c..14f79a5743 100644 --- a/tool/test/testunit/test4test_hideskip.rb +++ b/tool/test/testunit/test4test_hideskip.rb @@ -4,7 +4,11 @@ $LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" require 'test/unit' class TestForTestHideSkip < Test::Unit::TestCase - def test_skip - skip "do nothing" + def test_omit + omit "do nothing" + end + + def test_pend + pend "do nothing" end end diff --git a/tool/test/testunit/test4test_load_failure.rb b/tool/test/testunit/test4test_load_failure.rb new file mode 100644 index 0000000000..e1570c2542 --- /dev/null +++ b/tool/test/testunit/test4test_load_failure.rb @@ -0,0 +1 @@ +raise LoadError, "no-such-library" diff --git a/tool/test/testunit/test4test_sorting.rb b/tool/test/testunit/test4test_sorting.rb index 698c875b79..f5a6866425 100644 --- a/tool/test/testunit/test4test_sorting.rb +++ b/tool/test/testunit/test4test_sorting.rb @@ -5,7 +5,7 @@ require 'test/unit' class TestForTestHideSkip < Test::Unit::TestCase def test_c - skip "do nothing" + omit "do nothing" end def test_b diff --git a/tool/test/testunit/test4test_timeout.rb b/tool/test/testunit/test4test_timeout.rb new file mode 100644 index 0000000000..3225f66398 --- /dev/null +++ b/tool/test/testunit/test4test_timeout.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' +require 'timeout' + +class TestForTestTimeout < Test::Unit::TestCase + 10.times do |i| + define_method("test_timeout_#{i}") do + Timeout.timeout(0.001) do + sleep + end + end + end +end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb index 8c83b447a7..d9bdc8f3c5 100644 --- a/tool/test/testunit/test_assertion.rb +++ b/tool/test/testunit/test_assertion.rb @@ -8,6 +8,8 @@ class TestAssertion < Test::Unit::TestCase end def test_timeout_separately + pend "hang-up" if /mswin|mingw/ =~ RUBY_PLATFORM + assert_raise(Timeout::Error) do assert_separately([], <<~"end;", timeout: 0.1) sleep @@ -15,6 +17,29 @@ class TestAssertion < Test::Unit::TestCase end end + def test_assertion_count_separately + beginning = self._assertions + + assert_separately([], "") + assertions_at_nothing = self._assertions - beginning + + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "assert true") + assert_equal(1, self._assertions - prev_assertions) + + omit unless Process.respond_to?(:fork) + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "Process.fork {assert true}; assert true") + assert_equal(2, self._assertions - prev_assertions) + + prev_assertions = self._assertions + assertions_at_nothing + # TODO: assertions before `fork` are counted twice; it is possible + # to reset `_assertions` at `Process._fork`, but the hook can + # interfere in other tests. + assert_separately([], "assert true; Process.fork {assert true}") + assert_equal(3, self._assertions - prev_assertions) + end + def return_in_assert_raise assert_raise(RuntimeError) do return @@ -26,4 +51,178 @@ class TestAssertion < Test::Unit::TestCase return_in_assert_raise end end + + def test_assert_raise_with_message + my_error = Class.new(StandardError) + + assert_raise_with_message(my_error, "with message") do + raise my_error, "with message" + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(RuntimeError, "with message") do + raise my_error, "with message" + end + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(my_error, "without message") do + raise my_error, "with message" + end + end + end + + def test_assert_raise_kind_of + my_error = Class.new(StandardError) + + assert_raise_kind_of(my_error) do + raise my_error + end + + assert_raise_kind_of(StandardError) do + raise my_error + end + end + + def test_assert_pattern_list + assert_pattern_list([/foo?/], "foo") + assert_not_pattern_list([/foo?/], "afoo") + assert_not_pattern_list([/foo?/], "foo?") + assert_pattern_list([:*, /foo?/, :*], "foo") + assert_pattern_list([:*, /foo?/], "afoo") + assert_not_pattern_list([:*, /foo?/], "afoo?") + assert_pattern_list([/foo?/, :*], "foo?") + + assert_not_pattern_list(["foo?"], "foo") + assert_not_pattern_list(["foo?"], "afoo") + assert_pattern_list(["foo?"], "foo?") + assert_not_pattern_list([:*, "foo?", :*], "foo") + assert_not_pattern_list([:*, "foo?"], "afoo") + assert_pattern_list([:*, "foo?"], "afoo?") + assert_pattern_list(["foo?", :*], "foo?") + end + + def assert_not_pattern_list(pattern_list, actual, message=nil) + assert_raise(Test::Unit::AssertionFailedError) do + assert_pattern_list(pattern_list, actual, message) + end + end + + def test_caller_bactrace_location + begin + line = __LINE__; assert_fail_for_backtrace_location + rescue Test::Unit::AssertionFailedError => e + end + location = Test::Unit::Runner.new.location(e) + assert_equal "#{__FILE__}:#{line}", location + end + + def assert_fail_for_backtrace_location + assert false + end + + VersionClass = Struct.new(:version) do + def version?(*ver) + Test::Unit::CoreAssertions.version_match?(ver, self.version) + end + end + + V14_6_0 = VersionClass.new([14, 6, 0]) + V15_0_0 = VersionClass.new([15, 0, 0]) + + def test_version_match_integer + assert_not_operator(V14_6_0, :version?, 13) + assert_operator(V14_6_0, :version?, 14) + assert_not_operator(V14_6_0, :version?, 15) + assert_not_operator(V15_0_0, :version?, 14) + assert_operator(V15_0_0, :version?, 15) + end + + def test_version_match_integer_range + assert_operator(V14_6_0, :version?, 13..14) + assert_not_operator(V15_0_0, :version?, 13..14) + assert_not_operator(V14_6_0, :version?, 13...14) + assert_not_operator(V15_0_0, :version?, 13...14) + end + + def test_version_match_array_range + assert_operator(V14_6_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V14_6_0, :version?, [14, 0]...[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[14, 6]) + assert_operator(V14_6_0, :version?, [14, 0]..[15]) + assert_operator(V15_0_0, :version?, [14, 0]..[15]) + assert_operator(V14_6_0, :version?, [14, 0]...[15]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[15]) + end + + def test_version_match_integer_endless_range + assert_operator(V14_6_0, :version?, 14..) + assert_operator(V15_0_0, :version?, 14..) + assert_not_operator(V14_6_0, :version?, 15..) + assert_operator(V15_0_0, :version?, 15..) + end + + def test_version_match_integer_endless_range_exclusive + assert_operator(V14_6_0, :version?, 14...) + assert_operator(V15_0_0, :version?, 14...) + assert_not_operator(V14_6_0, :version?, 15...) + assert_operator(V15_0_0, :version?, 15...) + end + + def test_version_match_array_endless_range + assert_operator(V14_6_0, :version?, [14, 5]..) + assert_operator(V15_0_0, :version?, [14, 5]..) + assert_not_operator(V14_6_0, :version?, [14, 7]..) + assert_operator(V15_0_0, :version?, [14, 7]..) + assert_not_operator(V14_6_0, :version?, [15]..) + assert_operator(V15_0_0, :version?, [15]..) + assert_not_operator(V14_6_0, :version?, [15, 0]..) + assert_operator(V15_0_0, :version?, [15, 0]..) + end + + def test_version_match_array_endless_range_exclude_end + assert_operator(V14_6_0, :version?, [14, 5]...) + assert_operator(V15_0_0, :version?, [14, 5]...) + assert_not_operator(V14_6_0, :version?, [14, 7]...) + assert_operator(V15_0_0, :version?, [14, 7]...) + assert_not_operator(V14_6_0, :version?, [15]...) + assert_operator(V15_0_0, :version?, [15]...) + assert_not_operator(V14_6_0, :version?, [15, 0]...) + assert_operator(V15_0_0, :version?, [15, 0]...) + end + + def test_version_match_integer_beginless_range + assert_operator(V14_6_0, :version?, ..14) + assert_not_operator(V15_0_0, :version?, ..14) + assert_operator(V14_6_0, :version?, ..15) + assert_operator(V15_0_0, :version?, ..15) + + assert_not_operator(V14_6_0, :version?, ...14) + assert_not_operator(V15_0_0, :version?, ...14) + assert_operator(V14_6_0, :version?, ...15) + assert_not_operator(V15_0_0, :version?, ...15) + end + + def test_version_match_array_beginless_range + assert_not_operator(V14_6_0, :version?, ..[14, 5]) + assert_not_operator(V15_0_0, :version?, ..[14, 5]) + assert_operator(V14_6_0, :version?, ..[14, 6]) + assert_not_operator(V15_0_0, :version?, ..[14, 6]) + assert_operator(V14_6_0, :version?, ..[15]) + assert_operator(V15_0_0, :version?, ..[15]) + assert_operator(V14_6_0, :version?, ..[15, 0]) + assert_operator(V15_0_0, :version?, ..[15, 0]) + end + + def test_version_match_array_beginless_range_exclude_end + assert_not_operator(V14_6_0, :version?, ...[14, 5]) + assert_not_operator(V15_0_0, :version?, ...[14, 5]) + assert_not_operator(V14_6_0, :version?, ...[14, 6]) + assert_not_operator(V15_0_0, :version?, ...[14, 6]) + assert_operator(V14_6_0, :version?, ...[15]) + assert_not_operator(V15_0_0, :version?, ...[15]) + assert_operator(V14_6_0, :version?, ...[15, 0]) + assert_not_operator(V15_0_0, :version?, ...[15, 0]) + end end diff --git a/tool/test/testunit/test_hideskip.rb b/tool/test/testunit/test_hideskip.rb index e90be5da32..a470368bca 100644 --- a/tool/test/testunit/test_hideskip.rb +++ b/tool/test/testunit/test_hideskip.rb @@ -4,16 +4,15 @@ require 'test/unit' class TestHideSkip < Test::Unit::TestCase def test_hideskip assert_not_match(/^ *1\) Skipped/, hideskip) - assert_match(/^ *1\) Skipped/, hideskip("--show-skip")) + assert_match(/^ *1\) Skipped.*^ *2\) Skipped/m, hideskip("--show-skip")) output = hideskip("--hide-skip") - output.gsub!(/Successful MJIT finish\n/, '') if defined?(RubyVM::JIT) && RubyVM::JIT.enabled? - assert_match(/assertions\/s.\n+1 tests, 0 assertions, 0 failures, 0 errors, 1 skips/, output) + assert_match(/assertions\/s.\n+2 tests, 0 assertions, 0 failures, 0 errors, 2 skips/, output) end private def hideskip(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", "--verbose", *args], err: [:child, :out]) {|f| f.read } diff --git a/tool/test/testunit/test_launchable.rb b/tool/test/testunit/test_launchable.rb new file mode 100644 index 0000000000..76be876456 --- /dev/null +++ b/tool/test/testunit/test_launchable.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: false +require 'test/unit' +require 'tempfile' +require 'json' +require_relative '../../lib/launchable' + +class TestLaunchable < Test::Unit::TestCase + def test_json_stream_writer + Tempfile.create(['launchable-test-', '.json']) do |f| + json_stream_writer = Launchable::JsonStreamWriter.new(f.path) + json_stream_writer.write_array('testCases') + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 42, + status: "TEST_FAILED", + stdout: nil, + stderr: nil, + createdAt: "2021-10-05T12:34:00", + data: { + lineNumber: 1 + } + } + ) + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 45, + status: "TEST_PASSED", + stdout: "This is stdout", + stderr: "This is stderr", + createdAt: "2021-10-05T12:36:00", + data: { + lineNumber: 10 + } + } + ) + json_stream_writer.close() + expected = <<JSON +{ + "testCases": [ + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 42, + "status": "TEST_FAILED", + "stdout": null, + "stderr": null, + "createdAt": "2021-10-05T12:34:00", + "data": { + "lineNumber": 1 + } + }, + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 45, + "status": "TEST_PASSED", + "stdout": "This is stdout", + "stderr": "This is stderr", + "createdAt": "2021-10-05T12:36:00", + "data": { + "lineNumber": 10 + } + } + ] +} +JSON + assert_equal(expected, f.read) + end + end +end diff --git a/tool/test/testunit/test_load_failure.rb b/tool/test/testunit/test_load_failure.rb new file mode 100644 index 0000000000..8defa9e39a --- /dev/null +++ b/tool/test/testunit/test_load_failure.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +require 'test/unit' + +class TestLoadFailure < Test::Unit::TestCase + def test_load_failure + assert_not_predicate(load_failure, :success?) + end + + def test_load_failure_parallel + assert_not_predicate(load_failure("-j2"), :success?) + end + + private + + def load_failure(*args) + IO.popen([*@__runner_options__[:ruby], "#{__dir__}/../runner.rb", + "#{__dir__}/test4test_load_failure.rb", + "--verbose", *args], err: [:child, :out]) {|f| + assert_include(f.read, "test4test_load_failure.rb") + } + $? + end +end diff --git a/tool/test/testunit/test_minitest_unit.rb b/tool/test/testunit/test_minitest_unit.rb index 68e88f574e..84b6cf688c 100644 --- a/tool/test/testunit/test_minitest_unit.rb +++ b/tool/test/testunit/test_minitest_unit.rb @@ -171,7 +171,7 @@ class TestMiniTestUnit < MetaMetaMetaTestCase def test_passed_eh_teardown_skipped test_class = Class.new Test::Unit::TestCase do def teardown; assert true; end - def test_omg; skip "bork"; end + def test_omg; omit "bork"; end end test = test_class.new :test_omg @@ -238,7 +238,7 @@ class TestMiniTestRunner < MetaMetaMetaTestCase tc = Class.new(Test::Unit::TestCase) assert_equal 2, Test::Unit::TestCase.test_suites.size - assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites + assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites.sort_by {|ts| ts.name.to_s} end def assert_filtering name, expected, a = false @@ -896,7 +896,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase begin raise "blah" rescue - skip "skipped" + omit "skipped" end end end @@ -1323,42 +1323,33 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase end end - def test_skip + def test_omit @assertion_count = 0 util_assert_triggered "haha!", Test::Unit::PendedError do - @tc.skip "haha!" + @tc.omit "haha!" end end - def test_test_methods_random + def test_pend @assertion_count = 0 - sample_test_case = Class.new Test::Unit::TestCase do - def self.test_order; :random; end - def test_test1; assert "does not matter" end - def test_test2; assert "does not matter" end - def test_test3; assert "does not matter" end - @test_order = [1, 0, 2] - def self.rand(n) @test_order.shift; end + util_assert_triggered "haha!", Test::Unit::PendedError do + @tc.pend "haha!" end - - expected = %w(test_test2 test_test1 test_test3) - assert_equal expected, sample_test_case.test_methods end - def test_test_methods_sorted + def test_test_methods @assertion_count = 0 sample_test_case = Class.new Test::Unit::TestCase do - def self.test_order; :sorted end - def test_test3; assert "does not matter" end - def test_test2; assert "does not matter" end def test_test1; assert "does not matter" end + def test_test2; assert "does not matter" end + def test_test3; assert "does not matter" end end - expected = %w(test_test1 test_test2 test_test3) - assert_equal expected, sample_test_case.test_methods + expected = %i(test_test1 test_test2 test_test3) + assert_equal expected, sample_test_case.test_methods.sort end def assert_triggered expected, klass = Test::Unit::AssertionFailedError @@ -1484,7 +1475,13 @@ class TestMiniTestUnitRecording < MetaMetaMetaTestCase def test_record_skip assert_run_record Test::Unit::PendedError do def test_method - skip "not yet" + omit "not yet" + end + end + + assert_run_record Test::Unit::PendedError do + def test_method + pend "not yet" end end end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb index 2b76ee3f1d..adf7d62ecd 100644 --- a/tool/test/testunit/test_parallel.rb +++ b/tool/test/testunit/test_parallel.rb @@ -3,17 +3,25 @@ require 'test/unit' require 'timeout' module TestParallel - PARALLEL_RB = "#{File.dirname(__FILE__)}/../../lib/test/unit/parallel.rb" - TESTS = "#{File.dirname(__FILE__)}/tests_for_parallel" + PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" + TESTS = "#{__dir__}/tests_for_parallel" # use large timeout for --jit-wait - TIMEOUT = EnvUtil.apply_timeout_scale(defined?(RubyVM::JIT) && RubyVM::JIT.enabled? ? 100 : 30) + TIMEOUT = EnvUtil.apply_timeout_scale(100) + + def self.timeout(n, &blk) + start_time = Time.now + Timeout.timeout(n, &blk) + rescue Timeout::Error + end_time = Time.now + raise Timeout::Error, "execution expired (start: #{ start_time }, end: #{ end_time })" + end class TestParallelWorker < Test::Unit::TestCase def setup i, @worker_in = IO.pipe @worker_out, o = IO.pipe - @worker_pid = spawn(*@options[:ruby], PARALLEL_RB, - "--ruby", @options[:ruby].join(" "), + @worker_pid = spawn(*@__runner_options__[:ruby], PARALLEL_RB, + "--ruby", @__runner_options__[:ruby].join(" "), "-j", "t1", "-v", out: o, in: i) [i,o].each(&:close) end @@ -22,10 +30,10 @@ module TestParallel if @worker_pid && @worker_in begin begin - @worker_in.puts "quit" + @worker_in.puts "quit normal" rescue IOError, Errno::EPIPE end - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@worker_pid) end rescue Timeout::Error @@ -45,10 +53,11 @@ module TestParallel end def test_run - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) @@ -57,13 +66,15 @@ module TestParallel end def test_run_multiple_testcase_in_one_file - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_second.rb test" assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) @@ -72,19 +83,22 @@ module TestParallel end def test_accept_run_command_multiple_times - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_first.rb test" assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) assert_match(/^ready/,@worker_out.gets) @worker_in.puts "run #{TESTS}/ptest_second.rb test" assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) assert_match(/^record/,@worker_out.gets) assert_match(/^p/,@worker_out.gets) assert_match(/^done/,@worker_out.gets) @@ -93,62 +107,62 @@ module TestParallel end def test_p - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_first.rb test" while buf = @worker_out.gets break if /^p (.+?)$/ =~ buf end assert_not_nil($1, "'p' was not found") - assert_match(/TestA#test_nothing_test = \d+\.\d+ s = \.\n/, $1.chomp.unpack("m")[0]) + assert_match(/TestA#test_nothing_test = \d+\.\d+ s = \.\n/, $1.chomp.unpack1("m")) end end def test_done - Timeout.timeout(TIMEOUT) do + ::TestParallel.timeout(TIMEOUT) do @worker_in.puts "run #{TESTS}/ptest_forth.rb test" while buf = @worker_out.gets break if /^done (.+?)$/ =~ buf end assert_not_nil($1, "'done' was not found") - result = Marshal.load($1.chomp.unpack("m")[0]) - assert_equal(5, result[0]) - pend "TODO: result[1] returns 17. We should investigate it" do - assert_equal(12, result[1]) - end - assert_kind_of(Array,result[2]) - assert_kind_of(Array,result[3]) - assert_kind_of(Array,result[4]) - assert_kind_of(Array,result[2][1]) - assert_kind_of(Test::Unit::AssertionFailedError,result[2][0][2]) - assert_kind_of(Test::Unit::PendedError,result[2][1][2]) - assert_kind_of(Test::Unit::PendedError,result[2][2][2]) - assert_kind_of(Exception, result[2][3][2]) - assert_equal(result[5], "TestE") + result = Marshal.load($1.chomp.unpack1("m")) + tests, asserts, reports, failures, loadpaths, suite = result + assert_equal(5, tests) + assert_equal(12, asserts) + assert_kind_of(Array, reports) + assert_kind_of(Array, failures) + assert_kind_of(Array, loadpaths) + reports.sort_by! {|_, t| t} + assert_kind_of(Array, reports[1]) + assert_kind_of(Test::Unit::AssertionFailedError, reports[0][2]) + assert_kind_of(Test::Unit::PendedError, reports[1][2]) + assert_kind_of(Test::Unit::PendedError, reports[2][2]) + assert_kind_of(Exception, reports[3][2]) + assert_equal("TestE", suite) end end def test_quit - Timeout.timeout(TIMEOUT) do - @worker_in.puts "quit" + ::TestParallel.timeout(TIMEOUT) do + @worker_in.puts "quit normal" assert_match(/^bye$/m,@worker_out.read) end end end class TestParallel < Test::Unit::TestCase - def spawn_runner(*opt_args) + def spawn_runner(*opt_args, jobs: "t1", env: {}) @test_out, o = IO.pipe - @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", - "--ruby", @options[:ruby].join(" "), - "-j","t1",*opt_args, out: o, err: o) + @test_pid = spawn(env, *@__runner_options__[:ruby], TESTS+"/runner.rb", + "--ruby", @__runner_options__[:ruby].join(" "), + "-j", jobs, *opt_args, out: o, err: o) o.close end def teardown begin if @test_pid - Timeout.timeout(2) do + ::TestParallel.timeout(2) do Process.waitpid(@test_pid) end end @@ -160,47 +174,50 @@ module TestParallel end def test_ignore_jzero - @test_out, o = IO.pipe - @test_pid = spawn(*@options[:ruby], TESTS+"/runner.rb", - "--ruby", @options[:ruby].join(" "), - "-j","0", out: File::NULL, err: o) - o.close - Timeout.timeout(TIMEOUT) { + spawn_runner(jobs: "0") + ::TestParallel.timeout(TIMEOUT) { assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) } end def test_should_run_all_without_any_leaks spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^9 tests/,buf) end def test_should_retry_failed_on_workers - spawn_runner - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + spawn_runner "--retry" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/^Retrying\.+$/,buf) end def test_no_retry_option spawn_runner "--no-retry" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} refute_match(/^Retrying\.+$/,buf) assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf) end def test_jobs_status spawn_runner "--jobs-status" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert_match(/\d+=ptest_(first|second|third|forth) */,buf) end def test_separate # this test depends to --jobs-status spawn_runner "--jobs-status", "--separate" - buf = Timeout.timeout(TIMEOUT) {@test_out.read} + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1, message("retried tests should run in different processes") {buf}) end + + def test_hungup + spawn_runner("--worker-timeout=1", "--retry", "test4test_hungup.rb", env: {"RUBY_CRASH_REPORT"=>nil}) + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} + assert_match(/^Retrying hung up testcases\.+$/, buf) + assert_match(/^2 tests,.* 0 failures,/, buf) + end end end diff --git a/tool/test/testunit/test_redefinition.rb b/tool/test/testunit/test_redefinition.rb index 9129e55489..b4f5cabd4f 100644 --- a/tool/test/testunit/test_redefinition.rb +++ b/tool/test/testunit/test_redefinition.rb @@ -3,14 +3,9 @@ require 'test/unit' class TestRedefinition < Test::Unit::TestCase def test_redefinition - assert_match(/^test\/unit warning: method TestForTestRedefinition#test_redefinition is redefined$/, - redefinition) - end - - def redefinition(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_redefinition.rb", *args], - err: [:child, :out]) {|f| - f.read - } + message = %r[test/unit: method TestForTestRedefinition#test_redefinition is redefined$] + assert_raise_with_message(Test::Unit::AssertionFailedError, message) do + require_relative("test4test_redefinition.rb") + end end end diff --git a/tool/test/testunit/test_sorting.rb b/tool/test/testunit/test_sorting.rb index f9de3ec154..3e5d7bfdcc 100644 --- a/tool/test/testunit/test_sorting.rb +++ b/tool/test/testunit/test_sorting.rb @@ -10,9 +10,66 @@ class TestTestUnitSorting < Test::Unit::TestCase end def sorting(*args) - IO.popen([*@options[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", "--verbose", *args], err: [:child, :out]) {|f| f.read } end + + Item = Struct.new(:name) + SEED = 0x50975eed + + def make_test_list + (1..16).map {"test_%.3x" % rand(0x1000)}.freeze + end + + def test_sort_alpha + sorter = Test::Unit::Order::Types[:alpha].new(SEED) + assert_kind_of(Test::Unit::Order::Types[:sorted], sorter) + + list = make_test_list + sorted = list.sort + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + sorted = list.sort_by(&:name) + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + end + + def test_sort_nosort + sorter = Test::Unit::Order::Types[:nosort].new(SEED) + + list = make_test_list + 16.times do + assert_equal(list, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(list, sorter.sort_by_name(list)) + end + end + + def test_sort_random + type = Test::Unit::Order::Types[:random] + sorter = type.new(SEED) + + list = make_test_list + sorted = type.new(SEED).sort_by_string(list).freeze + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_string(list)) + + list = list.map {|s| Item.new(s)}.freeze + sorted = sorted.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_name(list)) + end end diff --git a/tool/test/testunit/test_timeout.rb b/tool/test/testunit/test_timeout.rb new file mode 100644 index 0000000000..452f5e1a7e --- /dev/null +++ b/tool/test/testunit/test_timeout.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestTiemout < Test::Unit::TestCase + def test_timeout + cmd = [*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_timeout.rb"] + result = IO.popen(cmd, err: [:child, :out], &:read) + assert_not_match(/^T{10}$/, result) + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb index 8831676e19..54474c828d 100644 --- a/tool/test/testunit/tests_for_parallel/ptest_forth.rb +++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb @@ -8,19 +8,19 @@ class TestE < Test::Unit::TestCase assert_equal(1,1) end - def test_always_skip - skip "always" + def test_always_omit + omit "always" end def test_always_fail assert_equal(0,1) end - def test_skip_after_unknown_error + def test_pend_after_unknown_error begin raise UnknownError, "unknown error" rescue - skip "after raise" + pend "after raise" end end diff --git a/tool/test/testunit/tests_for_parallel/slow_helper.rb b/tool/test/testunit/tests_for_parallel/slow_helper.rb new file mode 100644 index 0000000000..38067c1f47 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/slow_helper.rb @@ -0,0 +1,8 @@ +require 'test/unit' + +module TestSlowTimeout + def test_slow + sleep_for = EnvUtil.apply_timeout_scale((ENV['sec'] || 3).to_i) + sleep sleep_for if on_parallel_worker? + end +end diff --git a/tool/test/testunit/tests_for_parallel/test4test_hungup.rb b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb new file mode 100644 index 0000000000..49f503ba9e --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +require_relative '../../../lib/test/unit' + +class TestHung < Test::Unit::TestCase + def test_success_at_worker + assert true + end + + def test_hungup_at_worker + if on_parallel_worker? + sleep EnvUtil.apply_timeout_scale(10) + end + assert true + end +end diff --git a/tool/test/webrick/.htaccess b/tool/test/webrick/.htaccess deleted file mode 100644 index 69d4659b9f..0000000000 --- a/tool/test/webrick/.htaccess +++ /dev/null @@ -1 +0,0 @@ -this file should not be published. diff --git a/tool/test/webrick/test_cgi.rb b/tool/test/webrick/test_cgi.rb deleted file mode 100644 index 7a75cf565e..0000000000 --- a/tool/test/webrick/test_cgi.rb +++ /dev/null @@ -1,170 +0,0 @@ -# coding: US-ASCII -# frozen_string_literal: false -require_relative "utils" -require "webrick" -require "test/unit" - -class TestWEBrickCGI < Test::Unit::TestCase - CRLF = "\r\n" - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def start_cgi_server(log_tester=TestWEBrick::DefaultLogTester, &block) - config = { - :CGIInterpreter => TestWEBrick::RubyBin, - :DocumentRoot => File.dirname(__FILE__), - :DirectoryIndex => ["webrick.cgi"], - :RequestCallback => Proc.new{|req, res| - def req.meta_vars - meta = super - meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) - meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] - return meta - end - }, - } - if RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32/ - config[:CGIPathEnv] = ENV['PATH'] # runtime dll may not be in system dir. - end - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - block.call(server, addr, port, log) - } - end - - def test_cgi - start_cgi_server{|server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.cgi") - http.request(req){|res| assert_equal("/webrick.cgi", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi/path/info") - http.request(req){|res| assert_equal("/path/info", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi/%3F%3F%3F?foo=bar") - http.request(req){|res| assert_equal("/???", res.body, log.call)} - unless RUBY_PLATFORM =~ /mswin|mingw|cygwin|bccwin32|java/ - # Path info of res.body is passed via ENV. - # ENV[] returns different value on Windows depending on locale. - req = Net::HTTP::Get.new("/webrick.cgi/%A4%DB%A4%B2/%A4%DB%A4%B2") - http.request(req){|res| - assert_equal("/\xA4\xDB\xA4\xB2/\xA4\xDB\xA4\xB2", res.body, log.call)} - end - req = Net::HTTP::Get.new("/webrick.cgi?a=1;a=2;b=x") - http.request(req){|res| assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Get.new("/webrick.cgi?a=1&a=2&b=x") - http.request(req){|res| assert_equal("a=1, a=2, b=x", res.body, log.call)} - - req = Net::HTTP::Post.new("/webrick.cgi?a=x;a=y;b=1") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req, "a=1;a=2;b=x"){|res| - assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Post.new("/webrick.cgi?a=x&a=y&b=1") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req, "a=1&a=2&b=x"){|res| - assert_equal("a=1, a=2, b=x", res.body, log.call)} - req = Net::HTTP::Get.new("/") - http.request(req){|res| - ary = res.body.lines.to_a - assert_match(%r{/$}, ary[0], log.call) - assert_match(%r{/webrick.cgi$}, ary[1], log.call) - } - - req = Net::HTTP::Get.new("/webrick.cgi") - req["Cookie"] = "CUSTOMER=WILE_E_COYOTE; PART_NUMBER=ROCKET_LAUNCHER_0001" - http.request(req){|res| - assert_equal( - "CUSTOMER=WILE_E_COYOTE\nPART_NUMBER=ROCKET_LAUNCHER_0001\n", - res.body, log.call) - } - - req = Net::HTTP::Get.new("/webrick.cgi") - cookie = %{$Version="1"; } - cookie << %{Customer="WILE_E_COYOTE"; $Path="/acme"; } - cookie << %{Part_Number="Rocket_Launcher_0001"; $Path="/acme"; } - cookie << %{Shipping="FedEx"; $Path="/acme"} - req["Cookie"] = cookie - http.request(req){|res| - assert_equal("Customer=WILE_E_COYOTE, Shipping=FedEx", - res["Set-Cookie"], log.call) - assert_equal("Customer=WILE_E_COYOTE\n" + - "Part_Number=Rocket_Launcher_0001\n" + - "Shipping=FedEx\n", res.body, log.call) - } - } - end - - def test_bad_request - log_tester = lambda {|log, access_log| - assert_match(/BadRequest/, log.join) - } - start_cgi_server(log_tester) {|server, addr, port, log| - sock = TCPSocket.new(addr, port) - begin - sock << "POST /webrick.cgi HTTP/1.0" << CRLF - sock << "Content-Type: application/x-www-form-urlencoded" << CRLF - sock << "Content-Length: 1024" << CRLF - sock << CRLF - sock << "a=1&a=2&b=x" - sock.close_write - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, sock.read, log.call) - ensure - sock.close - end - } - end - - def test_cgi_env - start_cgi_server do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.cgi/dumpenv") - req['proxy'] = 'http://example.com/' - req['hello'] = 'world' - http.request(req) do |res| - env = Marshal.load(res.body) - assert_equal 'world', env['HTTP_HELLO'] - assert_not_operator env, :include?, 'HTTP_PROXY' - end - end - end - - CtrlSeq = [0x7f, *(1..31)].pack("C*").gsub(/\s+/, '') - CtrlPat = /#{Regexp.quote(CtrlSeq)}/o - DumpPat = /#{Regexp.quote(CtrlSeq.dump[1...-1])}/o - - def test_bad_uri - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR bad URI/, log[0]) - } - start_cgi_server(log_tester) {|server, addr, port, log| - res = TCPSocket.open(addr, port) {|sock| - sock << "GET /#{CtrlSeq}#{CRLF}#{CRLF}" - sock.close_write - sock.read - } - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, res) - s = log.call.each_line.grep(/ERROR bad URI/)[0] - assert_match(DumpPat, s) - assert_not_match(CtrlPat, s) - } - end - - def test_bad_header - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR bad header/, log[0]) - } - start_cgi_server(log_tester) {|server, addr, port, log| - res = TCPSocket.open(addr, port) {|sock| - sock << "GET / HTTP/1.0#{CRLF}#{CtrlSeq}#{CRLF}#{CRLF}" - sock.close_write - sock.read - } - assert_match(%r{\AHTTP/\d.\d 400 Bad Request}, res) - s = log.call.each_line.grep(/ERROR bad header/)[0] - assert_match(DumpPat, s) - assert_not_match(CtrlPat, s) - } - end -end diff --git a/tool/test/webrick/test_config.rb b/tool/test/webrick/test_config.rb deleted file mode 100644 index a54a667452..0000000000 --- a/tool/test/webrick/test_config.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/config" - -class TestWEBrickConfig < Test::Unit::TestCase - def test_server_name_default - config = WEBrick::Config::General.dup - assert_equal(false, config.key?(:ServerName)) - assert_equal(WEBrick::Utils.getservername, config[:ServerName]) - assert_equal(true, config.key?(:ServerName)) - end - - def test_server_name_set_nil - config = WEBrick::Config::General.dup.update(ServerName: nil) - assert_equal(nil, config[:ServerName]) - end -end diff --git a/tool/test/webrick/test_cookie.rb b/tool/test/webrick/test_cookie.rb deleted file mode 100644 index e46185f127..0000000000 --- a/tool/test/webrick/test_cookie.rb +++ /dev/null @@ -1,141 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/cookie" - -class TestWEBrickCookie < Test::Unit::TestCase - def test_new - cookie = WEBrick::Cookie.new("foo","bar") - assert_equal("foo", cookie.name) - assert_equal("bar", cookie.value) - assert_equal("foo=bar", cookie.to_s) - end - - def test_time - cookie = WEBrick::Cookie.new("foo","bar") - t = 1000000000 - cookie.max_age = t - assert_match(t.to_s, cookie.to_s) - - cookie = WEBrick::Cookie.new("foo","bar") - t = Time.at(1000000000) - cookie.expires = t - assert_equal(Time, cookie.expires.class) - assert_equal(t, cookie.expires) - ts = t.httpdate - cookie.expires = ts - assert_equal(Time, cookie.expires.class) - assert_equal(t, cookie.expires) - assert_match(ts, cookie.to_s) - end - - def test_parse - data = "" - data << '$Version="1"; ' - data << 'Customer="WILE_E_COYOTE"; $Path="/acme"; ' - data << 'Part_Number="Rocket_Launcher_0001"; $Path="/acme"; ' - data << 'Shipping="FedEx"; $Path="/acme"' - cookies = WEBrick::Cookie.parse(data) - assert_equal(3, cookies.size) - assert_equal(1, cookies[0].version) - assert_equal("Customer", cookies[0].name) - assert_equal("WILE_E_COYOTE", cookies[0].value) - assert_equal("/acme", cookies[0].path) - assert_equal(1, cookies[1].version) - assert_equal("Part_Number", cookies[1].name) - assert_equal("Rocket_Launcher_0001", cookies[1].value) - assert_equal(1, cookies[2].version) - assert_equal("Shipping", cookies[2].name) - assert_equal("FedEx", cookies[2].value) - - data = "hoge=moge; __div__session=9865ecfd514be7f7" - cookies = WEBrick::Cookie.parse(data) - assert_equal(2, cookies.size) - assert_equal(0, cookies[0].version) - assert_equal("hoge", cookies[0].name) - assert_equal("moge", cookies[0].value) - assert_equal("__div__session", cookies[1].name) - assert_equal("9865ecfd514be7f7", cookies[1].value) - - # don't allow ,-separator - data = "hoge=moge, __div__session=9865ecfd514be7f7" - cookies = WEBrick::Cookie.parse(data) - assert_equal(1, cookies.size) - assert_equal(0, cookies[0].version) - assert_equal("hoge", cookies[0].name) - assert_equal("moge, __div__session=9865ecfd514be7f7", cookies[0].value) - end - - def test_parse_no_whitespace - data = [ - '$Version="1"; ', - 'Customer="WILE_E_COYOTE";$Path="/acme";', # no SP between cookie-string - 'Part_Number="Rocket_Launcher_0001";$Path="/acme";', # no SP between cookie-string - 'Shipping="FedEx";$Path="/acme"' - ].join - cookies = WEBrick::Cookie.parse(data) - assert_equal(1, cookies.size) - end - - def test_parse_too_much_whitespaces - # According to RFC6265, - # cookie-string = cookie-pair *( ";" SP cookie-pair ) - # So single 0x20 is needed after ';'. We allow multiple spaces here for - # compatibility with older WEBrick versions. - data = [ - '$Version="1"; ', - 'Customer="WILE_E_COYOTE";$Path="/acme"; ', # no SP between cookie-string - 'Part_Number="Rocket_Launcher_0001";$Path="/acme"; ', # no SP between cookie-string - 'Shipping="FedEx";$Path="/acme"' - ].join - cookies = WEBrick::Cookie.parse(data) - assert_equal(3, cookies.size) - end - - def test_parse_set_cookie - data = %(Customer="WILE_E_COYOTE"; Version="1"; Path="/acme") - cookie = WEBrick::Cookie.parse_set_cookie(data) - assert_equal("Customer", cookie.name) - assert_equal("WILE_E_COYOTE", cookie.value) - assert_equal(1, cookie.version) - assert_equal("/acme", cookie.path) - - data = %(Shipping="FedEx"; Version="1"; Path="/acme"; Secure) - cookie = WEBrick::Cookie.parse_set_cookie(data) - assert_equal("Shipping", cookie.name) - assert_equal("FedEx", cookie.value) - assert_equal(1, cookie.version) - assert_equal("/acme", cookie.path) - assert_equal(true, cookie.secure) - end - - def test_parse_set_cookies - data = %(Shipping="FedEx"; Version="1"; Path="/acme"; Secure) - data << %(, CUSTOMER=WILE_E_COYOTE; path=/; expires=Wednesday, 09-Nov-99 23:12:40 GMT; path=/; Secure) - data << %(, name="Aaron"; Version="1"; path="/acme") - cookies = WEBrick::Cookie.parse_set_cookies(data) - assert_equal(3, cookies.length) - - fed_ex = cookies.find { |c| c.name == 'Shipping' } - assert_not_nil(fed_ex) - assert_equal("Shipping", fed_ex.name) - assert_equal("FedEx", fed_ex.value) - assert_equal(1, fed_ex.version) - assert_equal("/acme", fed_ex.path) - assert_equal(true, fed_ex.secure) - - name = cookies.find { |c| c.name == 'name' } - assert_not_nil(name) - assert_equal("name", name.name) - assert_equal("Aaron", name.value) - assert_equal(1, name.version) - assert_equal("/acme", name.path) - - customer = cookies.find { |c| c.name == 'CUSTOMER' } - assert_not_nil(customer) - assert_equal("CUSTOMER", customer.name) - assert_equal("WILE_E_COYOTE", customer.value) - assert_equal(0, customer.version) - assert_equal("/", customer.path) - assert_equal(Time.utc(1999, 11, 9, 23, 12, 40), customer.expires) - end -end diff --git a/tool/test/webrick/test_do_not_reverse_lookup.rb b/tool/test/webrick/test_do_not_reverse_lookup.rb deleted file mode 100644 index efcb5a9299..0000000000 --- a/tool/test/webrick/test_do_not_reverse_lookup.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick" -require_relative "utils" - -class TestDoNotReverseLookup < Test::Unit::TestCase - class DNRL < WEBrick::GenericServer - def run(sock) - sock << sock.do_not_reverse_lookup.to_s - end - end - - @@original_do_not_reverse_lookup_value = Socket.do_not_reverse_lookup - - def teardown - Socket.do_not_reverse_lookup = @@original_do_not_reverse_lookup_value - end - - def do_not_reverse_lookup?(config) - result = nil - TestWEBrick.start_server(DNRL, config) do |server, addr, port, log| - TCPSocket.open(addr, port) do |sock| - result = {'true' => true, 'false' => false}[sock.gets] - end - end - result - end - - # +--------------------------------------------------------------------------+ - # | Expected interaction between Socket.do_not_reverse_lookup | - # | and WEBrick::Config::General[:DoNotReverseLookup] | - # +----------------------------+---------------------------------------------+ - # | |WEBrick::Config::General[:DoNotReverseLookup]| - # +----------------------------+--------------+---------------+--------------+ - # |Socket.do_not_reverse_lookup| TRUE | FALSE | NIL | - # +----------------------------+--------------+---------------+--------------+ - # | TRUE | true | false | true | - # +----------------------------+--------------+---------------+--------------+ - # | FALSE | true | false | false | - # +----------------------------+--------------+---------------+--------------+ - - def test_socket_dnrl_true_server_dnrl_true - Socket.do_not_reverse_lookup = true - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => true)) - end - - def test_socket_dnrl_true_server_dnrl_false - Socket.do_not_reverse_lookup = true - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => false)) - end - - def test_socket_dnrl_true_server_dnrl_nil - Socket.do_not_reverse_lookup = true - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => nil)) - end - - def test_socket_dnrl_false_server_dnrl_true - Socket.do_not_reverse_lookup = false - assert_equal(true, do_not_reverse_lookup?(:DoNotReverseLookup => true)) - end - - def test_socket_dnrl_false_server_dnrl_false - Socket.do_not_reverse_lookup = false - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => false)) - end - - def test_socket_dnrl_false_server_dnrl_nil - Socket.do_not_reverse_lookup = false - assert_equal(false, do_not_reverse_lookup?(:DoNotReverseLookup => nil)) - end -end diff --git a/tool/test/webrick/test_filehandler.rb b/tool/test/webrick/test_filehandler.rb deleted file mode 100644 index 146d8ce792..0000000000 --- a/tool/test/webrick/test_filehandler.rb +++ /dev/null @@ -1,403 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require_relative "utils.rb" -require "webrick" -require "stringio" -require "tmpdir" - -class WEBrick::TestFileHandler < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def default_file_handler(filename) - klass = WEBrick::HTTPServlet::DefaultFileHandler - klass.new(WEBrick::Config::HTTP, filename) - end - - def windows? - File.directory?("\\") - end - - def get_res_body(res) - sio = StringIO.new - sio.binmode - res.send_body(sio) - sio.string - end - - def make_range_request(range_spec) - msg = <<-END_OF_REQUEST - GET / HTTP/1.0 - Range: #{range_spec} - - END_OF_REQUEST - return StringIO.new(msg.gsub(/^ {6}/, "")) - end - - def make_range_response(file, range_spec) - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(make_range_request(range_spec)) - res = WEBrick::HTTPResponse.new(WEBrick::Config::HTTP) - size = File.size(file) - handler = default_file_handler(file) - handler.make_partial_content(req, res, file, size) - return res - end - - def test_make_partial_content - filename = __FILE__ - filesize = File.size(filename) - - res = make_range_response(filename, "bytes=#{filesize-100}-") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=-100") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-99") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=100-199") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(100, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-0") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(1, get_res_body(res).size) - - res = make_range_response(filename, "bytes=-1") - assert_match(%r{^text/plain}, res["content-type"]) - assert_equal(1, get_res_body(res).size) - - res = make_range_response(filename, "bytes=0-0, -2") - assert_match(%r{^multipart/byteranges}, res["content-type"]) - body = get_res_body(res) - boundary = /; boundary=(.+)/.match(res['content-type'])[1] - off = filesize - 2 - last = filesize - 1 - - exp = "--#{boundary}\r\n" \ - "Content-Type: text/plain\r\n" \ - "Content-Range: bytes 0-0/#{filesize}\r\n" \ - "\r\n" \ - "#{IO.read(__FILE__, 1)}\r\n" \ - "--#{boundary}\r\n" \ - "Content-Type: text/plain\r\n" \ - "Content-Range: bytes #{off}-#{last}/#{filesize}\r\n" \ - "\r\n" \ - "#{IO.read(__FILE__, 2, off)}\r\n" \ - "--#{boundary}--\r\n" - assert_equal exp, body - end - - def test_filehandler - config = { :DocumentRoot => File.dirname(__FILE__), } - this_file = File.basename(__FILE__) - filesize = File.size(__FILE__) - this_data = File.binread(__FILE__) - range = nil - bug2593 = '[ruby-dev:40030]' - - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - begin - server[:DocumentRootOptions][:NondisclosureName] = [] - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/html", res.content_type, log.call) - assert_match(/HREF="#{this_file}"/, res.body, log.call) - } - req = Net::HTTP::Get.new("/#{this_file}") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_equal(this_data, res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=#{filesize-100}-") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-100)..(filesize-1), range, log.call) - assert_equal(this_data[-100..-1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=-100") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-100)..(filesize-1), range, log.call) - assert_equal(this_data[-100..-1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-99") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(0..99, range, log.call) - assert_equal(this_data[0..99], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=100-199") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(100..199, range, log.call) - assert_equal(this_data[100..199], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-0") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal(0..0, range, log.call) - assert_equal(this_data[0..0], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=-1") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("text/plain", res.content_type, log.call) - assert_nothing_raised(bug2593) {range = res.content_range} - assert_equal((filesize-1)..(filesize-1), range, log.call) - assert_equal(this_data[-1, 1], res.body, log.call) - } - - req = Net::HTTP::Get.new("/#{this_file}", "range"=>"bytes=0-0, -2") - http.request(req){|res| - assert_equal("206", res.code, log.call) - assert_equal("multipart/byteranges", res.content_type, log.call) - } - ensure - server[:DocumentRootOptions].delete :NondisclosureName - end - end - end - - def test_non_disclosure_name - config = { :DocumentRoot => File.dirname(__FILE__), } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } - assert_equal([], log) - } - this_file = File.basename(__FILE__) - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - doc_root_opts = server[:DocumentRootOptions] - doc_root_opts[:NondisclosureName] = %w(.ht* *~ test_*) - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("200", res.code, log.call) - assert_equal("text/html", res.content_type, log.call) - assert_no_match(/HREF="#{File.basename(__FILE__)}"/, res.body) - } - req = Net::HTTP::Get.new("/#{this_file}") - http.request(req){|res| - assert_equal("404", res.code, log.call) - } - doc_root_opts[:NondisclosureName] = %w(.ht* *~ TEST_*) - http.request(req){|res| - assert_equal("404", res.code, log.call) - } - end - end - - def test_directory_traversal - return if File.executable?(__FILE__) # skip on strange file system - - config = { :DocumentRoot => File.dirname(__FILE__), } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR bad URI/ =~ s } - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/../../") - http.request(req){|res| assert_equal("400", res.code, log.call) } - req = Net::HTTP::Get.new("/..%5c../#{File.basename(__FILE__)}") - http.request(req){|res| assert_equal(windows? ? "200" : "404", res.code, log.call) } - req = Net::HTTP::Get.new("/..%5c..%5cruby.c") - http.request(req){|res| assert_equal("404", res.code, log.call) } - end - end - - def test_unwise_in_path - if windows? - config = { :DocumentRoot => File.dirname(__FILE__), } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/..%5c..") - http.request(req){|res| assert_equal("301", res.code, log.call) } - end - end - end - - def test_short_filename - return if File.executable?(__FILE__) # skip on strange file system - return if /mswin/ =~ RUBY_PLATFORM && ENV.key?('GITHUB_ACTIONS') # not working from the beginning - - config = { - :CGIInterpreter => TestWEBrick::RubyBin, - :DocumentRoot => File.dirname(__FILE__), - :CGIPathEnv => ENV['PATH'], - } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - log = log.reject {|s| /WARN the request refers nondisclosure name/ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - if windows? - root = config[:DocumentRoot].tr("/", "\\") - fname = IO.popen(%W[dir /x #{root}\\webrick_long_filename.cgi], encoding: "binary", &:read) - fname.sub!(/\A.*$^$.*$^$/m, '') - if fname - fname = fname[/\s(w.+?cgi)\s/i, 1] - fname.downcase! - end - else - fname = "webric~1.cgi" - end - req = Net::HTTP::Get.new("/#{fname}/test") - http.request(req) do |res| - if windows? - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - else - assert_equal("404", res.code, log.call) - end - end - - req = Net::HTTP::Get.new("/.htaccess") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - req = Net::HTTP::Get.new("/htacce~1") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - req = Net::HTTP::Get.new("/HTACCE~1") - http.request(req) {|res| assert_equal("404", res.code, log.call) } - end - end - - def test_multibyte_char_in_path - if Encoding.default_external == Encoding.find('US-ASCII') - reset_encoding = true - verb = $VERBOSE - $VERBOSE = false - Encoding.default_external = Encoding.find('UTF-8') - end - - c = "\u00a7" - begin - c = c.encode('filesystem') - rescue EncodingError - c = c.b - end - Dir.mktmpdir(c) do |dir| - basename = "#{c}.txt" - File.write("#{dir}/#{basename}", "test_multibyte_char_in_path") - Dir.mkdir("#{dir}/#{c}") - File.write("#{dir}/#{c}/#{basename}", "nested") - config = { - :DocumentRoot => dir, - :DirectoryIndex => [basename], - } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - path = "/#{basename}" - req = Net::HTTP::Get.new(WEBrick::HTTPUtils::escape(path)) - http.request(req){|res| assert_equal("200", res.code, log.call + "\nFilesystem encoding is #{Encoding.find('filesystem')}") } - path = "/#{c}/#{basename}" - req = Net::HTTP::Get.new(WEBrick::HTTPUtils::escape(path)) - http.request(req){|res| assert_equal("200", res.code, log.call) } - req = Net::HTTP::Get.new('/') - http.request(req){|res| - assert_equal("test_multibyte_char_in_path", res.body, log.call) - } - end - end - ensure - if reset_encoding - Encoding.default_external = Encoding.find('US-ASCII') - $VERBOSE = verb - end - end - - def test_script_disclosure - return if File.executable?(__FILE__) # skip on strange file system - - config = { - :CGIInterpreter => TestWEBrick::RubyBinArray, - :DocumentRoot => File.dirname(__FILE__), - :CGIPathEnv => ENV['PATH'], - :RequestCallback => Proc.new{|req, res| - def req.meta_vars - meta = super - meta["RUBYLIB"] = $:.join(File::PATH_SEPARATOR) - meta[RbConfig::CONFIG['LIBPATHENV']] = ENV[RbConfig::CONFIG['LIBPATHENV']] if RbConfig::CONFIG['LIBPATHENV'] - return meta - end - }, - } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - http.read_timeout = EnvUtil.apply_timeout_scale(60) - http.write_timeout = EnvUtil.apply_timeout_scale(60) if http.respond_to?(:write_timeout=) - - req = Net::HTTP::Get.new("/webrick.cgi/test") - http.request(req) do |res| - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - end - - resok = windows? - response_assertion = Proc.new do |res| - if resok - assert_equal("200", res.code, log.call) - assert_equal("/test", res.body, log.call) - else - assert_equal("404", res.code, log.call) - end - end - req = Net::HTTP::Get.new("/webrick.cgi%20/test") - http.request(req, &response_assertion) - req = Net::HTTP::Get.new("/webrick.cgi./test") - http.request(req, &response_assertion) - resok &&= File.exist?(__FILE__+"::$DATA") - req = Net::HTTP::Get.new("/webrick.cgi::$DATA/test") - http.request(req, &response_assertion) - end - end - - def test_erbhandler - config = { :DocumentRoot => File.dirname(__FILE__) } - log_tester = lambda {|log, access_log| - log = log.reject {|s| /ERROR `.*\' not found\./ =~ s } - assert_equal([], log) - } - TestWEBrick.start_httpserver(config, log_tester) do |server, addr, port, log| - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/webrick.rhtml") - http.request(req) do |res| - assert_equal("200", res.code, log.call) - assert_match %r!\Areq to http://[^/]+/webrick\.rhtml {}\n!, res.body - end - end - end -end diff --git a/tool/test/webrick/test_htgroup.rb b/tool/test/webrick/test_htgroup.rb deleted file mode 100644 index 8749711df5..0000000000 --- a/tool/test/webrick/test_htgroup.rb +++ /dev/null @@ -1,19 +0,0 @@ -require "tempfile" -require "test/unit" -require "webrick/httpauth/htgroup" - -class TestHtgroup < Test::Unit::TestCase - def test_htgroup - Tempfile.create('test_htgroup') do |tmpfile| - tmpfile.close - tmp_group = WEBrick::HTTPAuth::Htgroup.new(tmpfile.path) - tmp_group.add 'superheroes', %w[spiderman batman] - tmp_group.add 'supervillains', %w[joker] - tmp_group.flush - - htgroup = WEBrick::HTTPAuth::Htgroup.new(tmpfile.path) - assert_equal(htgroup.members('superheroes'), %w[spiderman batman]) - assert_equal(htgroup.members('supervillains'), %w[joker]) - end - end -end diff --git a/tool/test/webrick/test_htmlutils.rb b/tool/test/webrick/test_htmlutils.rb deleted file mode 100644 index ae1b8efa95..0000000000 --- a/tool/test/webrick/test_htmlutils.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/htmlutils" - -class TestWEBrickHTMLUtils < Test::Unit::TestCase - include WEBrick::HTMLUtils - - def test_escape - assert_equal("foo", escape("foo")) - assert_equal("foo bar", escape("foo bar")) - assert_equal("foo&bar", escape("foo&bar")) - assert_equal("foo"bar", escape("foo\"bar")) - assert_equal("foo>bar", escape("foo>bar")) - assert_equal("foo<bar", escape("foo<bar")) - assert_equal("\u{3053 3093 306B 3061 306F}", escape("\u{3053 3093 306B 3061 306F}")) - bug8425 = '[Bug #8425] [ruby-core:55052]' - assert_nothing_raised(ArgumentError, Encoding::CompatibilityError, bug8425) { - assert_equal("\u{3053 3093 306B}\xff<", escape("\u{3053 3093 306B}\xff<")) - } - end -end diff --git a/tool/test/webrick/test_httpauth.rb b/tool/test/webrick/test_httpauth.rb deleted file mode 100644 index 9fe8af8be2..0000000000 --- a/tool/test/webrick/test_httpauth.rb +++ /dev/null @@ -1,366 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "tempfile" -require "webrick" -require "webrick/httpauth/basicauth" -require "stringio" -require_relative "utils" - -class TestWEBrickHTTPAuth < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_basic_auth - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, log[0]) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth" - - server.mount_proc(path){|req, res| - WEBrick::HTTPAuth.basic_auth(req, res, realm){|user, pass| - user == "webrick" && pass == "supersecretpassword" - } - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("webrick", "supersecretpassword") - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - g.basic_auth("webrick", "not super") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - } - end - - def test_basic_auth_sha - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.puts("webrick:{SHA}GJYFRpBbdchp595jlh3Bhfmgp8k=") - tmpfile.flush - assert_raise(NotImplementedError){ - WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path) - } - } - end - - def test_basic_auth_md5 - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.puts("webrick:$apr1$IOVMD/..$rmnOSPXr0.wwrLPZHBQZy0") - tmpfile.flush - assert_raise(NotImplementedError){ - WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path) - } - } - end - - [nil, :crypt, :bcrypt].each do |hash_algo| - # OpenBSD does not support insecure DES-crypt - next if /openbsd/ =~ RUBY_PLATFORM && hash_algo != :bcrypt - - begin - case hash_algo - when :crypt - # require 'string/crypt' - when :bcrypt - require 'bcrypt' - end - rescue LoadError - next - end - - define_method(:"test_basic_auth_htpasswd_#{hash_algo}") do - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Basic WEBrick's realm: webrick: password unmatch\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth2" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htpasswd = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - users = [] - htpasswd.each{|user, pass| users << user } - assert_equal(2, users.size, log.call) - assert(users.member?("webrick"), log.call) - assert(users.member?("foo"), log.call) - - server.mount_proc(path){|req, res| - auth = WEBrick::HTTPAuth::BasicAuth.new( - :Realm => realm, :UserDB => htpasswd, - :Logger => server.logger - ) - auth.authenticate(req, res) - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("webrick", "supersecretpassword") - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - g.basic_auth("webrick", "not super") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - } - } - end - - define_method(:"test_basic_auth_bad_username_htpasswd_#{hash_algo}") do - log_tester = lambda {|log, access_log| - assert_equal(2, log.length) - assert_match(/ERROR Basic WEBrick's realm: foo\\ebar: the user is not allowed\./, log[0]) - assert_match(/ERROR WEBrick::HTTPStatus::Unauthorized/, log[1]) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/basic_auth" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htpasswd = WEBrick::HTTPAuth::Htpasswd.new(tmpfile.path, password_hash: hash_algo) - users = [] - htpasswd.each{|user, pass| users << user } - server.mount_proc(path){|req, res| - auth = WEBrick::HTTPAuth::BasicAuth.new( - :Realm => realm, :UserDB => htpasswd, - :Logger => server.logger - ) - auth.authenticate(req, res) - res.body = "hoge" - } - http = Net::HTTP.new(addr, port) - g = Net::HTTP::Get.new(path) - g.basic_auth("foo\ebar", "passwd") - http.request(g){|res| assert_not_equal("hoge", res.body, log.call) } - } - } - end - end - - DIGESTRES_ = / - ([a-zA-Z\-]+) - [ \t]*(?:\r\n[ \t]*)* - = - [ \t]*(?:\r\n[ \t]*)* - (?: - "((?:[^"]+|\\[\x00-\x7F])*)" | - ([!\#$%&'*+\-.0-9A-Z^_`a-z|~]+) - )/x - - def test_digest_auth - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Digest WEBrick's realm: no credentials in the request\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/, - /ERROR Digest WEBrick's realm: webrick: digest unmatch\./ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "WEBrick's realm" - path = "/digest_auth" - - Tempfile.create("test_webrick_auth") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - tmp_pass.set_passwd(realm, "webrick", "supersecretpassword") - tmp_pass.set_passwd(realm, "foo", "supersecretpassword") - tmp_pass.flush - - htdigest = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - users = [] - htdigest.each{|user, pass| users << user } - assert_equal(2, users.size, log.call) - assert(users.member?("webrick"), log.call) - assert(users.member?("foo"), log.call) - - auth = WEBrick::HTTPAuth::DigestAuth.new( - :Realm => realm, :UserDB => htdigest, - :Algorithm => 'MD5', - :Logger => server.logger - ) - server.mount_proc(path){|req, res| - auth.authenticate(req, res) - res.body = "hoge" - } - - Net::HTTP.start(addr, port) do |http| - g = Net::HTTP::Get.new(path) - params = {} - http.request(g) do |res| - assert_equal('401', res.code, log.call) - res["www-authenticate"].scan(DIGESTRES_) do |key, quoted, token| - params[key.downcase] = token || quoted.delete('\\') - end - params['uri'] = "http://#{addr}:#{port}#{path}" - end - - g['Authorization'] = credentials_for_request('webrick', "supersecretpassword", params) - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - - params['algorithm'].downcase! #4936 - g['Authorization'] = credentials_for_request('webrick', "supersecretpassword", params) - http.request(g){|res| assert_equal("hoge", res.body, log.call)} - - g['Authorization'] = credentials_for_request('webrick', "not super", params) - http.request(g){|res| assert_not_equal("hoge", res.body, log.call)} - end - } - } - end - - def test_digest_auth_int - log_tester = lambda {|log, access_log| - log.reject! {|line| /\A\s*\z/ =~ line } - pats = [ - /ERROR Digest wb auth-int realm: no credentials in the request\./, - /ERROR WEBrick::HTTPStatus::Unauthorized/, - /ERROR Digest wb auth-int realm: foo: digest unmatch\./ - ] - pats.each {|pat| - assert(!log.grep(pat).empty?, "webrick log doesn't have expected error: #{pat.inspect}") - log.reject! {|line| pat =~ line } - } - assert_equal([], log) - } - TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| - realm = "wb auth-int realm" - path = "/digest_auth_int" - - Tempfile.create("test_webrick_auth_int") {|tmpfile| - tmpfile.close - tmp_pass = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - tmp_pass.set_passwd(realm, "foo", "Hunter2") - tmp_pass.flush - - htdigest = WEBrick::HTTPAuth::Htdigest.new(tmpfile.path) - users = [] - htdigest.each{|user, pass| users << user } - assert_equal %w(foo), users - - auth = WEBrick::HTTPAuth::DigestAuth.new( - :Realm => realm, :UserDB => htdigest, - :Algorithm => 'MD5', - :Logger => server.logger, - :Qop => %w(auth-int), - ) - server.mount_proc(path){|req, res| - auth.authenticate(req, res) - res.body = "bbb" - } - Net::HTTP.start(addr, port) do |http| - post = Net::HTTP::Post.new(path) - params = {} - data = 'hello=world' - body = StringIO.new(data) - post.content_length = data.bytesize - post['Content-Type'] = 'application/x-www-form-urlencoded' - post.body_stream = body - - http.request(post) do |res| - assert_equal('401', res.code, log.call) - res["www-authenticate"].scan(DIGESTRES_) do |key, quoted, token| - params[key.downcase] = token || quoted.delete('\\') - end - params['uri'] = "http://#{addr}:#{port}#{path}" - end - - body.rewind - cred = credentials_for_request('foo', 'Hunter3', params, body) - post['Authorization'] = cred - post.body_stream = body - http.request(post){|res| - assert_equal('401', res.code, log.call) - assert_not_equal("bbb", res.body, log.call) - } - - body.rewind - cred = credentials_for_request('foo', 'Hunter2', params, body) - post['Authorization'] = cred - post.body_stream = body - http.request(post){|res| assert_equal("bbb", res.body, log.call)} - end - } - } - end - - def test_digest_auth_invalid - digest_auth = WEBrick::HTTPAuth::DigestAuth.new(Realm: 'realm', UserDB: '') - - def digest_auth.error(fmt, *) - end - - def digest_auth.try_bad_request(len) - request = {"Authorization" => %[Digest a="#{'\b'*len}]} - authenticate request, nil - end - - bad_request = WEBrick::HTTPStatus::BadRequest - t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC) - assert_raise(bad_request) {digest_auth.try_bad_request(10)} - limit = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) - [20, 50, 100, 200].each do |len| - assert_raise(bad_request) do - Timeout.timeout(len*limit) {digest_auth.try_bad_request(len)} - end - end - end - - private - def credentials_for_request(user, password, params, body = nil) - cnonce = "hoge" - nonce_count = 1 - ha1 = "#{user}:#{params['realm']}:#{password}" - if body - dig = Digest::MD5.new - while buf = body.read(16384) - dig.update(buf) - end - body.rewind - ha2 = "POST:#{params['uri']}:#{dig.hexdigest}" - else - ha2 = "GET:#{params['uri']}" - end - - request_digest = - "#{Digest::MD5.hexdigest(ha1)}:" \ - "#{params['nonce']}:#{'%08x' % nonce_count}:#{cnonce}:#{params['qop']}:" \ - "#{Digest::MD5.hexdigest(ha2)}" - "Digest username=\"#{user}\"" \ - ", realm=\"#{params['realm']}\"" \ - ", nonce=\"#{params['nonce']}\"" \ - ", uri=\"#{params['uri']}\"" \ - ", qop=#{params['qop']}" \ - ", nc=#{'%08x' % nonce_count}" \ - ", cnonce=\"#{cnonce}\"" \ - ", response=\"#{Digest::MD5.hexdigest(request_digest)}\"" \ - ", opaque=\"#{params['opaque']}\"" \ - ", algorithm=#{params['algorithm']}" - end -end diff --git a/tool/test/webrick/test_httpproxy.rb b/tool/test/webrick/test_httpproxy.rb deleted file mode 100644 index 66dae6f6f6..0000000000 --- a/tool/test/webrick/test_httpproxy.rb +++ /dev/null @@ -1,467 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require "webrick/httpproxy" -begin - require "webrick/ssl" - require "net/https" -rescue LoadError - # test_connect will be skipped -end -require File.expand_path("utils.rb", File.dirname(__FILE__)) - -class TestWEBrickHTTPProxy < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_fake_proxy - assert_nil(WEBrick::FakeProxyURI.scheme) - assert_nil(WEBrick::FakeProxyURI.host) - assert_nil(WEBrick::FakeProxyURI.port) - assert_nil(WEBrick::FakeProxyURI.path) - assert_nil(WEBrick::FakeProxyURI.userinfo) - assert_raise(NoMethodError){ WEBrick::FakeProxyURI.foo } - end - - def test_proxy - # Testing GET or POST to the proxy server - # Note that the proxy server works as the origin server. - # +------+ - # V | - # client -------> proxy ---+ - # GET / POST GET / POST - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - http = Net::HTTP.new(addr, port, addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_equal("GET / ", res.body, log.call) - } - assert_equal(1, proxy_handler_called, log.call) - assert_equal(2, request_handler_called, log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_nil(res.body, log.call) - } - assert_equal(2, proxy_handler_called, log.call) - assert_equal(4, request_handler_called, log.call) - - req = Net::HTTP::Post.new("/") - req.body = "post-data" - req.content_type = "application/x-www-form-urlencoded" - http.request(req){|res| - assert_equal("1.1 localhost.localdomain:#{port}", res["via"], log.call) - assert_equal("POST / post-data", res.body, log.call) - } - assert_equal(3, proxy_handler_called, log.call) - assert_equal(6, request_handler_called, log.call) - } - end - - def test_no_proxy - # Testing GET or POST to the proxy server without proxy request. - # - # client -------> proxy - # GET / POST - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - http = Net::HTTP.new(addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_equal("GET / ", res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(1, request_handler_called, log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_nil(res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(2, request_handler_called, log.call) - - req = Net::HTTP::Post.new("/") - req.content_type = "application/x-www-form-urlencoded" - req.body = "post-data" - http.request(req){|res| - assert_nil(res["via"], log.call) - assert_equal("POST / post-data", res.body, log.call) - } - assert_equal(0, proxy_handler_called, log.call) - assert_equal(3, request_handler_called, log.call) - } - end - - def test_big_bodies - require 'digest/md5' - rand_str = File.read(__FILE__) - rand_str.freeze - nr = 1024 ** 2 / rand_str.size # bigger works, too - exp = Digest::MD5.new - nr.times { exp.update(rand_str) } - exp = exp.hexdigest - TestWEBrick.start_httpserver do |o_server, o_addr, o_port, o_log| - o_server.mount_proc('/') do |req, res| - case req.request_method - when 'GET' - res['content-type'] = 'application/octet-stream' - if req.path == '/length' - res['content-length'] = (nr * rand_str.size).to_s - else - res.chunked = true - end - res.body = ->(socket) { nr.times { socket.write(rand_str) } } - when 'POST' - dig = Digest::MD5.new - req.body { |buf| dig.update(buf); buf.clear } - res['content-type'] = 'text/plain' - res['content-length'] = '32' - res.body = dig.hexdigest - end - end - - http = Net::HTTP.new(o_addr, o_port) - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Transfer-Encoding' => 'chunked', - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - - TestWEBrick.start_httpproxy do |p_server, p_addr, p_port, p_log| - http = Net::HTTP.new(o_addr, o_port, p_addr, p_port) - http.request_get('/length') do |res| - assert_equal(nr * rand_str.size, res.content_length) - dig = Digest::MD5.new - res.read_body { |buf| dig.update(buf); buf.clear } - assert_equal exp, dig.hexdigest - end - http.request_get('/') do |res| - assert_predicate res, :chunked? - dig = Digest::MD5.new - res.read_body { |buf| dig.update(buf); buf.clear } - assert_equal exp, dig.hexdigest - end - - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Content-Length' => (nr * rand_str.size).to_s, - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - - IO.pipe do |rd, wr| - headers = { - 'Content-Type' => 'application/octet-stream', - 'Transfer-Encoding' => 'chunked', - } - post = Net::HTTP::Post.new('/', headers) - th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } - post.body_stream = rd - http.request(post) do |res| - assert_equal 'text/plain', res['content-type'] - assert_equal 32, res.content_length - assert_equal exp, res.body - end - assert_nil th.value - end - end - end - end if RUBY_VERSION >= '2.5' - - def test_http10_proxy_chunked - # Testing HTTP/1.0 client request and HTTP/1.1 chunked response - # from origin server. - # +------+ - # V | - # client -------> proxy ---+ - # GET GET - # HTTP/1.0 HTTP/1.1 - # non-chunked chunked - # - proxy_handler_called = request_handler_called = 0 - config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1 }, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1 } - } - log_tester = lambda {|log, access_log| - log.reject! {|str| - %r{WARN chunked is set for an HTTP/1\.0 request\. \(ignored\)} =~ str - } - assert_equal([], log) - } - TestWEBrick.start_httpproxy(config, log_tester){|server, addr, port, log| - body = nil - server.mount_proc("/"){|req, res| - body = "#{req.request_method} #{req.path} #{req.body}" - res.chunked = true - res.body = -> (socket) { body.each_char {|c| socket.write c } } - } - - # Don't use Net::HTTP because it uses HTTP/1.1. - TCPSocket.open(addr, port) {|s| - s.write "GET / HTTP/1.0\r\nHost: localhost.localdomain\r\n\r\n" - response = s.read - assert_equal(body, response[/.*\z/]) - } - } - end - - def make_certificate(key, cn) - subject = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=#{cn}") - exts = [ - ["keyUsage", "keyEncipherment,digitalSignature", true], - ] - cert = OpenSSL::X509::Certificate.new - cert.version = 2 - cert.serial = 1 - cert.subject = subject - cert.issuer = subject - cert.public_key = key - cert.not_before = Time.now - 3600 - cert.not_after = Time.now + 3600 - ef = OpenSSL::X509::ExtensionFactory.new(cert, cert) - exts.each {|args| cert.add_extension(ef.create_extension(*args)) } - cert.sign(key, "sha256") - return cert - end if defined?(OpenSSL::SSL) - - def test_connect - # Testing CONNECT to proxy server - # - # client -----------> proxy -----------> https - # 1. CONNECT establish TCP - # 2. ---- establish SSL session ---> - # 3. ------- GET or POST ----------> - # - key = TEST_KEY_RSA2048 - cert = make_certificate(key, "127.0.0.1") - s_config = { - :SSLEnable =>true, - :ServerName => "localhost", - :SSLCertificate => cert, - :SSLPrivateKey => key, - } - config = { - :ServerName => "localhost.localdomain", - :RequestCallback => Proc.new{|req, res| - assert_equal("CONNECT", req.request_method) - }, - } - TestWEBrick.start_httpserver(s_config){|s_server, s_addr, s_port, s_log| - s_server.mount_proc("/"){|req, res| - res.body = "SSL #{req.request_method} #{req.path} #{req.body}" - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - http = Net::HTTP.new("127.0.0.1", s_port, addr, port) - http.use_ssl = true - http.verify_callback = Proc.new do |preverify_ok, store_ctx| - store_ctx.current_cert.to_der == cert.to_der - end - - req = Net::HTTP::Get.new("/") - req["Content-Type"] = "application/x-www-form-urlencoded" - http.request(req){|res| - assert_equal("SSL GET / ", res.body, s_log.call + log.call) - } - - req = Net::HTTP::Post.new("/") - req["Content-Type"] = "application/x-www-form-urlencoded" - req.body = "post-data" - http.request(req){|res| - assert_equal("SSL POST / post-data", res.body, s_log.call + log.call) - } - } - } - end if defined?(OpenSSL::SSL) - - def test_upstream_proxy - return if /mswin/ =~ RUBY_PLATFORM && ENV.key?('GITHUB_ACTIONS') # not working from the beginning - # Testing GET or POST through the upstream proxy server - # Note that the upstream proxy server works as the origin server. - # +------+ - # V | - # client -------> proxy -------> proxy ---+ - # GET / POST GET / POST GET / POST - # - up_proxy_handler_called = up_request_handler_called = 0 - proxy_handler_called = request_handler_called = 0 - up_config = { - :ServerName => "localhost.localdomain", - :ProxyContentHandler => Proc.new{|req, res| up_proxy_handler_called += 1}, - :RequestCallback => Proc.new{|req, res| up_request_handler_called += 1} - } - TestWEBrick.start_httpproxy(up_config){|up_server, up_addr, up_port, up_log| - up_server.mount_proc("/"){|req, res| - res.body = "#{req.request_method} #{req.path} #{req.body}" - } - config = { - :ServerName => "localhost.localdomain", - :ProxyURI => URI.parse("http://localhost:#{up_port}"), - :ProxyContentHandler => Proc.new{|req, res| proxy_handler_called += 1}, - :RequestCallback => Proc.new{|req, res| request_handler_called += 1}, - } - TestWEBrick.start_httpproxy(config){|server, addr, port, log| - http = Net::HTTP.new(up_addr, up_port, addr, port) - - req = Net::HTTP::Get.new("/") - http.request(req){|res| - skip res.message unless res.code == '200' - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_equal("GET / ", res.body) - } - assert_equal(1, up_proxy_handler_called, up_log.call + log.call) - assert_equal(2, up_request_handler_called, up_log.call + log.call) - assert_equal(1, proxy_handler_called, up_log.call + log.call) - assert_equal(1, request_handler_called, up_log.call + log.call) - - req = Net::HTTP::Head.new("/") - http.request(req){|res| - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_nil(res.body, up_log.call + log.call) - } - assert_equal(2, up_proxy_handler_called, up_log.call + log.call) - assert_equal(4, up_request_handler_called, up_log.call + log.call) - assert_equal(2, proxy_handler_called, up_log.call + log.call) - assert_equal(2, request_handler_called, up_log.call + log.call) - - req = Net::HTTP::Post.new("/") - req.body = "post-data" - req.content_type = "application/x-www-form-urlencoded" - http.request(req){|res| - via = res["via"].split(/,\s+/) - assert(via.include?("1.1 localhost.localdomain:#{up_port}"), up_log.call + log.call) - assert(via.include?("1.1 localhost.localdomain:#{port}"), up_log.call + log.call) - assert_equal("POST / post-data", res.body, up_log.call + log.call) - } - assert_equal(3, up_proxy_handler_called, up_log.call + log.call) - assert_equal(6, up_request_handler_called, up_log.call + log.call) - assert_equal(3, proxy_handler_called, up_log.call + log.call) - assert_equal(3, request_handler_called, up_log.call + log.call) - - if defined?(OpenSSL::SSL) - # Testing CONNECT to the upstream proxy server - # - # client -------> proxy -------> proxy -------> https - # 1. CONNECT CONNECT establish TCP - # 2. -------- establish SSL session ------> - # 3. ---------- GET or POST --------------> - # - key = TEST_KEY_RSA2048 - cert = make_certificate(key, "127.0.0.1") - s_config = { - :SSLEnable =>true, - :ServerName => "localhost", - :SSLCertificate => cert, - :SSLPrivateKey => key, - } - TestWEBrick.start_httpserver(s_config){|s_server, s_addr, s_port, s_log| - s_server.mount_proc("/"){|req2, res| - res.body = "SSL #{req2.request_method} #{req2.path} #{req2.body}" - } - http = Net::HTTP.new("127.0.0.1", s_port, addr, port, up_log.call + log.call + s_log.call) - http.use_ssl = true - http.verify_callback = Proc.new do |preverify_ok, store_ctx| - store_ctx.current_cert.to_der == cert.to_der - end - - req2 = Net::HTTP::Get.new("/") - http.request(req2){|res| - assert_equal("SSL GET / ", res.body, up_log.call + log.call + s_log.call) - } - - req2 = Net::HTTP::Post.new("/") - req2.body = "post-data" - req2.content_type = "application/x-www-form-urlencoded" - http.request(req2){|res| - assert_equal("SSL POST / post-data", res.body, up_log.call + log.call + s_log.call) - } - } - end - } - } - end - - if defined?(OpenSSL::SSL) - TEST_KEY_RSA2048 = OpenSSL::PKey.read <<-_end_of_pem_ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAuV9ht9J7k4NBs38jOXvvTKY9gW8nLICSno5EETR1cuF7i4pN -s9I1QJGAFAX0BEO4KbzXmuOvfCpD3CU+Slp1enenfzq/t/e/1IRW0wkJUJUFQign -4CtrkJL+P07yx18UjyPlBXb81ApEmAB5mrJVSrWmqbjs07JbuS4QQGGXLc+Su96D -kYKmSNVjBiLxVVSpyZfAY3hD37d60uG+X8xdW5v68JkRFIhdGlb6JL8fllf/A/bl -NwdJOhVr9mESHhwGjwfSeTDPfd8ZLE027E5lyAVX9KZYcU00mOX+fdxOSnGqS/8J -DRh0EPHDL15RcJjV2J6vZjPb0rOYGDoMcH+94wIDAQABAoIBAAzsamqfYQAqwXTb -I0CJtGg6msUgU7HVkOM+9d3hM2L791oGHV6xBAdpXW2H8LgvZHJ8eOeSghR8+dgq -PIqAffo4x1Oma+FOg3A0fb0evyiACyrOk+EcBdbBeLo/LcvahBtqnDfiUMQTpy6V -seSoFCwuN91TSCeGIsDpRjbG1vxZgtx+uI+oH5+ytqJOmfCksRDCkMglGkzyfcl0 -Xc5CUhIJ0my53xijEUQl19rtWdMnNnnkdbG8PT3LZlOta5Do86BElzUYka0C6dUc -VsBDQ0Nup0P6rEQgy7tephHoRlUGTYamsajGJaAo1F3IQVIrRSuagi7+YpSpCqsW -wORqorkCgYEA7RdX6MDVrbw7LePnhyuaqTiMK+055/R1TqhB1JvvxJ1CXk2rDL6G -0TLHQ7oGofd5LYiemg4ZVtWdJe43BPZlVgT6lvL/iGo8JnrncB9Da6L7nrq/+Rvj -XGjf1qODCK+LmreZWEsaLPURIoR/Ewwxb9J2zd0CaMjeTwafJo1CZvcCgYEAyCgb -aqoWvUecX8VvARfuA593Lsi50t4MEArnOXXcd1RnXoZWhbx5rgO8/ATKfXr0BK/n -h2GF9PfKzHFm/4V6e82OL7gu/kLy2u9bXN74vOvWFL5NOrOKPM7Kg+9I131kNYOw -Ivnr/VtHE5s0dY7JChYWE1F3vArrOw3T00a4CXUCgYEA0SqY+dS2LvIzW4cHCe9k -IQqsT0yYm5TFsUEr4sA3xcPfe4cV8sZb9k/QEGYb1+SWWZ+AHPV3UW5fl8kTbSNb -v4ng8i8rVVQ0ANbJO9e5CUrepein2MPL0AkOATR8M7t7dGGpvYV0cFk8ZrFx0oId -U0PgYDotF/iueBWlbsOM430CgYEAqYI95dFyPI5/AiSkY5queeb8+mQH62sdcCCr -vd/w/CZA/K5sbAo4SoTj8dLk4evU6HtIa0DOP63y071eaxvRpTNqLUOgmLh+D6gS -Cc7TfLuFrD+WDBatBd5jZ+SoHccVrLR/4L8jeodo5FPW05A+9gnKXEXsTxY4LOUC -9bS4e1kCgYAqVXZh63JsMwoaxCYmQ66eJojKa47VNrOeIZDZvd2BPVf30glBOT41 -gBoDG3WMPZoQj9pb7uMcrnvs4APj2FIhMU8U15LcPAj59cD6S6rWnAxO8NFK7HQG -4Jxg3JNNf8ErQoCHb1B3oVdXJkmbJkARoDpBKmTCgKtP8ADYLmVPQw== ------END RSA PRIVATE KEY----- - _end_of_pem_ - end -end diff --git a/tool/test/webrick/test_httprequest.rb b/tool/test/webrick/test_httprequest.rb deleted file mode 100644 index 759ccbdada..0000000000 --- a/tool/test/webrick/test_httprequest.rb +++ /dev/null @@ -1,488 +0,0 @@ -# frozen_string_literal: false -require "webrick" -require "stringio" -require "test/unit" - -class TestWEBrickHTTPRequest < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_simple_request - msg = <<-_end_of_message_ -GET / - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert(req.meta_vars) # fails if @header was not initialized and iteration is attempted on the nil reference - end - - def test_parse_09 - msg = <<-_end_of_message_ - GET / - foobar # HTTP/0.9 request don't have header nor entity body. - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/", req.unparsed_uri) - assert_equal(WEBrick::HTTPVersion.new("0.9"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(false, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_parse_10 - msg = <<-_end_of_message_ - GET / HTTP/1.0 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/", req.unparsed_uri) - assert_equal(WEBrick::HTTPVersion.new("1.0"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(false, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_parse_11 - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("GET", req.request_method) - assert_equal("/path", req.unparsed_uri) - assert_equal("", req.script_name) - assert_equal("/path", req.path_info) - assert_equal(WEBrick::HTTPVersion.new("1.1"), req.http_version) - assert_equal(WEBrick::Config::HTTP[:ServerName], req.host) - assert_equal(80, req.port) - assert_equal(true, req.keep_alive?) - assert_equal(nil, req.body) - assert(req.query.empty?) - end - - def test_request_uri_too_large - msg = <<-_end_of_message_ - GET /#{"a"*2084} HTTP/1.1 - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - assert_raise(WEBrick::HTTPStatus::RequestURITooLarge){ - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - } - end - - def test_parse_headers - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: test.ruby-lang.org:8080 - Connection: close - Accept: text/*;q=0.3, text/html;q=0.7, text/html;level=1, - text/html;level=2;q=0.4, */*;q=0.5 - Accept-Encoding: compress;q=0.5 - Accept-Encoding: gzip;q=1.0, identity; q=0.4, *;q=0 - Accept-Language: en;q=0.5, *; q=0 - Accept-Language: ja - Content-Type: text/plain - Content-Length: 7 - X-Empty-Header: - - foobar - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal( - URI.parse("http://test.ruby-lang.org:8080/path"), req.request_uri) - assert_equal("test.ruby-lang.org", req.host) - assert_equal(8080, req.port) - assert_equal(false, req.keep_alive?) - assert_equal( - %w(text/html;level=1 text/html */* text/html;level=2 text/*), - req.accept) - assert_equal(%w(gzip compress identity *), req.accept_encoding) - assert_equal(%w(ja en *), req.accept_language) - assert_equal(7, req.content_length) - assert_equal("text/plain", req.content_type) - assert_equal("foobar\n", req.body) - assert_equal("", req["x-empty-header"]) - assert_equal(nil, req["x-no-header"]) - assert(req.query.empty?) - end - - def test_parse_header2() - msg = <<-_end_of_message_ - POST /foo/bar/../baz?q=a HTTP/1.0 - Content-Length: 9 - User-Agent: - FOO BAR - BAZ - - hogehoge - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal("POST", req.request_method) - assert_equal("/foo/baz", req.path) - assert_equal("", req.script_name) - assert_equal("/foo/baz", req.path_info) - assert_equal("9", req['content-length']) - assert_equal("FOO BAR BAZ", req['user-agent']) - assert_equal("hogehoge\n", req.body) - end - - def test_parse_headers3 - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: test.ruby-lang.org - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://test.ruby-lang.org/path"), req.request_uri) - assert_equal("test.ruby-lang.org", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: 192.168.1.1 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://192.168.1.1/path"), req.request_uri) - assert_equal("192.168.1.1", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: [fe80::208:dff:feef:98c7] - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://[fe80::208:dff:feef:98c7]/path"), - req.request_uri) - assert_equal("[fe80::208:dff:feef:98c7]", req.host) - assert_equal(80, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: 192.168.1.1:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://192.168.1.1:8080/path"), req.request_uri) - assert_equal("192.168.1.1", req.host) - assert_equal(8080, req.port) - - msg = <<-_end_of_message_ - GET /path HTTP/1.1 - Host: [fe80::208:dff:feef:98c7]:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - assert_equal(URI.parse("http://[fe80::208:dff:feef:98c7]:8080/path"), - req.request_uri) - assert_equal("[fe80::208:dff:feef:98c7]", req.host) - assert_equal(8080, req.port) - end - - def test_parse_get_params - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - GET /path?#{param} HTTP/1.1 - Host: test.ruby-lang.org:8080 - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - query = req.query - assert_equal("1", query["foo"]) - assert_equal(["1", "2", "3"], query["foo"].to_ary) - assert_equal(["1", "2", "3"], query["foo"].list) - assert_equal("x", query["bar"]) - assert_equal(["x"], query["bar"].list) - end - - def test_parse_post_params - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Length: #{param.size} - Content-Type: application/x-www-form-urlencoded - - #{param} - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - query = req.query - assert_equal("1", query["foo"]) - assert_equal(["1", "2", "3"], query["foo"].to_ary) - assert_equal(["1", "2", "3"], query["foo"].list) - assert_equal("x", query["bar"]) - assert_equal(["x"], query["bar"].list) - end - - def test_chunked - crlf = "\x0d\x0a" - expect = File.binread(__FILE__).freeze - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - Host: test.ruby-lang.org:8080 - Transfer-Encoding: chunked - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - open(__FILE__){|io| - while chunk = io.read(100) - msg << chunk.size.to_s(16) << crlf - msg << chunk << crlf - end - } - msg << "0" << crlf - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal(expect, req.body) - - # chunked req.body_reader - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - dst = StringIO.new - IO.copy_stream(req.body_reader, dst) - assert_equal(expect, dst.string) - end - - def test_forwarded - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - User-Agent: w3m/0.5.2 - X-Forwarded-For: 123.123.123.123 - X-Forwarded-Host: forward.example.com - X-Forwarded-Server: server.example.com - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("http://forward.example.com/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(80, req.port) - assert_equal("123.123.123.123", req.remote_ip) - assert(!req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - User-Agent: w3m/0.5.2 - X-Forwarded-For: 192.168.1.10, 172.16.1.1, 123.123.123.123 - X-Forwarded-Host: forward.example.com:8080 - X-Forwarded-Server: server.example.com - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("http://forward.example.com:8080/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(8080, req.port) - assert_equal("123.123.123.123", req.remote_ip) - assert(!req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https, http - X-Forwarded-For: 192.168.1.10, 10.0.0.1, 123.123.123.123 - X-Forwarded-Host: forward.example.com - X-Forwarded-Server: server.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server.example.com", req.server_name) - assert_equal("https://forward.example.com/foo", req.request_uri.to_s) - assert_equal("forward.example.com", req.host) - assert_equal(443, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: forward1.example.com:1234, forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://forward1.example.com:1234/foo", req.request_uri.to_s) - assert_equal("forward1.example.com", req.host) - assert_equal(1234, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: [fd20:8b1e:b255:8154:250:56ff:fea8:4d84], forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]/foo", req.request_uri.to_s) - assert_equal("[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]", req.host) - assert_equal(443, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - - msg = <<-_end_of_message_ - GET /foo HTTP/1.1 - Host: localhost:10080 - Client-IP: 234.234.234.234 - X-Forwarded-Proto: https - X-Forwarded-For: 192.168.1.10 - X-Forwarded-Host: [fd20:8b1e:b255:8154:250:56ff:fea8:4d84]:1234, forward2.example.com:5678 - X-Forwarded-Server: server1.example.com, server2.example.com - X-Requested-With: XMLHttpRequest - Connection: Keep-Alive - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert_equal("server1.example.com", req.server_name) - assert_equal("https://[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]:1234/foo", req.request_uri.to_s) - assert_equal("[fd20:8b1e:b255:8154:250:56ff:fea8:4d84]", req.host) - assert_equal(1234, req.port) - assert_equal("234.234.234.234", req.remote_ip) - assert(req.ssl?) - end - - def test_continue_sent - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - Expect: 100-continue - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert req['expect'] - l = msg.size - req.continue - assert_not_equal l, msg.size - assert_match(/HTTP\/1.1 100 continue\r\n\r\n\z/, msg) - assert !req['expect'] - end - - def test_continue_not_sent - msg = <<-_end_of_message_ - POST /path HTTP/1.1 - - _end_of_message_ - msg.gsub!(/^ {6}/, "") - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg)) - assert !req['expect'] - l = msg.size - req.continue - assert_equal l, msg.size - end - - def test_empty_post - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Type: application/x-www-form-urlencoded - - _end_of_message_ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - end - - def test_bad_messages - param = "foo=1;foo=2;foo=3;bar=x" - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Type: application/x-www-form-urlencoded - - #{param} - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::LengthRequired){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Content-Length: 100000 - - body is too short. - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::BadRequest){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - - msg = <<-_end_of_message_ - POST /path?foo=x;foo=y;foo=z;bar=1 HTTP/1.1 - Host: test.ruby-lang.org:8080 - Transfer-Encoding: foobar - - body is too short. - _end_of_message_ - assert_raise(WEBrick::HTTPStatus::NotImplemented){ - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new(msg.gsub(/^ {6}/, ""))) - req.body - } - end - - def test_eof_raised_when_line_is_nil - assert_raise(WEBrick::HTTPStatus::EOFError) { - req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) - req.parse(StringIO.new("")) - } - end -end diff --git a/tool/test/webrick/test_httpresponse.rb b/tool/test/webrick/test_httpresponse.rb deleted file mode 100644 index 4410f63e89..0000000000 --- a/tool/test/webrick/test_httpresponse.rb +++ /dev/null @@ -1,282 +0,0 @@ -# frozen_string_literal: false -require "webrick" -require "test/unit" -require "stringio" -require "net/http" - -module WEBrick - class TestHTTPResponse < Test::Unit::TestCase - class FakeLogger - attr_reader :messages - - def initialize - @messages = [] - end - - def warn msg - @messages << msg - end - end - - attr_reader :config, :logger, :res - - def setup - super - @logger = FakeLogger.new - @config = Config::HTTP - @config[:Logger] = logger - @res = HTTPResponse.new config - @res.keep_alive = true - end - - def test_prevent_response_splitting_headers_crlf - res['X-header'] = "malicious\r\nCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_crlf - user_input = "malicious\r\nCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_headers_cr - res['X-header'] = "malicious\rCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_cr - user_input = "malicious\rCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_headers_lf - res['X-header'] = "malicious\nCookie: cracked_indicator_for_test" - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_prevent_response_splitting_cookie_headers_lf - user_input = "malicious\nCookie: cracked_indicator_for_test" - res.cookies << WEBrick::Cookie.new('author', user_input) - io = StringIO.new - res.send_response io - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '500', res.code - refute_match 'cracked_indicator_for_test', io.string - end - - def test_set_redirect_response_splitting - url = "malicious\r\nCookie: cracked_indicator_for_test" - assert_raise(URI::InvalidURIError) do - res.set_redirect(WEBrick::HTTPStatus::MultipleChoices, url) - end - end - - def test_set_redirect_html_injection - url = 'http://example.com////?a</a><head></head><body><img src=1></body>' - assert_raise(WEBrick::HTTPStatus::MultipleChoices) do - res.set_redirect(WEBrick::HTTPStatus::MultipleChoices, url) - end - res.status = 300 - io = StringIO.new - res.send_response(io) - io.rewind - res = Net::HTTPResponse.read_new(Net::BufferedIO.new(io)) - assert_equal '300', res.code - refute_match(/<img/, io.string) - end - - def test_304_does_not_log_warning - res.status = 304 - res.setup_header - assert_equal 0, logger.messages.length - end - - def test_204_does_not_log_warning - res.status = 204 - res.setup_header - - assert_equal 0, logger.messages.length - end - - def test_1xx_does_not_log_warnings - res.status = 105 - res.setup_header - - assert_equal 0, logger.messages.length - end - - def test_200_chunked_does_not_set_content_length - res.chunked = false - res["Transfer-Encoding"] = 'chunked' - res.setup_header - assert_nil res.header.fetch('content-length', nil) - end - - def test_send_body_io - IO.pipe {|body_r, body_w| - body_w.write 'hello' - body_w.close - - @res.body = body_r - - IO.pipe {|r, w| - - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string - @res.body = 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_io - @res.body = StringIO.new 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - assert_equal 'hello', r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_io_chunked - @res.chunked = true - - IO.pipe {|body_r, body_w| - - body_w.write 'hello' - body_w.close - - @res.body = body_r - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_chunked - @res.chunked = true - - @res.body = 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_string_io_chunked - @res.chunked = true - - @res.body = StringIO.new 'hello' - - IO.pipe {|r, w| - @res.send_body w - - w.close - - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - } - assert_equal 0, logger.messages.length - end - - def test_send_body_proc - @res.body = Proc.new { |out| out.write('hello') } - IO.pipe do |r, w| - @res.send_body(w) - w.close - r.binmode - assert_equal 'hello', r.read - end - assert_equal 0, logger.messages.length - end - - def test_send_body_proc_chunked - @res.body = Proc.new { |out| out.write('hello') } - @res.chunked = true - IO.pipe do |r, w| - @res.send_body(w) - w.close - r.binmode - assert_equal "5\r\nhello\r\n0\r\n\r\n", r.read - end - assert_equal 0, logger.messages.length - end - - def test_set_error - status = 400 - message = 'missing attribute' - @res.status = status - error = WEBrick::HTTPStatus[status].new(message) - body = @res.set_error(error) - assert_match(/#{@res.reason_phrase}/, body) - assert_match(/#{message}/, body) - end - - def test_no_extraneous_space - [200, 300, 400, 500].each do |status| - @res.status = status - assert_match(/\S\r\n/, @res.status_line) - end - end - end -end diff --git a/tool/test/webrick/test_https.rb b/tool/test/webrick/test_https.rb deleted file mode 100644 index ec0aac354a..0000000000 --- a/tool/test/webrick/test_https.rb +++ /dev/null @@ -1,112 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require "webrick/https" -require "webrick/utils" -require_relative "utils" - -class TestWEBrickHTTPS < Test::Unit::TestCase - empty_log = Object.new - def empty_log.<<(str) - assert_equal('', str) - self - end - NoLog = WEBrick::Log.new(empty_log, WEBrick::BasicLog::WARN) - - class HTTPSNITest < ::Net::HTTP - attr_accessor :sni_hostname - - def ssl_socket_connect(s, timeout) - s.hostname = sni_hostname - super - end - end - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def https_get(addr, port, hostname, path, verifyname = nil) - subject = nil - http = HTTPSNITest.new(addr, port) - http.use_ssl = true - http.verify_mode = OpenSSL::SSL::VERIFY_NONE - http.verify_callback = proc { |x, store| subject = store.chain[0].subject.to_s; x } - http.sni_hostname = hostname - req = Net::HTTP::Get.new(path) - req["Host"] = "#{hostname}:#{port}" - response = http.start { http.request(req).body } - assert_equal("/CN=#{verifyname || hostname}", subject) - response - end - - def test_sni - config = { - :ServerName => "localhost", - :SSLEnable => true, - :SSLCertName => "/CN=localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc("/") {|req, res| res.body = "master" } - - # catch stderr in create_self_signed_cert - stderr_buffer = StringIO.new - old_stderr, $stderr = $stderr, stderr_buffer - - begin - vhost_config1 = { - :ServerName => "vhost1", - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :SSLEnable => true, - :SSLCertName => "/CN=vhost1", - } - vhost1 = WEBrick::HTTPServer.new(vhost_config1) - vhost1.mount_proc("/") {|req, res| res.body = "vhost1" } - server.virtual_host(vhost1) - - vhost_config2 = { - :ServerName => "vhost2", - :ServerAlias => ["vhost2alias"], - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :SSLEnable => true, - :SSLCertName => "/CN=vhost2", - } - vhost2 = WEBrick::HTTPServer.new(vhost_config2) - vhost2.mount_proc("/") {|req, res| res.body = "vhost2" } - server.virtual_host(vhost2) - ensure - # restore stderr - $stderr = old_stderr - end - - assert_match(/\A([.+*]+\n)+\z/, stderr_buffer.string) - assert_equal("master", https_get(addr, port, "localhost", "/localhost")) - assert_equal("master", https_get(addr, port, "unknown", "/unknown", "localhost")) - assert_equal("vhost1", https_get(addr, port, "vhost1", "/vhost1")) - assert_equal("vhost2", https_get(addr, port, "vhost2", "/vhost2")) - assert_equal("vhost2", https_get(addr, port, "vhost2alias", "/vhost2alias", "vhost2")) - } - end - - def test_check_ssl_virtual - config = { - :ServerName => "localhost", - :SSLEnable => true, - :SSLCertName => "/CN=localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - assert_raise ArgumentError do - vhost = WEBrick::HTTPServer.new({:DoNotListen => true, :Logger => NoLog}) - server.virtual_host(vhost) - end - } - end -end diff --git a/tool/test/webrick/test_httpserver.rb b/tool/test/webrick/test_httpserver.rb deleted file mode 100644 index f905ff04e8..0000000000 --- a/tool/test/webrick/test_httpserver.rb +++ /dev/null @@ -1,543 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "net/http" -require "webrick" -require_relative "utils" - -class TestWEBrickHTTPServer < Test::Unit::TestCase - empty_log = Object.new - def empty_log.<<(str) - assert_equal('', str) - self - end - NoLog = WEBrick::Log.new(empty_log, WEBrick::BasicLog::WARN) - - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def test_mount - httpd = WEBrick::HTTPServer.new( - :Logger => NoLog, - :DoNotListen=>true - ) - httpd.mount("/", :Root) - httpd.mount("/foo", :Foo) - httpd.mount("/foo/bar", :Bar, :bar1) - httpd.mount("/foo/bar/baz", :Baz, :baz1, :baz2) - - serv, opts, script_name, path_info = httpd.search_servlet("/") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/sub") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/sub", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/sub/") - assert_equal(:Root, serv) - assert_equal([], opts) - assert_equal("", script_name) - assert_equal("/sub/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("/", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/sub") - assert_equal(:Foo, serv) - assert_equal([], opts) - assert_equal("/foo", script_name) - assert_equal("/sub", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/bar") - assert_equal(:Bar, serv) - assert_equal([:bar1], opts) - assert_equal("/foo/bar", script_name) - assert_equal("", path_info) - - serv, opts, script_name, path_info = httpd.search_servlet("/foo/bar/baz") - assert_equal(:Baz, serv) - assert_equal([:baz1, :baz2], opts) - assert_equal("/foo/bar/baz", script_name) - assert_equal("", path_info) - end - - class Req - attr_reader :port, :host - def initialize(addr, port, host) - @addr, @port, @host = addr, port, host - end - def addr - [0,0,0,@addr] - end - end - - def httpd(addr, port, host, ali) - config ={ - :Logger => NoLog, - :DoNotListen => true, - :BindAddress => addr, - :Port => port, - :ServerName => host, - :ServerAlias => ali, - } - return WEBrick::HTTPServer.new(config) - end - - def assert_eql?(v1, v2) - assert_equal(v1.object_id, v2.object_id) - end - - def test_lookup_server - addr1 = "192.168.100.1" - addr2 = "192.168.100.2" - addrz = "192.168.100.254" - local = "127.0.0.1" - port1 = 80 - port2 = 8080 - port3 = 10080 - portz = 32767 - name1 = "www.example.com" - name2 = "www2.example.com" - name3 = "www3.example.com" - namea = "www.example.co.jp" - nameb = "www.example.jp" - namec = "www2.example.co.jp" - named = "www2.example.jp" - namez = "foobar.example.com" - alias1 = [namea, nameb] - alias2 = [namec, named] - - host1 = httpd(nil, port1, name1, nil) - hosts = [ - host2 = httpd(addr1, port1, name1, nil), - host3 = httpd(addr1, port1, name2, alias1), - host4 = httpd(addr1, port2, name1, nil), - host5 = httpd(addr1, port2, name2, alias1), - httpd(addr1, port2, name3, alias2), - host7 = httpd(addr2, nil, name1, nil), - host8 = httpd(addr2, nil, name2, alias1), - httpd(addr2, nil, name3, alias2), - host10 = httpd(local, nil, nil, nil), - host11 = httpd(nil, port3, nil, nil), - ].sort_by{ rand } - hosts.each{|h| host1.virtual_host(h) } - - # connect to addr1 - assert_eql?(host2, host1.lookup_server(Req.new(addr1, port1, name1))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, name2))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, namea))) - assert_eql?(host3, host1.lookup_server(Req.new(addr1, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, port1, namez))) - assert_eql?(host4, host1.lookup_server(Req.new(addr1, port2, name1))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, name2))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, namea))) - assert_eql?(host5, host1.lookup_server(Req.new(addr1, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, port2, namez))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, name1))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, name2))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, namea))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addr1, port3, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr1, portz, namez))) - - # connect to addr2 - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port1, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, port1, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port2, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, port2, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, port3, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addr2, port3, namez))) - assert_eql?(host7, host1.lookup_server(Req.new(addr2, portz, name1))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, name2))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, namea))) - assert_eql?(host8, host1.lookup_server(Req.new(addr2, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addr2, portz, namez))) - - # connect to addrz - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port1, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, port2, namez))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, name1))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, name2))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, namea))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, nameb))) - assert_eql?(host11, host1.lookup_server(Req.new(addrz, port3, namez))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, name1))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, name2))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, namea))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, nameb))) - assert_eql?(nil, host1.lookup_server(Req.new(addrz, portz, namez))) - - # connect to localhost - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port1, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port2, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, port3, namez))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, name1))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, name2))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, namea))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, nameb))) - assert_eql?(host10, host1.lookup_server(Req.new(local, portz, namez))) - end - - def test_callbacks - accepted = started = stopped = 0 - requested0 = requested1 = 0 - config = { - :ServerName => "localhost", - :AcceptCallback => Proc.new{ accepted += 1 }, - :StartCallback => Proc.new{ started += 1 }, - :StopCallback => Proc.new{ stopped += 1 }, - :RequestCallback => Proc.new{|req, res| requested0 += 1 }, - } - log_tester = lambda {|log, access_log| - assert(log.find {|s| %r{ERROR `/' not found\.} =~ s }) - assert_equal([], log.reject {|s| %r{ERROR `/' not found\.} =~ s }) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - vhost_config = { - :ServerName => "myhostname", - :BindAddress => addr, - :Port => port, - :DoNotListen => true, - :Logger => NoLog, - :AccessLog => [], - :RequestCallback => Proc.new{|req, res| requested1 += 1 }, - } - server.virtual_host(WEBrick::HTTPServer.new(vhost_config)) - - Thread.pass while server.status != :Running - sleep 1 if defined?(RubyVM::JIT) && RubyVM::JIT.enabled? # server.status behaves unexpectedly with --jit-wait - assert_equal(1, started, log.call) - assert_equal(0, stopped, log.call) - assert_equal(0, accepted, log.call) - - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req["Host"] = "myhostname:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - req["Host"] = "localhost:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - http.request(req){|res| assert_equal("404", res.code, log.call)} - assert_equal(6, accepted, log.call) - assert_equal(3, requested0, log.call) - assert_equal(3, requested1, log.call) - } - assert_equal(started, 1) - assert_equal(stopped, 1) - end - - class CustomRequest < ::WEBrick::HTTPRequest; end - class CustomResponse < ::WEBrick::HTTPResponse; end - class CustomServer < ::WEBrick::HTTPServer - def create_request(config) - CustomRequest.new(config) - end - - def create_response(config) - CustomResponse.new(config) - end - end - - def test_custom_server_request_and_response - config = { :ServerName => "localhost" } - TestWEBrick.start_server(CustomServer, config){|server, addr, port, log| - server.mount_proc("/", lambda {|req, res| - assert_kind_of(CustomRequest, req) - assert_kind_of(CustomResponse, res) - res.body = "via custom response" - }) - Thread.pass while server.status != :Running - - Net::HTTP.start(addr, port) do |http| - req = Net::HTTP::Get.new("/") - http.request(req){|res| - assert_equal("via custom response", res.body) - } - server.shutdown - end - } - end - - # This class is needed by test_response_io_with_chunked_set method - class EventManagerForChunkedResponseTest - def initialize - @listeners = [] - end - def add_listener( &block ) - @listeners << block - end - def raise_str_event( str ) - @listeners.each{ |e| e.call( :str, str ) } - end - def raise_close_event() - @listeners.each{ |e| e.call( :cls ) } - end - end - def test_response_io_with_chunked_set - evt_man = EventManagerForChunkedResponseTest.new - t = Thread.new do - begin - config = { - :ServerName => "localhost" - } - TestWEBrick.start_httpserver(config) do |server, addr, port, log| - body_strs = [ 'aaaaaa', 'bb', 'cccc' ] - server.mount_proc( "/", ->( req, res ){ - # Test for setting chunked... - res.chunked = true - r,w = IO.pipe - evt_man.add_listener do |type,str| - type == :cls ? ( w.close ) : ( w << str ) - end - res.body = r - } ) - Thread.pass while server.status != :Running - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - http.request(req) do |res| - i = 0 - evt_man.raise_str_event( body_strs[i] ) - res.read_body do |s| - assert_equal( body_strs[i], s ) - i += 1 - if i < body_strs.length - evt_man.raise_str_event( body_strs[i] ) - else - evt_man.raise_close_event() - end - end - assert_equal( body_strs.length, i ) - end - end - rescue => err - flunk( 'exception raised in thread: ' + err.to_s ) - end - end - if t.join( 3 ).nil? - evt_man.raise_close_event() - flunk( 'timeout' ) - if t.join( 1 ).nil? - Thread.kill t - end - end - end - - def test_response_io_without_chunked_set - config = { - :ServerName => "localhost" - } - log_tester = lambda {|log, access_log| - assert_equal(1, log.length) - assert_match(/WARN Could not determine content-length of response body./, log[0]) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - server.mount_proc("/", lambda { |req, res| - r,w = IO.pipe - # Test for not setting chunked... - # res.chunked = true - res.body = r - w << "foo" - w.close - }) - Thread.pass while server.status != :Running - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req['Connection'] = 'Keep-Alive' - begin - Timeout.timeout(2) do - http.request(req){|res| assert_equal("foo", res.body) } - end - rescue Timeout::Error - flunk('corrupted response') - end - } - end - - def test_request_handler_callback_is_deprecated - requested = 0 - config = { - :ServerName => "localhost", - :RequestHandler => Proc.new{|req, res| requested += 1 }, - } - log_tester = lambda {|log, access_log| - assert_equal(2, log.length) - assert_match(/WARN :RequestHandler is deprecated, please use :RequestCallback/, log[0]) - assert_match(%r{ERROR `/' not found\.}, log[1]) - } - TestWEBrick.start_httpserver(config, log_tester){|server, addr, port, log| - Thread.pass while server.status != :Running - - http = Net::HTTP.new(addr, port) - req = Net::HTTP::Get.new("/") - req["Host"] = "localhost:#{port}" - http.request(req){|res| assert_equal("404", res.code, log.call)} - assert_match(%r{:RequestHandler is deprecated, please use :RequestCallback$}, log.call, log.call) - } - assert_equal(1, requested) - end - - def test_shutdown_with_busy_keepalive_connection - requested = 0 - config = { - :ServerName => "localhost", - } - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc("/", lambda {|req, res| res.body = "heffalump" }) - Thread.pass while server.status != :Running - - Net::HTTP.start(addr, port) do |http| - req = Net::HTTP::Get.new("/") - http.request(req){|res| assert_equal('Keep-Alive', res['Connection'], log.call) } - server.shutdown - begin - 10.times {|n| http.request(req); requested += 1 } - rescue - # Errno::ECONNREFUSED or similar - end - end - } - assert_equal(0, requested, "Server responded to #{requested} requests after shutdown") - end - - def test_cntrl_in_path - log_ary = [] - access_log_ary = [] - config = { - :Port => 0, - :BindAddress => '127.0.0.1', - :Logger => WEBrick::Log.new(log_ary, WEBrick::BasicLog::WARN), - :AccessLog => [[access_log_ary, '']], - } - s = WEBrick::HTTPServer.new(config) - s.mount('/foo', WEBrick::HTTPServlet::FileHandler, __FILE__) - th = Thread.new { s.start } - addr = s.listeners[0].addr - - http = Net::HTTP.new(addr[3], addr[1]) - req = Net::HTTP::Get.new('/notexist%0a/foo') - http.request(req) { |res| assert_equal('404', res.code) } - exp = %Q(ERROR `/notexist\\n/foo' not found.\n) - assert_equal 1, log_ary.size - assert_include log_ary[0], exp - ensure - s&.shutdown - th&.join - end - - def test_gigantic_request_header - log_tester = lambda {|log, access_log| - assert_equal 1, log.size - assert_include log[0], 'ERROR headers too large' - } - TestWEBrick.start_httpserver({}, log_tester){|server, addr, port, log| - server.mount('/', WEBrick::HTTPServlet::FileHandler, __FILE__) - TCPSocket.open(addr, port) do |c| - c.write("GET / HTTP/1.0\r\n") - junk = -"X-Junk: #{' ' * 1024}\r\n" - assert_raise(Errno::ECONNRESET, Errno::EPIPE, Errno::EPROTOTYPE) do - loop { c.write(junk) } - end - end - } - end - - def test_eof_in_chunk - log_tester = lambda do |log, access_log| - assert_equal 1, log.size - assert_include log[0], 'ERROR bad chunk data size' - end - TestWEBrick.start_httpserver({}, log_tester){|server, addr, port, log| - server.mount_proc('/', ->(req, res) { res.body = req.body }) - TCPSocket.open(addr, port) do |c| - c.write("POST / HTTP/1.1\r\nHost: example.com\r\n" \ - "Transfer-Encoding: chunked\r\n\r\n5\r\na") - c.shutdown(Socket::SHUT_WR) # trigger EOF in server - res = c.read - assert_match %r{\AHTTP/1\.1 400 }, res - end - } - end - - def test_big_chunks - nr_out = 3 - buf = 'big' # 3 bytes is bigger than 2! - config = { :InputBufferSize => 2 }.freeze - total = 0 - all = '' - TestWEBrick.start_httpserver(config){|server, addr, port, log| - server.mount_proc('/', ->(req, res) { - err = [] - ret = req.body do |chunk| - n = chunk.bytesize - n > config[:InputBufferSize] and err << "#{n} > :InputBufferSize" - total += n - all << chunk - end - ret.nil? or err << 'req.body should return nil' - (buf * nr_out) == all or err << 'input body does not match expected' - res.header['connection'] = 'close' - res.body = err.join("\n") - }) - TCPSocket.open(addr, port) do |c| - c.write("POST / HTTP/1.1\r\nHost: example.com\r\n" \ - "Transfer-Encoding: chunked\r\n\r\n") - chunk = "#{buf.bytesize.to_s(16)}\r\n#{buf}\r\n" - nr_out.times { c.write(chunk) } - c.write("0\r\n\r\n") - head, body = c.read.split("\r\n\r\n") - assert_match %r{\AHTTP/1\.1 200 OK}, head - assert_nil body - end - } - end -end diff --git a/tool/test/webrick/test_httpstatus.rb b/tool/test/webrick/test_httpstatus.rb deleted file mode 100644 index fd0570d5c6..0000000000 --- a/tool/test/webrick/test_httpstatus.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick" - -class TestWEBrickHTTPStatus < Test::Unit::TestCase - def test_info? - assert WEBrick::HTTPStatus.info?(100) - refute WEBrick::HTTPStatus.info?(200) - end - - def test_success? - assert WEBrick::HTTPStatus.success?(200) - refute WEBrick::HTTPStatus.success?(300) - end - - def test_redirect? - assert WEBrick::HTTPStatus.redirect?(300) - refute WEBrick::HTTPStatus.redirect?(400) - end - - def test_error? - assert WEBrick::HTTPStatus.error?(400) - refute WEBrick::HTTPStatus.error?(600) - end - - def test_client_error? - assert WEBrick::HTTPStatus.client_error?(400) - refute WEBrick::HTTPStatus.client_error?(500) - end - - def test_server_error? - assert WEBrick::HTTPStatus.server_error?(500) - refute WEBrick::HTTPStatus.server_error?(600) - end -end diff --git a/tool/test/webrick/test_httputils.rb b/tool/test/webrick/test_httputils.rb deleted file mode 100644 index 00f297bd09..0000000000 --- a/tool/test/webrick/test_httputils.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/httputils" - -class TestWEBrickHTTPUtils < Test::Unit::TestCase - include WEBrick::HTTPUtils - - def test_normilize_path - assert_equal("/foo", normalize_path("/foo")) - assert_equal("/foo/bar/", normalize_path("/foo/bar/")) - - assert_equal("/", normalize_path("/foo/../")) - assert_equal("/", normalize_path("/foo/..")) - assert_equal("/", normalize_path("/foo/bar/../../")) - assert_equal("/", normalize_path("/foo/bar/../..")) - assert_equal("/", normalize_path("/foo/bar/../..")) - assert_equal("/baz", normalize_path("/foo/bar/../../baz")) - assert_equal("/baz", normalize_path("/foo/../bar/../baz")) - assert_equal("/baz/", normalize_path("/foo/../bar/../baz/")) - assert_equal("/...", normalize_path("/bar/../...")) - assert_equal("/.../", normalize_path("/bar/../.../")) - - assert_equal("/foo/", normalize_path("/foo/./")) - assert_equal("/foo/", normalize_path("/foo/.")) - assert_equal("/foo/", normalize_path("/foo/././")) - assert_equal("/foo/", normalize_path("/foo/./.")) - assert_equal("/foo/bar", normalize_path("/foo/./bar")) - assert_equal("/foo/bar/", normalize_path("/foo/./bar/.")) - assert_equal("/foo/bar/", normalize_path("/./././foo/./bar/.")) - - assert_equal("/foo/bar/", normalize_path("//foo///.//bar/.///.//")) - assert_equal("/", normalize_path("//foo///..///bar/.///..//.//")) - - assert_raise(RuntimeError){ normalize_path("foo/bar") } - assert_raise(RuntimeError){ normalize_path("..") } - assert_raise(RuntimeError){ normalize_path("/..") } - assert_raise(RuntimeError){ normalize_path("/./..") } - assert_raise(RuntimeError){ normalize_path("/./../") } - assert_raise(RuntimeError){ normalize_path("/./../..") } - assert_raise(RuntimeError){ normalize_path("/./../../") } - assert_raise(RuntimeError){ normalize_path("/./../") } - assert_raise(RuntimeError){ normalize_path("/../..") } - assert_raise(RuntimeError){ normalize_path("/../../") } - assert_raise(RuntimeError){ normalize_path("/../../..") } - assert_raise(RuntimeError){ normalize_path("/../../../") } - assert_raise(RuntimeError){ normalize_path("/../foo/../") } - assert_raise(RuntimeError){ normalize_path("/../foo/../../") } - assert_raise(RuntimeError){ normalize_path("/foo/bar/../../../../") } - assert_raise(RuntimeError){ normalize_path("/foo/../bar/../../") } - assert_raise(RuntimeError){ normalize_path("/./../bar/") } - assert_raise(RuntimeError){ normalize_path("/./../") } - end - - def test_split_header_value - assert_equal(['foo', 'bar'], split_header_value('foo, bar')) - assert_equal(['"foo"', 'bar'], split_header_value('"foo", bar')) - assert_equal(['foo', '"bar"'], split_header_value('foo, "bar"')) - assert_equal(['*'], split_header_value('*')) - assert_equal(['W/"xyzzy"', 'W/"r2d2xxxx"', 'W/"c3piozzzz"'], - split_header_value('W/"xyzzy", W/"r2d2xxxx", W/"c3piozzzz"')) - end - - def test_escape - assert_equal("/foo/bar", escape("/foo/bar")) - assert_equal("/~foo/bar", escape("/~foo/bar")) - assert_equal("/~foo%20bar", escape("/~foo bar")) - assert_equal("/~foo%20bar", escape("/~foo bar")) - assert_equal("/~foo%09bar", escape("/~foo\tbar")) - assert_equal("/~foo+bar", escape("/~foo+bar")) - bug8425 = '[Bug #8425] [ruby-core:55052]' - assert_nothing_raised(ArgumentError, Encoding::CompatibilityError, bug8425) { - assert_equal("%E3%83%AB%E3%83%93%E3%83%BC%E3%81%95%E3%82%93", escape("\u{30EB 30D3 30FC 3055 3093}")) - } - end - - def test_escape_form - assert_equal("%2Ffoo%2Fbar", escape_form("/foo/bar")) - assert_equal("%2F~foo%2Fbar", escape_form("/~foo/bar")) - assert_equal("%2F~foo+bar", escape_form("/~foo bar")) - assert_equal("%2F~foo+%2B+bar", escape_form("/~foo + bar")) - end - - def test_unescape - assert_equal("/foo/bar", unescape("%2ffoo%2fbar")) - assert_equal("/~foo/bar", unescape("/%7efoo/bar")) - assert_equal("/~foo/bar", unescape("%2f%7efoo%2fbar")) - assert_equal("/~foo+bar", unescape("/%7efoo+bar")) - end - - def test_unescape_form - assert_equal("//foo/bar", unescape_form("/%2Ffoo/bar")) - assert_equal("//foo/bar baz", unescape_form("/%2Ffoo/bar+baz")) - assert_equal("/~foo/bar baz", unescape_form("/%7Efoo/bar+baz")) - end - - def test_escape_path - assert_equal("/foo/bar", escape_path("/foo/bar")) - assert_equal("/foo/bar/", escape_path("/foo/bar/")) - assert_equal("/%25foo/bar/", escape_path("/%foo/bar/")) - end -end diff --git a/tool/test/webrick/test_httpversion.rb b/tool/test/webrick/test_httpversion.rb deleted file mode 100644 index e50ee17971..0000000000 --- a/tool/test/webrick/test_httpversion.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/httpversion" - -class TestWEBrickHTTPVersion < Test::Unit::TestCase - def setup - @v09 = WEBrick::HTTPVersion.new("0.9") - @v10 = WEBrick::HTTPVersion.new("1.0") - @v11 = WEBrick::HTTPVersion.new("1.001") - end - - def test_to_s() - assert_equal("0.9", @v09.to_s) - assert_equal("1.0", @v10.to_s) - assert_equal("1.1", @v11.to_s) - end - - def test_major() - assert_equal(0, @v09.major) - assert_equal(1, @v10.major) - assert_equal(1, @v11.major) - end - - def test_minor() - assert_equal(9, @v09.minor) - assert_equal(0, @v10.minor) - assert_equal(1, @v11.minor) - end - - def test_compar() - assert_equal(0, @v09 <=> "0.9") - assert_equal(0, @v09 <=> "0.09") - - assert_equal(-1, @v09 <=> @v10) - assert_equal(-1, @v09 <=> "1.00") - - assert_equal(1, @v11 <=> @v09) - assert_equal(1, @v11 <=> "1.0") - assert_equal(1, @v11 <=> "0.9") - end -end diff --git a/tool/test/webrick/test_server.rb b/tool/test/webrick/test_server.rb deleted file mode 100644 index 0fd3f3b63b..0000000000 --- a/tool/test/webrick/test_server.rb +++ /dev/null @@ -1,191 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "tempfile" -require "webrick" -require_relative "utils" - -class TestWEBrickServer < Test::Unit::TestCase - class Echo < WEBrick::GenericServer - def run(sock) - while line = sock.gets - sock << line - end - end - end - - def test_server - TestWEBrick.start_server(Echo){|server, addr, port, log| - TCPSocket.open(addr, port){|sock| - sock.puts("foo"); assert_equal("foo\n", sock.gets, log.call) - sock.puts("bar"); assert_equal("bar\n", sock.gets, log.call) - sock.puts("baz"); assert_equal("baz\n", sock.gets, log.call) - sock.puts("qux"); assert_equal("qux\n", sock.gets, log.call) - } - } - end - - def test_start_exception - stopped = 0 - - log = [] - logger = WEBrick::Log.new(log, WEBrick::BasicLog::WARN) - - assert_raise(SignalException) do - listener = Object.new - def listener.to_io # IO.select invokes #to_io. - raise SignalException, 'SIGTERM' # simulate signal in main thread - end - def listener.shutdown - end - def listener.close - end - - server = WEBrick::HTTPServer.new({ - :BindAddress => "127.0.0.1", :Port => 0, - :StopCallback => Proc.new{ stopped += 1 }, - :Logger => logger, - }) - server.listeners[0].close - server.listeners[0] = listener - - server.start - end - - assert_equal(1, stopped) - assert_equal(1, log.length) - assert_match(/FATAL SignalException: SIGTERM/, log[0]) - end - - def test_callbacks - accepted = started = stopped = 0 - config = { - :AcceptCallback => Proc.new{ accepted += 1 }, - :StartCallback => Proc.new{ started += 1 }, - :StopCallback => Proc.new{ stopped += 1 }, - } - TestWEBrick.start_server(Echo, config){|server, addr, port, log| - true while server.status != :Running - sleep 1 if defined?(RubyVM::JIT) && RubyVM::JIT.enabled? # server.status behaves unexpectedly with --jit-wait - assert_equal(1, started, log.call) - assert_equal(0, stopped, log.call) - assert_equal(0, accepted, log.call) - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - TCPSocket.open(addr, port){|sock| (sock << "foo\n").gets } - assert_equal(3, accepted, log.call) - } - assert_equal(1, started) - assert_equal(1, stopped) - end - - def test_daemon - begin - r, w = IO.pipe - pid1 = Process.fork{ - r.close - WEBrick::Daemon.start - w.puts(Process.pid) - sleep 10 - } - pid2 = r.gets.to_i - assert(Process.kill(:KILL, pid2)) - assert_not_equal(pid1, pid2) - rescue NotImplementedError - # snip this test - ensure - Process.wait(pid1) if pid1 - r.close - w.close - end - end - - def test_restart_after_shutdown - address = '127.0.0.1' - port = 0 - log = [] - config = { - :BindAddress => address, - :Port => port, - :Logger => WEBrick::Log.new(log, WEBrick::BasicLog::WARN), - } - server = Echo.new(config) - client_proc = lambda {|str| - begin - ret = server.listeners.first.connect_address.connect {|s| - s.write(str) - s.close_write - s.read - } - assert_equal(str, ret) - ensure - server.shutdown - end - } - server_thread = Thread.new { server.start } - client_thread = Thread.new { client_proc.call("a") } - assert_join_threads([client_thread, server_thread]) - server.listen(address, port) - server_thread = Thread.new { server.start } - client_thread = Thread.new { client_proc.call("b") } - assert_join_threads([client_thread, server_thread]) - assert_equal([], log) - end - - def test_restart_after_stop - log = Object.new - class << log - include Test::Unit::Assertions - def <<(msg) - flunk "unexpected log: #{msg.inspect}" - end - end - client_thread = nil - wakeup = -> {client_thread.wakeup} - warn_flunk = WEBrick::Log.new(log, WEBrick::BasicLog::WARN) - server = WEBrick::HTTPServer.new( - :StartCallback => wakeup, - :StopCallback => wakeup, - :BindAddress => '0.0.0.0', - :Port => 0, - :Logger => warn_flunk) - 2.times { - server_thread = Thread.start { - server.start - } - client_thread = Thread.start { - sleep 0.1 until server.status == :Running || !server_thread.status - server.stop - sleep 0.1 until server.status == :Stop || !server_thread.status - } - assert_join_threads([client_thread, server_thread]) - } - end - - def test_port_numbers - config = { - :BindAddress => '0.0.0.0', - :Logger => WEBrick::Log.new([], WEBrick::BasicLog::WARN), - } - - ports = [0, "0"] - - ports.each do |port| - config[:Port]= port - server = WEBrick::GenericServer.new(config) - server_thread = Thread.start { server.start } - client_thread = Thread.start { - sleep 0.1 until server.status == :Running || !server_thread.status - server_port = server.listeners[0].addr[1] - server.stop - assert_equal server.config[:Port], server_port - sleep 0.1 until server.status == :Stop || !server_thread.status - } - assert_join_threads([client_thread, server_thread]) - end - - assert_raise(ArgumentError) do - config[:Port]= "FOO" - WEBrick::GenericServer.new(config) - end - end -end diff --git a/tool/test/webrick/test_ssl_server.rb b/tool/test/webrick/test_ssl_server.rb deleted file mode 100644 index 4e52598bf5..0000000000 --- a/tool/test/webrick/test_ssl_server.rb +++ /dev/null @@ -1,67 +0,0 @@ -require "test/unit" -require "webrick" -require "webrick/ssl" -require_relative "utils" -require 'timeout' - -class TestWEBrickSSLServer < Test::Unit::TestCase - class Echo < WEBrick::GenericServer - def run(sock) - while line = sock.gets - sock << line - end - end - end - - def test_self_signed_cert_server - assert_self_signed_cert( - :SSLEnable => true, - :SSLCertName => [["C", "JP"], ["O", "www.ruby-lang.org"], ["CN", "Ruby"]], - ) - end - - def test_self_signed_cert_server_with_string - assert_self_signed_cert( - :SSLEnable => true, - :SSLCertName => "/C=JP/O=www.ruby-lang.org/CN=Ruby", - ) - end - - def assert_self_signed_cert(config) - TestWEBrick.start_server(Echo, config){|server, addr, port, log| - io = TCPSocket.new(addr, port) - sock = OpenSSL::SSL::SSLSocket.new(io) - sock.connect - sock.puts(server.ssl_context.cert.subject.to_s) - assert_equal("/C=JP/O=www.ruby-lang.org/CN=Ruby\n", sock.gets, log.call) - sock.close - io.close - } - end - - def test_slow_connect - poke = lambda do |io, msg| - begin - sock = OpenSSL::SSL::SSLSocket.new(io) - sock.connect - sock.puts(msg) - assert_equal "#{msg}\n", sock.gets, msg - ensure - sock&.close - io.close - end - end - config = { - :SSLEnable => true, - :SSLCertName => "/C=JP/O=www.ruby-lang.org/CN=Ruby", - } - EnvUtil.timeout(10) do - TestWEBrick.start_server(Echo, config) do |server, addr, port, log| - outer = TCPSocket.new(addr, port) - inner = TCPSocket.new(addr, port) - poke.call(inner, 'fast TLS negotiation') - poke.call(outer, 'slow TLS negotiation') - end - end - end -end diff --git a/tool/test/webrick/test_utils.rb b/tool/test/webrick/test_utils.rb deleted file mode 100644 index c2b7a36e8a..0000000000 --- a/tool/test/webrick/test_utils.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: false -require "test/unit" -require "webrick/utils" - -class TestWEBrickUtils < Test::Unit::TestCase - def teardown - WEBrick::Utils::TimeoutHandler.terminate - super - end - - def assert_expired(m) - Thread.handle_interrupt(Timeout::Error => :never, EX => :never) do - assert_empty(m::TimeoutHandler.instance.instance_variable_get(:@timeout_info)) - end - end - - def assert_not_expired(m) - Thread.handle_interrupt(Timeout::Error => :never, EX => :never) do - assert_not_empty(m::TimeoutHandler.instance.instance_variable_get(:@timeout_info)) - end - end - - EX = Class.new(StandardError) - - def test_no_timeout - m = WEBrick::Utils - assert_equal(:foo, m.timeout(10){ :foo }) - assert_expired(m) - end - - def test_nested_timeout_outer - m = WEBrick::Utils - i = 0 - assert_raise(Timeout::Error){ - m.timeout(1){ - assert_raise(Timeout::Error){ m.timeout(0.1){ i += 1; sleep(1) } } - assert_not_expired(m) - i += 1 - sleep(2) - } - } - assert_equal(2, i) - assert_expired(m) - end - - def test_timeout_default_exception - m = WEBrick::Utils - assert_raise(Timeout::Error){ m.timeout(0.01){ sleep } } - assert_expired(m) - end - - def test_timeout_custom_exception - m = WEBrick::Utils - ex = EX - assert_raise(ex){ m.timeout(0.01, ex){ sleep } } - assert_expired(m) - end - - def test_nested_timeout_inner_custom_exception - m = WEBrick::Utils - ex = EX - i = 0 - assert_raise(ex){ - m.timeout(10){ - m.timeout(0.01, ex){ i += 1; sleep } - } - sleep - } - assert_equal(1, i) - assert_expired(m) - end - - def test_nested_timeout_outer_custom_exception - m = WEBrick::Utils - ex = EX - i = 0 - assert_raise(Timeout::Error){ - m.timeout(0.01){ - m.timeout(1.0, ex){ i += 1; sleep } - } - sleep - } - assert_equal(1, i) - assert_expired(m) - end - - def test_create_listeners - addr = listener_address(0) - port = addr.slice!(1) - assert_kind_of(Integer, port, "dynamically chosen port number") - assert_equal(["AF_INET", "127.0.0.1", "127.0.0.1"], addr) - - assert_equal(["AF_INET", port, "127.0.0.1", "127.0.0.1"], - listener_address(port), - "specific port number") - - assert_equal(["AF_INET", port, "127.0.0.1", "127.0.0.1"], - listener_address(port.to_s), - "specific port number string") - end - - def listener_address(port) - listeners = WEBrick::Utils.create_listeners("127.0.0.1", port) - srv = listeners.first - assert_kind_of TCPServer, srv - srv.addr - ensure - listeners.each(&:close) if listeners - end -end diff --git a/tool/test/webrick/utils.rb b/tool/test/webrick/utils.rb deleted file mode 100644 index a8568d0a43..0000000000 --- a/tool/test/webrick/utils.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: false -require "webrick" -begin - require "webrick/https" -rescue LoadError -end -require "webrick/httpproxy" - -module TestWEBrick - NullWriter = Object.new - def NullWriter.<<(msg) - puts msg if $DEBUG - return self - end - - class WEBrick::HTTPServlet::CGIHandler - remove_const :Ruby - require "envutil" unless defined?(EnvUtil) - Ruby = EnvUtil.rubybin - remove_const :CGIRunner - CGIRunner = "\"#{Ruby}\" \"#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb\"" # :nodoc: - remove_const :CGIRunnerArray - CGIRunnerArray = [Ruby, "#{WEBrick::Config::LIBDIR}/httpservlet/cgi_runner.rb"] # :nodoc: - end - - RubyBin = "\"#{EnvUtil.rubybin}\"" - RubyBin << " --disable-gems" - RubyBin << " \"-I#{File.expand_path("../..", File.dirname(__FILE__))}/lib\"" - RubyBin << " \"-I#{File.dirname(EnvUtil.rubybin)}/.ext/common\"" - RubyBin << " \"-I#{File.dirname(EnvUtil.rubybin)}/.ext/#{RUBY_PLATFORM}\"" - - RubyBinArray = [EnvUtil.rubybin] - RubyBinArray << "--disable-gems" - RubyBinArray << "-I" << "#{File.expand_path("../..", File.dirname(__FILE__))}/lib" - RubyBinArray << "-I" << "#{File.dirname(EnvUtil.rubybin)}/.ext/common" - RubyBinArray << "-I" << "#{File.dirname(EnvUtil.rubybin)}/.ext/#{RUBY_PLATFORM}" - - require "test/unit" unless defined?(Test::Unit) - include Test::Unit::Assertions - extend Test::Unit::Assertions - include Test::Unit::CoreAssertions - extend Test::Unit::CoreAssertions - - module_function - - DefaultLogTester = lambda {|log, access_log| assert_equal([], log) } - - def start_server(klass, config={}, log_tester=DefaultLogTester, &block) - log_ary = [] - access_log_ary = [] - log = proc { "webrick log start:\n" + (log_ary+access_log_ary).join.gsub(/^/, " ").chomp + "\nwebrick log end" } - config = ({ - :BindAddress => "127.0.0.1", :Port => 0, - :ServerType => Thread, - :Logger => WEBrick::Log.new(log_ary, WEBrick::BasicLog::WARN), - :AccessLog => [[access_log_ary, ""]] - }.update(config)) - server = capture_output {break klass.new(config)} - server_thread = server.start - server_thread2 = Thread.new { - server_thread.join - if log_tester - log_tester.call(log_ary, access_log_ary) - end - } - addr = server.listeners[0].addr - client_thread = Thread.new { - begin - block.yield([server, addr[3], addr[1], log]) - ensure - server.shutdown - end - } - assert_join_threads([client_thread, server_thread2]) - end - - def start_httpserver(config={}, log_tester=DefaultLogTester, &block) - start_server(WEBrick::HTTPServer, config, log_tester, &block) - end - - def start_httpproxy(config={}, log_tester=DefaultLogTester, &block) - start_server(WEBrick::HTTPProxyServer, config, log_tester, &block) - end -end diff --git a/tool/test/webrick/webrick.cgi b/tool/test/webrick/webrick.cgi deleted file mode 100644 index a294fa72f9..0000000000 --- a/tool/test/webrick/webrick.cgi +++ /dev/null @@ -1,38 +0,0 @@ -#!ruby -require "webrick/cgi" - -class TestApp < WEBrick::CGI - def do_GET(req, res) - res["content-type"] = "text/plain" - if req.path_info == "/dumpenv" - res.body = Marshal.dump(ENV.to_hash) - elsif (p = req.path_info) && p.length > 0 - res.body = p - elsif (q = req.query).size > 0 - res.body = q.keys.sort.collect{|key| - q[key].list.sort.collect{|v| - "#{key}=#{v}" - }.join(", ") - }.join(", ") - elsif %r{/$} =~ req.request_uri.to_s - res.body = "" - res.body << req.request_uri.to_s << "\n" - res.body << req.script_name - elsif !req.cookies.empty? - res.body = req.cookies.inject(""){|result, cookie| - result << "%s=%s\n" % [cookie.name, cookie.value] - } - res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") - res.cookies << WEBrick::Cookie.new("Shipping", "FedEx") - else - res.body = req.script_name - end - end - - def do_POST(req, res) - do_GET(req, res) - end -end - -cgi = TestApp.new -cgi.start diff --git a/tool/test/webrick/webrick.rhtml b/tool/test/webrick/webrick.rhtml deleted file mode 100644 index a7bbe43fb5..0000000000 --- a/tool/test/webrick/webrick.rhtml +++ /dev/null @@ -1,4 +0,0 @@ -req to <%= -servlet_request.request_uri -%> <%= -servlet_request.query.inspect %> diff --git a/tool/test/webrick/webrick_long_filename.cgi b/tool/test/webrick/webrick_long_filename.cgi deleted file mode 100644 index 43c1af825c..0000000000 --- a/tool/test/webrick/webrick_long_filename.cgi +++ /dev/null @@ -1,36 +0,0 @@ -#!ruby -require "webrick/cgi" - -class TestApp < WEBrick::CGI - def do_GET(req, res) - res["content-type"] = "text/plain" - if (p = req.path_info) && p.length > 0 - res.body = p - elsif (q = req.query).size > 0 - res.body = q.keys.sort.collect{|key| - q[key].list.sort.collect{|v| - "#{key}=#{v}" - }.join(", ") - }.join(", ") - elsif %r{/$} =~ req.request_uri.to_s - res.body = "" - res.body << req.request_uri.to_s << "\n" - res.body << req.script_name - elsif !req.cookies.empty? - res.body = req.cookies.inject(""){|result, cookie| - result << "%s=%s\n" % [cookie.name, cookie.value] - } - res.cookies << WEBrick::Cookie.new("Customer", "WILE_E_COYOTE") - res.cookies << WEBrick::Cookie.new("Shipping", "FedEx") - else - res.body = req.script_name - end - end - - def do_POST(req, res) - do_GET(req, res) - end -end - -cgi = TestApp.new -cgi.start diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb index dba6f33ff9..1257a92d38 100644 --- a/tool/transcode-tblgen.rb +++ b/tool/transcode-tblgen.rb @@ -725,7 +725,7 @@ def citrus_decode_mapsrc(ces, csid, mapsrcs) path << ".src" path[path.rindex('/')] = '%' STDOUT.puts 'load mapsrc %s' % path if VERBOSE_MODE > 1 - open(path, 'rb') do |f| + File.open(path, 'rb') do |f| f.each_line do |l| break if /^BEGIN_MAP/ =~ l end @@ -1078,11 +1078,7 @@ if __FILE__ == $0 end libs1 = $".dup - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(src, trim_mode: '%') - else - erb = ERB.new(src, nil, '%') - end + erb = ERB.new(src, trim_mode: '%') erb.filename = arg erb_result = erb.result(binding) libs2 = $".dup diff --git a/tool/transform_mjit_header.rb b/tool/transform_mjit_header.rb deleted file mode 100644 index 2359ceab7c..0000000000 --- a/tool/transform_mjit_header.rb +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright (C) 2017 Vladimir Makarov, <vmakarov@redhat.com> -# This is a script to transform functions to static inline. -# Usage: transform_mjit_header.rb <c-compiler> <header file> <out> - -require 'fileutils' -require 'tempfile' - -PROGRAM = File.basename($0, ".*") - -module MJITHeader - ATTR_VALUE_REGEXP = /[^()]|\([^()]*\)/ - ATTR_REGEXP = /__attribute__\s*\(\(#{ATTR_VALUE_REGEXP}*\)\)/ - # Example: - # VALUE foo(int bar) - # VALUE __attribute__ ((foo)) bar(int baz) - # __attribute__ ((foo)) VALUE bar(int baz) - FUNC_HEADER_REGEXP = /\A[^\[{(]*(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/ - TARGET_NAME_REGEXP = /\A(rb|ruby|vm|insn|attr|Init)_/ - - # Predefined macros for compilers which are already supported by MJIT. - # We're going to support cl.exe too (WIP) but `cl.exe -E` can't produce macro. - SUPPORTED_CC_MACROS = [ - '__GNUC__', # gcc - '__clang__', # clang - ] - - # These macros are relied on this script's transformation - PREFIXED_MACROS = [ - 'ALWAYS_INLINE', - 'COLDFUNC', - 'inline', - 'RBIMPL_ATTR_COLD', - ] - - # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times. - RECURSIVE_MACROS = %w[ - RASCTRYINFO - RASIPADDR - ] - - IGNORED_FUNCTIONS = [ - 'rb_vm_search_method_slowpath', # This increases the time to compile when inlined. So we use it as external function. - 'rb_equal_opt', # Not used from VM and not compilable - ] - - ALWAYS_INLINED_FUNCTIONS = [ - 'vm_opt_plus', - 'vm_opt_minus', - 'vm_opt_mult', - 'vm_opt_div', - 'vm_opt_mod', - 'vm_opt_neq', - 'vm_opt_lt', - 'vm_opt_le', - 'vm_opt_gt', - 'vm_opt_ge', - 'vm_opt_ltlt', - 'vm_opt_and', - 'vm_opt_or', - 'vm_opt_aref', - 'vm_opt_aset', - 'vm_opt_aref_with', - 'vm_opt_aset_with', - 'vm_opt_not', - ] - - COLD_FUNCTIONS = %w[ - setup_parameters_complex - vm_call_iseq_setup - vm_call_iseq_setup_2 - vm_call_iseq_setup_tailcall - vm_call_method_each_type - vm_ic_update - ] - - # Return start..stop of last decl in CODE ending STOP - def self.find_decl(code, stop) - level = 0 - i = stop - while i = code.rindex(/[;{}]/, i) - if level == 0 && stop != i && decl_found?($&, i) - return decl_start($&, i)..stop - end - case $& - when '}' - level += 1 - when '{' - level -= 1 - end - i -= 1 - end - nil - end - - def self.decl_found?(code, i) - i == 0 || code == ';' || code == '}' - end - - def self.decl_start(code, i) - if i == 0 && code != ';' && code != '}' - 0 - else - i + 1 - end - end - - # Given DECL return the name of it, nil if failed - def self.decl_name_of(decl) - ident_regex = /\w+/ - decl = decl.gsub(/^#.+$/, '') # remove macros - reduced_decl = decl.gsub(ATTR_REGEXP, '') # remove attributes - su1_regex = /{[^{}]*}/ - su2_regex = /{([^{}]|#{su1_regex})*}/ - su3_regex = /{([^{}]|#{su2_regex})*}/ # 3 nested structs/unions is probably enough - reduced_decl.gsub!(su3_regex, '') # remove structs/unions in the header - id_seq_regex = /\s*(?:#{ident_regex}(?:\s+|\s*[*]+\s*))*/ - # Process function header: - match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl) - return match[:name] if match - # Process non-function declaration: - reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization - match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl); - return match[:name] if match - nil - end - - # Return true if CC with CFLAGS compiles successfully the current code. - # Use STAGE in the message in case of a compilation failure - def self.check_code!(code, cc, cflags, stage) - with_code(code) do |path| - cmd = "#{cc} #{cflags} #{path}" - out = IO.popen(cmd, err: [:child, :out], &:read) - unless $?.success? - STDERR.puts "error in #{stage} header file:\n#{out}" - exit false - end - end - end - - # Remove unpreprocessable macros - def self.remove_harmful_macros!(code) - code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '') - end - - # -dD outputs those macros, and it produces redefinition warnings or errors - # This assumes common.mk passes `-DMJIT_HEADER` first when it creates rb_mjit_header.h. - def self.remove_predefined_macros!(code) - code.sub!(/\A(#define [^\n]+|\n)*(#define MJIT_HEADER 1\n)/, '\2') - end - - # Return [macro, others]. But others include PREFIXED_MACROS to be used in code. - def self.separate_macro_and_code(code) - code.lines.partition do |l| - l.start_with?('#') && PREFIXED_MACROS.all? { |m| !l.start_with?("#define #{m}") } - end.map! { |lines| lines.join('') } - end - - def self.write(code, out) - # create with strict permission, then will install proper - # permission - FileUtils.mkdir_p(File.dirname(out), mode: 0700) - File.binwrite("#{out}.new", code, perm: 0600) - FileUtils.mv("#{out}.new", out) - end - - # Note that this checks runruby. This conservatively covers platform names. - def self.windows? - RUBY_PLATFORM =~ /mswin|mingw|msys/ - end - - def self.cl_exe?(cc) - cc =~ /\Acl(\z| |\.exe)/ - end - - # If code has macro which only supported compilers predefine, return true. - def self.supported_header?(code) - SUPPORTED_CC_MACROS.any? { |macro| code =~ /^#\s*define\s+#{Regexp.escape(macro)}\b/ } - end - - # This checks if syntax check outputs one of the following messages. - # "error: conflicting types for 'restrict'" - # "error: redefinition of parameter 'restrict'" - # If it's true, this script regards platform as AIX or Solaris and adds -std=c99 as workaround. - def self.conflicting_types?(code, cc, cflags) - with_code(code) do |path| - cmd = "#{cc} #{cflags} #{path}" - out = IO.popen(cmd, err: [:child, :out], &:read) - !$?.success? && - (out.match?(/error: conflicting types for '[^']+'/) || - out.match?(/error: redefinition of parameter '[^']+'/)) - end - end - - def self.with_code(code) - # for `system_header` pragma which can't be in the main file. - Tempfile.open(['', '.h'], mode: File::BINARY) do |f| - f.puts code - f.close - Tempfile.open(['', '.c'], mode: File::BINARY) do |c| - c.puts <<SRC -#include "#{f.path}" -SRC - c.close - return yield(c.path) - end - end - end - private_class_method :with_code -end - -if ARGV.size != 3 - abort "Usage: #{$0} <c-compiler> <header file> <out>" -end - -if STDOUT.tty? - require_relative 'lib/colorize' - color = Colorize.new -end -cc = ARGV[0] -code = File.binread(ARGV[1]) # Current version of the header file. -outfile = ARGV[2] -if MJITHeader.cl_exe?(cc) - cflags = '-DMJIT_HEADER -Zs' -else - cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors' -end - -if !MJITHeader.cl_exe?(cc) && !MJITHeader.supported_header?(code) - puts "This compiler (#{cc}) looks not supported for MJIT. Giving up to generate MJIT header." - MJITHeader.write("#error MJIT does not support '#{cc}' yet", outfile) - exit -end - -MJITHeader.remove_predefined_macros!(code) - -if MJITHeader.windows? # transformation is broken with Windows headers for now - MJITHeader.remove_harmful_macros!(code) - MJITHeader.check_code!(code, cc, cflags, 'initial') - puts "\nSkipped transforming external functions to static on Windows." - MJITHeader.write(code, outfile) - exit -end - -macro, code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW -code = <<header + code -#ifdef __GNUC__ -# pragma GCC system_header -#endif -header -code_to_check = "#{code}#{macro}" # macro should not affect code again - -if MJITHeader.conflicting_types?(code_to_check, cc, cflags) - cflags = "#{cflags} -std=c99" # For AIX gcc -end - -# Check initial file correctness in the manner of final output. -MJITHeader.check_code!(code_to_check, cc, cflags, 'initial') - -stop_pos = -1 -extern_names = [] -transform_logs = Hash.new { |h, k| h[k] = [] } - -# This loop changes function declarations to static inline. -while (decl_range = MJITHeader.find_decl(code, stop_pos)) - stop_pos = decl_range.begin - 1 - decl = code[decl_range] - decl_name = MJITHeader.decl_name_of(decl) - - if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - transform_logs[:def_to_decl] << decl_name - code[decl_range] = decl.sub(/{.+}/m, ';') - elsif MJITHeader::COLD_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - header = match[0].sub(/{\z/, '').strip - header = "static #{header.sub(/\A((static|inline) )+/, '')}" - decl[match.begin(0)...match.end(0)] = '{' # remove header - code[decl_range] = "\nCOLDFUNC #{header} #{decl}" - elsif MJITHeader::ALWAYS_INLINED_FUNCTIONS.include?(decl_name) && match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) - header = match[0].sub(/{\z/, '').strip - header = "static inline #{header.sub(/\A((static|inline) )+/, '')}" - decl[match.begin(0)...match.end(0)] = '{' # remove header - code[decl_range] = "\nALWAYS_INLINE(#{header});\n#{header} #{decl}" - elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/) - decl.sub!(/(extern|static|inline) /, ' ') - unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. - transform_logs[:static_inline_decl] << decl_name - end - - code[decl_range] = "static inline #{decl}" - elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/ - unless decl_name.match(MJITHeader::TARGET_NAME_REGEXP) - transform_logs[:skipped] << decl_name - next - end - - extern_names << decl_name - decl[match.begin(0)...match.end(0)] = '' - - if decl =~ /\bstatic\b/ - abort "#{PROGRAM}: a static decl was found inside external definition #{decl_name.dump}" - end - - header.sub!(/(extern|inline) /, ' ') - unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. - transform_logs[:static_inline_def] << decl_name - end - code[decl_range] = "static inline #{header}#{decl}" - end -end - -code << macro - -# Check the final file correctness -MJITHeader.check_code!(code, cc, cflags, 'final') - -MJITHeader.write(code, outfile) - -messages = { - def_to_decl: 'changing definition to declaration', - static_inline_def: 'making external definition static inline', - static_inline_decl: 'making declaration static inline', - skipped: 'SKIPPED to transform', -} -transform_logs.each do |key, decl_names| - decl_names = decl_names.map { |s| color.bold(s) } if color - puts("#{PROGRAM}: #{messages.fetch(key)}: #{decl_names.join(', ')}") -end diff --git a/tool/update-NEWS-gemlist.rb b/tool/update-NEWS-gemlist.rb new file mode 100755 index 0000000000..0b5503580d --- /dev/null +++ b/tool/update-NEWS-gemlist.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby +require 'json' +news = File.read("NEWS.md") +prev = news[/since the \*+(\d+\.\d+\.\d+)\*+/, 1] +prevs = [prev, prev.sub(/\.\d+\z/, '')] + +update = ->(list, type, desc = "updated") do + item = ->(mark = "* ") do + "### The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + + list.map {|g, v|"#{mark}#{g} #{v}\n"}.join("") + "\n" + end + news.sub!(/^(?:\*( +)|#+ *)?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do + item["#{$1&.<< " "}*#{$2 || ' '}"] + end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+( *\* *).*)*)*\n+\K/) do + item[$1 || "* "] + end +end + +load_gems_json = ->(type) do + JSON.parse(File.read("#{type}_gems.json"))['gems'].filter_map do |g| + v = g['versions'].values_at(*prevs).compact.first + g = g['gem'] + g = 'RubyGems' if g == 'rubygems' + [g, v] if v + end.to_h +end + +ARGV.each do |type| + last = load_gems_json[type] + changed = File.foreach("gems/#{type}_gems").filter_map do |l| + next if l.start_with?("#") + g, v = l.split(" ", 3) + next unless v + [g, v] unless last[g] == v + end + changed, added = changed.partition {|g, _| last[g]} + update[changed, type] or next + if added and !added.empty? + if type == 'bundled' + default_gems = load_gems_json['default'] + promoted = {} + added.delete_if do |k, v| + default_gems.key?(k) && promoted[k] = v + end + update[added, type, 'added'] + update[promoted, type, 'promoted from default gems'] or next + else + update[added, type, 'added'] or next + end + end + File.write("NEWS.md", news) +end diff --git a/tool/update-NEWS-refs.rb b/tool/update-NEWS-refs.rb new file mode 100644 index 0000000000..f48cac5ee1 --- /dev/null +++ b/tool/update-NEWS-refs.rb @@ -0,0 +1,38 @@ +# Usage: ruby tool/update-NEWS-refs.rb + +orig_src = File.read(File.join(__dir__, "../NEWS.md")) +lines = orig_src.lines(chomp: true) + +links = {} +while lines.last =~ %r{\A\[(.*?)\]:\s+(.*)\z} + links[$1] = $2 + lines.pop +end + +if links.empty? || lines.last != "" + raise "NEWS.md must end with a sequence of links" +end + +trackers = ["Feature", "Bug", "Misc"] +labels = links.keys.reject {|k| k.start_with?(*trackers)} +new_src = lines.join("\n").gsub(/\[?\[(#{Regexp.union(trackers)}\s+#(\d+))\]\]?/) do + links[$1] ||= "https://bugs.ruby-lang.org/issues/#$2" + "[[#$1]]" +end.gsub(/\[\[#{Regexp.union(labels)}\]\]?/) do + "[#$1]" +end.chomp + "\n\n" + +label_width = links.max_by {|k, _| k.size}.first.size + 4 +redmine_links, non_redmine_links = links.partition {|k,| k =~ /\A#{Regexp.union(trackers)}\s+#\d+\z/ } + +(redmine_links.sort_by {|k,| k[/\d+/].to_i } + non_redmine_links.reverse).each do |k, v| + new_src << "[#{k}]:".ljust(label_width) << v << "\n" +end + +if orig_src != new_src + print "Update NEWS.md? [y/N]" + $stdout.flush + if gets.chomp == "y" + File.write(File.join(__dir__, "../NEWS.md"), new_src) + end +end diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb index 48dcaa9f2c..dec6b49cee 100755 --- a/tool/update-bundled_gems.rb +++ b/tool/update-bundled_gems.rb @@ -1,20 +1,41 @@ -#!ruby -pla +#!ruby -alpF\s+|#.* BEGIN { require 'rubygems' + date = nil + # STDOUT is not usable in inplace edit mode + output = $-i ? STDOUT : STDERR } -unless /^[^#]/ !~ (gem = $F[0]) +output = STDERR if ARGF.file == STDIN +END { + output.print date.strftime("latest_date=%F") if date +} +if gem = $F[0] + ver = Gem::Version.new($F[1]) (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| s.platform == "ruby" && s.name == gem } - gem = src.fetch_spec(gem) - uri = gem.metadata["source_code_uri"] || gem.homepage - uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") - if $F[3] - if $F[3].include?($F[1]) - $F[3][$F[1]] = gem.version.to_s - elsif $F[2] != gem.version and /\A\h+\z/ =~ $F[3] - $F[3..-1] = [] + if gem.version > ver + gem = src.fetch_spec(gem) + if ENV["UPDATE_BUNDLED_GEMS_ALL"] + uri = gem.metadata["source_code_uri"] || gem.homepage + uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") + else + uri = $F[2] + end + if (!date or gem.date && gem.date > date) and gem.date.to_i != 315_619_200 + # DEFAULT_SOURCE_DATE_EPOCH is meaningless + date = gem.date + end + if $F[3] + if $F[3].include?($F[1]) + $F[3][$F[1]] = gem.version.to_s + elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] + $F[3..-1] = [] + end end + f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] + $_.gsub!(/\S+\s*(?=\s|$)/) {|s| (f.shift || "").ljust(s.size)} + $_ = [$_, *f].join(" ") unless f.empty? + $_.rstrip! end - $_ = [gem.name, gem.version, uri, *$F[3..-1]].join(" ") end diff --git a/tool/update-deps b/tool/update-deps index 2348b36e33..2d4a5674be 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -17,6 +17,14 @@ # 3. Use --fix to fix makefiles. # Ex. ./ruby tool/update-deps --fix # +# Usage to create a depend file initially: +# 1. Copy the dependency section from the Makefile generated by extconf.rb. +# Ex. ext/cgi/escape/Makefile +# 2. Add `# AUTOGENERATED DEPENDENCIES START` and `# AUTOGENERATED DEPENDENCIES END` +# sections to top and end of the depend file. +# 3. Run tool/update-deps --fix to fix the depend file. +# 4. Commit the depend file. +# # Other usages: # * Fix makefiles using previously detected dependency problems # Ex. ruby tool/update-deps --actual-fix [file] @@ -88,7 +96,15 @@ result.each {|k,v| # They can be referenced as $(top_srcdir)/filename. # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' FILES_IN_SOURCE_DIRECTORY = %w[ - revision.h + prism/api_node.c + prism/ast.h + prism/diagnostic.c + prism/diagnostic.h + prism/node.c + prism/prettyprint.c + prism/serialize.c + prism/token_type.c + prism/version.h ] # Files built in the build directory (except extconf.h). @@ -112,6 +128,7 @@ FILES_NEED_VPATH = %w[ ext/ripper/eventids1.c ext/ripper/eventids2table.c ext/ripper/ripper.c + ext/ripper/ripper_init.c golf_prelude.c id.c id.h @@ -120,15 +137,14 @@ FILES_NEED_VPATH = %w[ known_errors.inc lex.c miniprelude.c - mjit_compile.inc newline.c node_name.inc - opt_sc.inc optinsn.inc optunifs.inc parse.c parse.h probes.dmyh + revision.h vm.inc vmtc.inc @@ -167,13 +183,17 @@ FILES_SAME_NAME_TOP = %w[ version.h ] +# Files that may or may not exist on CI for some reason. +# Windows build generally seems to have missing dependencies. +UNSTABLE_FILES = %r{\Awin32/[^/]+\.o\z} + # Other source files exist in the source directory. def in_makefile(target, source) target = target.to_s source = source.to_s case target - when %r{\A[^/]*\z}, %r{\Acoroutine/} + when %r{\A[^/]*\z}, %r{\Acoroutine/}, %r{\Aprism/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" @@ -193,7 +213,7 @@ def in_makefile(target, source) when %r{\Acoroutine/} then source2 = "{$(VPATH)}$(COROUTINE_H)" else source2 = "$(top_srcdir)/#{source}" end - ["common.mk", target2, source2] + ["depend", target2, source2] when %r{\Aenc/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source @@ -230,6 +250,10 @@ def in_makefile(target, source) else source2 = "$(top_srcdir)/#{source}" end ["#{File.dirname(target)}/depend", target2, source2] + # Files that may or may not exist on CI for some reason. + # Windows build generally seems to have missing dependencies. + when UNSTABLE_FILES + warn "warning: ignoring: #{target}" else raise "unexpected target: #{target}" end @@ -301,6 +325,10 @@ def read_make_deps(cwd) deps = deps.scan(%r{[/0-9a-zA-Z._-]+}) deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp next if /\.o\z/ !~ target.to_s + next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source) + next if /libzjit.o\z/ =~ target.to_s # skip ZJIT Rust object (no corresponding C source) + next if /target\/release\/libruby.o\z/ =~ target.to_s # skip YJIT+ZJIT Rust object (no corresponding C source) + next if /\.bundle\// =~ target.to_s next if /\A\./ =~ target.to_s # skip rules such as ".c.o" #p [curdir, target, deps] dir = curdir || dirstack.last @@ -332,24 +360,25 @@ end # raise ArgumentError, "can not find #{filename} (hint: #{hint0})" #end -def read_single_cc_deps(path_i, cwd) +def read_single_cc_deps(path_i, cwd, fn_o) files = {} compiler_wd = nil path_i.each_line {|line| next if /\A\# \d+ "(.*)"/ !~ line dep = $1 + next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc. - next if /\.erb\z/ =~ dep - compiler_wd ||= dep + next if /\.e?rb\z/ =~ dep + # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. + if /\/\/\z/ =~ dep + compiler_wd = Pathname(dep.sub(%r{//\z}, '')) + next + end + files[dep] = true } - # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. - if %r{\A/.*//\z} =~ compiler_wd - files.delete compiler_wd - compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, '')) - elsif !(compiler_wd = yield) - raise "compiler working directory not found: #{path_i}" - end + compiler_wd ||= fn_o.to_s.start_with?("enc/") ? cwd : path_i.parent + deps = [] files.each_key {|dep| dep = Pathname(dep) @@ -381,13 +410,7 @@ def read_cc_deps(cwd) end path_o = cwd + fn_o path_i = cwd + fn_i - deps[path_o] = read_single_cc_deps(path_i, cwd) do - if fn_o.to_s.start_with?("enc/") - cwd - else - path_i.parent - end - end + deps[path_o] = read_single_cc_deps(path_i, cwd, fn_o) } deps end @@ -475,7 +498,7 @@ def compare_deps(make_deps, cc_deps, out=$stdout) } } - makefiles.keys.sort.each {|makefile| + makefiles.keys.compact.sort.each {|makefile| cc_lines = cc_lines_hash[makefile] || Hash.new(false) make_lines = make_lines_hash[makefile] || Hash.new(false) content = begin diff --git a/tool/wasm-clangw b/tool/wasm-clangw new file mode 100755 index 0000000000..9ebdfda75a --- /dev/null +++ b/tool/wasm-clangw @@ -0,0 +1,9 @@ +#!/bin/sh +# A Clang wrapper script to fake the clang linker driver. +# Clang linker automatically uses wasm-opt with -O if it found. +# However optimization before asyncify causes misoptimization, +# so wrap clang to insert our fake wasm-opt, which does nothing, in PATH. + +src_dir="$(cd "$(dirname "$0")/../wasm" && pwd)" +export PATH="$src_dir:$PATH" +exec "$@" diff --git a/tool/ytab.sed b/tool/ytab.sed deleted file mode 100755 index 95a9b3e1eb..0000000000 --- a/tool/ytab.sed +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sed -f -# This file is used when generating code for the Ruby parser. -/^int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^extern int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^yydestruct.*yymsg/,/{/{ - /^yydestruct/{ - /,$/N - /[, *]p)/!{ - H - s/^/ruby_parser_&/ - s/)$/, p)/ - /\*/s/p)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /yydestruct/{ - i\ -\ struct parser_params *p; - } - x - } - /^{/{ - x - /yydestruct/{ - i\ -#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, p) - } - x - } -} -/^yy_stack_print /,/{/{ - /^yy_stack_print/{ - /[, *]p)/!{ - H - s/^/ruby_parser_&/ - s/)$/, p)/ - /\*/s/p)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /yy_stack_print/{ - i\ -\ struct parser_params *p; - } - x - } - /^{/{ - x - /yy_stack_print/{ - i\ -#define yy_stack_print(b, t) ruby_parser_yy_stack_print(b, t, p) - } - x - } -} -/^yy_reduce_print/,/^}/{ - s/fprintf *(stderr,/YYFPRINTF (p,/g -} -s/^yysyntax_error (/&struct parser_params *p, / -s/ yysyntax_error (/&p, / -s/\( YYFPRINTF *(\)yyoutput,/\1p,/ -s/\( YYFPRINTF *(\)yyo,/\1p,/ -s/\( YYFPRINTF *(\)stderr,/\1p,/ -s/\( YYDPRINTF *((\)stderr,/\1p,/ -s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/ -s!^ *extern char \*getenv();!/* & */! -s/^\(#.*\)".*\.tab\.c"/\1"parse.c"/ -/^\(#.*\)".*\.y"/s:\\\\:/:g diff --git a/tool/zjit_bisect.rb b/tool/zjit_bisect.rb new file mode 100755 index 0000000000..997c572f51 --- /dev/null +++ b/tool/zjit_bisect.rb @@ -0,0 +1,158 @@ +#!/usr/bin/env ruby +require 'logger' +require 'optparse' +require 'shellwords' +require 'tempfile' +require 'timeout' + +ARGS = {timeout: 5} +OptionParser.new do |opts| + opts.banner += " <path_to_ruby> -- <options>" + opts.on("--timeout=TIMEOUT_SEC", "Seconds until child process is killed") do |timeout| + ARGS[:timeout] = Integer(timeout) + end + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end +end.parse! + +usage = "Usage: zjit_bisect.rb <path_to_ruby> -- <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +raise(usage) if OPTIONS.empty? +LOGGER = Logger.new($stdout) + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +# Attempt to reduce the `items` argument as much as possible, returning the +# shorter version. `fixed` will always be used as part of the items when +# running `command`. +# `command` should return True if the command succeeded (the failure did not +# reproduce) and False if the command failed (the failure reproduced). +def bisect_impl(command, fixed, items, indent="") + LOGGER.info("#{indent}step fixed[#{fixed.length}] and items[#{items.length}]") + while items.length > 1 + LOGGER.info("#{indent}#{fixed.length + items.length} candidates") + # Return two halves of the given list. For odd-length lists, the second + # half will be larger. + half = items.length / 2 + left = items[0...half] + right = items[half..] + if !command.call(fixed + left) + items = left + next + end + if !command.call(fixed + right) + items = right + next + end + # We need something from both halves to trigger the failure. Try + # holding each half fixed and bisecting the other half to reduce the + # candidates. + new_right = bisect_impl(command, fixed + left, right, indent + "< ") + new_left = bisect_impl(command, fixed + new_right, left, indent + "> ") + return new_left + new_right + end + items +end + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +def run_bisect(command, items) + LOGGER.info("Verifying items") + if command.call(items) + raise StandardError.new("Command succeeded with full items") + end + if !command.call([]) + raise StandardError.new("Command failed with empty items") + end + bisect_impl(command, [], items) +end + +def add_zjit_options cmd + if RUBY == "make" + # Automatically detect that we're running a make command instead of a Ruby + # one. Pass the bisection options via RUN_OPTS/SPECOPTS instead. + zjit_opts = cmd.select { |arg| arg.start_with?("--zjit") } + run_opts_index = cmd.find_index { |arg| arg.start_with?("RUN_OPTS=") } + specopts_index = cmd.find_index { |arg| arg.start_with?("SPECOPTS=") } + if run_opts_index + run_opts = Shellwords.split(cmd[run_opts_index].delete_prefix("RUN_OPTS=")) + run_opts.concat(zjit_opts) + cmd[run_opts_index] = "RUN_OPTS=#{run_opts.shelljoin}" + elsif specopts_index + specopts = Shellwords.split(cmd[specopts_index].delete_prefix("SPECOPTS=")) + specopts.concat(zjit_opts) + cmd[specopts_index] = "SPECOPTS=#{specopts.shelljoin}" + else + raise "Expected RUN_OPTS or SPECOPTS to be present in make command" + end + cmd = cmd - zjit_opts + end + cmd +end + +def run_ruby *cmd + cmd = add_zjit_options(cmd) + pid = Process.spawn(*cmd, { + in: :close, + out: [File::NULL, File::RDWR], + err: [File::NULL, File::RDWR], + }) + begin + status = Timeout.timeout(ARGS[:timeout]) do + Process::Status.wait(pid) + end + rescue Timeout::Error + Process.kill("KILL", pid) + LOGGER.warn("Timed out after #{ARGS[:timeout]} seconds") + status = Process::Status.wait(pid) + end + + status +end + +def run_with_jit_list(ruby, options, jit_list) + # Make a new temporary file containing the JIT list + Tempfile.create("jit_list") do |temp_file| + temp_file.write(jit_list.join("\n")) + temp_file.flush + temp_file.close + # Run the JIT with the temporary file + run_ruby ruby, "--zjit-allowed-iseqs=#{temp_file.path}", *options + end +end + +# Try running with no JIT list to get a stable baseline +unless run_with_jit_list(RUBY, OPTIONS, []).success? + cmd = [RUBY, "--zjit-allowed-iseqs=/dev/null", *OPTIONS].shelljoin + raise "The command failed unexpectedly with an empty JIT list. To reproduce, try running the following: `#{cmd}`" +end +# Collect the JIT list from the failing Ruby process +jit_list = nil +Tempfile.create "jit_list" do |temp_file| + run_ruby RUBY, "--zjit-log-compiled-iseqs=#{temp_file.path}", *OPTIONS + jit_list = File.readlines(temp_file.path).map(&:strip).reject(&:empty?) +end +LOGGER.info("Starting with JIT list of #{jit_list.length} items.") +# Try running without the optimizer +status = run_with_jit_list(RUBY, ["--zjit-disable-hir-opt", *OPTIONS], jit_list) +if status.success? + LOGGER.warn "*** Command suceeded with HIR optimizer disabled. HIR optimizer is probably at fault. ***" +end +# Now narrow it down +command = lambda do |items| + run_with_jit_list(RUBY, OPTIONS, items).success? +end +result = run_bisect(command, jit_list) +File.open("jitlist.txt", "w") do |file| + file.puts(result) +end +puts "Run:" +jitlist_path = File.expand_path("jitlist.txt") +puts add_zjit_options([RUBY, "--zjit-allowed-iseqs=#{jitlist_path}", *OPTIONS]).shelljoin +puts "Reduced JIT list (available in jitlist.txt):" +puts result diff --git a/tool/zjit_iongraph.html b/tool/zjit_iongraph.html new file mode 100644 index 0000000000..993cce9045 --- /dev/null +++ b/tool/zjit_iongraph.html @@ -0,0 +1,551 @@ +<!-- Copyright Mozilla and licensed under Mozilla Public License Version 2.0. + Source can be found at https://github.com/mozilla-spidermonkey/iongraph --> +<!-- Generated by `npm run build-www` on + 39b04fa18f23cbf3fd2ca7339a45341ff3351ba1 in tekknolagi/iongraph --> +<!DOCTYPE html> + +<head> + <title>iongraph</title> + <style>/* iongraph-specific styles inspired by Tachyons */ + +:root { + --ig-size-1: 1rem; + --ig-size-2: 2rem; + --ig-size-3: 4rem; + --ig-size-4: 8rem; + --ig-size-5: 16rem; + + --ig-spacing-1: .25rem; + --ig-spacing-2: .5rem; + --ig-spacing-3: 1rem; + --ig-spacing-4: 2rem; + --ig-spacing-5: 4rem; + --ig-spacing-6: 8rem; + --ig-spacing-7: 16rem; + + --ig-text-color: black; + --ig-text-color-dim: #777; + --ig-background-primary: #ffb54e; + --ig-background-light: white; + --ig-border-color: #0c0c0d; + + --ig-block-header-color: #0c0c0d; + --ig-loop-header-color: #1fa411; + --ig-movable-color: #1048af; + --ig-rob-color: #444; + --ig-in-worklist-color: red; + + --ig-block-selected: #ffc863; + --ig-block-last-selected: #ffb54e; + + --ig-highlight-0: #ffb54e; + --ig-highlight-1: #ffb5c5; + --ig-highlight-2: #a4cbff; + --ig-highlight-3: #8be182; + --ig-highlight-4: #d9a4fd; + + --ig-flash-color: #ffb54e; + + /* + * The heatmap of sample counts will effectively be sampled from a gradient: + * + * |----------|---------------------------------------| + * cold "cool" hot + * + * The "cold" color will simply be transparent. Therefore, the "cool" + * threshold indicates where the instruction will be fully colored and + * noticeable to the user. + */ + --ig-hot-color: #ff849e; + --ig-cool-color: #ffe546; + --ig-cool-threshold: 0.2; +} + +a.ig-link-normal { + color: inherit; + text-decoration: inherit; +} + +.ig-flex { + display: flex; +} + +.ig-flex-column { + flex-direction: column; +} + +.ig-flex-basis-0 { + flex-basis: 0; +} + +.ig-flex-grow-1 { + flex-grow: 1; +} + +.ig-flex-shrink-0 { + flex-shrink: 0; +} + +.ig-flex-shrink-1 { + flex-shrink: 1; +} + +.ig-items-center { + align-items: center; +} + +.ig-relative { + position: relative; +} + +.ig-absolute { + position: absolute; +} + +.ig-absolute-fill { + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.ig-g1 { + gap: var(--ig-spacing-1); +} + +.ig-g2 { + gap: var(--ig-spacing-2); +} + +.ig-g3 { + gap: var(--ig-spacing-3); +} + +.ig-w1 { + width: var(--ig-size-1); +} + +.ig-w2 { + width: var(--ig-size-2); +} + +.ig-w3 { + width: var(--ig-size-3); +} + +.ig-w4 { + width: var(--ig-size-4); +} + +.ig-w5 { + width: var(--ig-size-5); +} + +.ig-w-100 { + width: 100%; +} + +.ig-ba { + border-style: solid; + border-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bt { + border-top-style: solid; + border-top-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-br { + border-right-style: solid; + border-right-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bb { + border-bottom-style: solid; + border-bottom-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bl { + border-left-style: solid; + border-left-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-pa1 { + padding: var(--ig-spacing-1); +} + +.ig-pa2 { + padding: var(--ig-spacing-2); +} + +.ig-pa3 { + padding: var(--ig-spacing-3); +} + +.ig-ph1 { + padding-left: var(--ig-spacing-1); + padding-right: var(--ig-spacing-1); +} + +.ig-ph2 { + padding-left: var(--ig-spacing-2); + padding-right: var(--ig-spacing-2); +} + +.ig-ph3 { + padding-left: var(--ig-spacing-3); + padding-right: var(--ig-spacing-3); +} + +.ig-pv1 { + padding-top: var(--ig-spacing-1); + padding-bottom: var(--ig-spacing-1); +} + +.ig-pv2 { + padding-top: var(--ig-spacing-2); + padding-bottom: var(--ig-spacing-2); +} + +.ig-pv3 { + padding-top: var(--ig-spacing-3); + padding-bottom: var(--ig-spacing-3); +} + +.ig-pt1 { + padding-top: var(--ig-spacing-1); +} + +.ig-pt2 { + padding-top: var(--ig-spacing-2); +} + +.ig-pt3 { + padding-top: var(--ig-spacing-3); +} + +.ig-pr1 { + padding-right: var(--ig-spacing-1); +} + +.ig-pr2 { + padding-right: var(--ig-spacing-2); +} + +.ig-pr3 { + padding-right: var(--ig-spacing-3); +} + +.ig-pb1 { + padding-bottom: var(--ig-spacing-1); +} + +.ig-pb2 { + padding-bottom: var(--ig-spacing-2); +} + +.ig-pb3 { + padding-bottom: var(--ig-spacing-3); +} + +.ig-pl1 { + padding-left: var(--ig-spacing-1); +} + +.ig-pl2 { + padding-left: var(--ig-spacing-2); +} + +.ig-pl3 { + padding-left: var(--ig-spacing-3); +} + +.ig-f1 { + font-size: 3rem; +} + +.ig-f2 { + font-size: 2.25rem; +} + +.ig-f3 { + font-size: 1.5rem; +} + +.ig-f4 { + font-size: 1.25rem; +} + +.ig-f5 { + font-size: 1rem; +} + +.ig-f6 { + font-size: .875rem; +} + +.ig-f7 { + font-size: .75rem; +} + +.ig-text-normal { + color: var(--ig-text-color); +} + +.ig-text-dim { + color: var(--ig-text-color-dim); +} + +.ig-tl { + text-align: left; +} + +.ig-tr { + text-align: right; +} + +.ig-tc { + text-align: center; +} + +.ig-bg-white { + background-color: var(--ig-background-light); +} + +.ig-bg-primary { + background-color: var(--ig-background-primary); +} + +.ig-overflow-hidden { + overflow: hidden; +} + +.ig-overflow-auto { + overflow: auto; +} + +.ig-overflow-x-auto { + overflow-x: auto; +} + +.ig-overflow-y-auto { + overflow-y: auto; +} + +.ig-hide-if-empty:empty { + display: none; +} + +/* Non-utility styles */ + +.ig-graph { + color: var(--ig-text-color); + position: absolute; + left: 0; + top: 0; + isolation: isolate; +} + +.ig-block { + position: absolute; + + .ig-block-header { + font-weight: bold; + text-align: center; + background-color: var(--ig-block-header-color); + color: white; + padding: 0 1em; + border: 1px solid var(--ig-border-color); + border-width: 1px 1px 0; + } + + .ig-instructions { + padding: 0.5em; + border: 1px solid var(--ig-border-color); + border-width: 0 1px 1px; + + table { + border-collapse: collapse; + } + + td, + th { + white-space: nowrap; + padding: 0.1em 0.5em; + } + + th { + font-weight: normal; + } + } + + &.ig-selected { + outline: 4px solid var(--ig-block-selected); + } + + &.ig-last-selected { + outline-color: var(--ig-block-last-selected); + } +} + +.ig-block-att-loopheader { + .ig-block-header { + background-color: var(--ig-loop-header-color); + } +} + +.ig-block-att-splitedge { + .ig-instructions { + border-style: dotted; + border-width: 0 2px 2px; + } +} + +.ig-ins-num { + text-align: right; + cursor: pointer; +} + +.ig-ins-type { + text-align: right; +} + +.ig-ins-samples { + font-size: 0.875em; + text-align: right; + cursor: pointer; +} + +.ig-use { + padding: 0 0.25em; + border-radius: 2px; + cursor: pointer; +} + +.ig-edge-label { + position: absolute; + font-size: 0.8em; + line-height: 1; + bottom: -1em; + padding-left: 4px; +} + +.ig-ins-att-RecoveredOnBailout { + color: var(--ig-rob-color); +} + +.ig-ins-att-Movable { + color: var(--ig-movable-color); +} + +.ig-ins-att-Guard { + text-decoration: underline; +} + +.ig-ins-att-InWorklist { + color: var(--ig-in-worklist-color); +} + +.ig-can-flash { + transition: outline-color 1s ease-out; + outline: 3px solid color-mix(in srgb, var(--ig-flash-color) 0%, transparent); +} + +.ig-flash { + transition: outline-color 0s; + outline-color: var(--ig-flash-color); +} + +.ig-hotness { + --ig-hotness: 0; + + --ig-cold-color: color-mix(in srgb, var(--ig-cool-color) 20%, transparent); + background-color: + /* cool <-> hot */ + color-mix(in oklab, + /* cold <-> cool */ + color-mix(in oklab, + /* dead or cold */ + color-mix(in srgb, transparent, var(--ig-cold-color) clamp(0%, calc(var(--ig-hotness) * 100000000%), 100%)), + var(--ig-cool-color) clamp(0%, calc((var(--ig-hotness) / var(--ig-cool-threshold)) * 100%), 100%)), + var(--ig-hot-color) clamp(0%, calc(((var(--ig-hotness) - var(--ig-cool-threshold)) / (1 - var(--ig-cool-threshold))) * 100%), 100%)); +} + +.ig-highlight { + --ig-highlight-color: transparent; + background-color: var(--ig-highlight-color); +}</style> + <style>* { + box-sizing: border-box; +} + +:root { + font-size: 0.875rem; +} + +body { + margin: 0; + background-color: #e5e8ea; + /* Font, and many other styles, taken from the Firefox Profiler/ */ + font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen-Sans, Ubuntu, Noto Sans, Liberation Sans, Cantarell, Helvetica Neue, sans-serif; +} + +#container { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +.tweaks-panel { + position: absolute; + bottom: 0; + right: 0; + padding: 1rem; + border: 1px solid black; + border-width: 1px 0 0 1px; + background-color: white; +}</style> +</head> + +<body> + <script>"use strict";var iongraph=(()=>{var J=Object.defineProperty;var mt=Object.getOwnPropertyDescriptor;var gt=Object.getOwnPropertyNames;var ft=Object.prototype.hasOwnProperty;var bt=(a,t)=>{for(var e in t)J(a,e,{get:t[e],enumerable:!0})},kt=(a,t,e,s)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of gt(t))!ft.call(a,i)&&i!==e&&J(a,i,{get:()=>t[i],enumerable:!(s=mt(t,i))||s.enumerable});return a};var yt=a=>kt(J({},"__esModule",{value:!0}),a);var Yt={};bt(Yt,{StandaloneUI:()=>et,WebUI:()=>tt});function X(a){a.version===void 0&&(a.version=0);for(let t of a.functions)xt(t,a.version);return a.version=1,a}function xt(a,t){for(let e of a.passes){for(let s of e.mir.blocks)wt(s,t);for(let s of e.lir.blocks)It(s,t)}return a}function wt(a,t){t===0&&(a.ptr=(a.id??a.number)+1,a.id=a.number);for(let e of a.instructions)vt(e,t);return a}function vt(a,t){return t===0&&(a.ptr=a.id),a}function It(a,t){t===0&&(a.ptr=a.id??a.number,a.id=a.number);for(let e of a.instructions)Lt(e,t);return a}function Lt(a,t){return t===0&&(a.ptr=a.id,a.mirPtr=null),a}function q(a,t,e){return Math.max(t,Math.min(e,a))}function R(a,t,e,s){return(a-t)*Math.pow(e,s)+t}function k(a,t,e=!1){if(!a)if(e)console.error(t??"Assertion failed");else throw new Error(t??"Assertion failed")}function L(a,t){return k(a,t),a}function Pt(a){return typeof a=="string"?document.createTextNode(a):a}function Bt(a,t){for(let e of t)e&&a.appendChild(Pt(e))}function x(a,t,e,s){let i=document.createElement(a);if(t&&t.length>0){let c=t.filter(l=>!!l);i.classList.add(...c)}return e?.(i),s&&Bt(i,s),i}var st=Object.prototype.hasOwnProperty;function E(a,t){var e,s;if(a===t)return!0;if(a&&t&&(e=a.constructor)===t.constructor){if(e===Date)return a.getTime()===t.getTime();if(e===RegExp)return a.toString()===t.toString();if(e===Array){if((s=a.length)===t.length)for(;s--&&E(a[s],t[s]););return s===-1}if(!e||typeof a=="object"){s=0;for(e in a)if(st.call(a,e)&&++s&&!st.call(t,e)||!(e in t)||!E(a[e],t[e]))return!1;return Object.keys(t).length===s}}return a!==a&&t!==t}function N(a,t,e={}){let s=t,i=[],c={get(){return s},set(l){s=l;for(let o of i)o(s)},valueOf(){return s},toString(){return String(s)},[Symbol.toPrimitive](l){return l==="string"?String(s):s},onChange(l){i.push(l)},initial:t,name:a,min:e.min??0,max:e.max??100,step:e.step??1};return(e.tweaksObject??K).add(c)}var F=class{constructor(t){this.container=t.container,this.tweaks=[],this.callbacks=[]}add(t){let e=this.tweaks.find(r=>r.name===t.name);if(e)return e;this.tweaks.push(t);let s=document.createElement("div");this.container.appendChild(s),s.style.display="flex",s.style.alignItems="center",s.style.justifyContent="end",s.style.gap="0.5rem";let i=t.name.replace(/[a-zA-Z0-9]/g,"_"),c=document.createElement("label");s.appendChild(c),c.innerText=t.name,c.htmlFor=`tweak-${i}-input`;let l=document.createElement("input");s.appendChild(l),l.type="number",l.value=String(t),l.id=`tweak-${i}-input`,l.style.width="4rem",l.addEventListener("input",()=>{t.set(l.valueAsNumber)});let o=document.createElement("input");s.appendChild(o),o.type="range",o.value=String(t),o.min=String(t.min),o.max=String(t.max),o.step=t.step===0?"any":String(t.step),o.addEventListener("input",()=>{t.set(o.valueAsNumber)});let n=document.createElement("button");return s.appendChild(n),n.innerText="Reset",n.disabled=t.get()===t.initial,n.addEventListener("click",()=>{t.set(t.initial)}),t.onChange(r=>{l.value=String(r),o.value=String(r),n.disabled=t.get()===t.initial;for(let d of this.callbacks)d(t)}),t}onTweak(t){this.callbacks.push(t)}},nt=document.createElement("div");nt.classList.add("tweaks-panel");var K=new F({container:nt});K.onTweak(a=>{window.dispatchEvent(new CustomEvent("tweak",{detail:a}))});window.tweaks=K;var Nt=document.createElement("div"),St=new F({container:Nt}),ot=N("Test Value",3,{tweaksObject:St});ot.set(4);ot=4;var dt=N("Debug?",0,{min:0,max:1}),P=20,C=44,I=16,S=60,T=12,H=36,it=16,D=16,Tt=N("Layout Iterations",2,{min:0,max:6}),rt=N("Nearly Straight Threshold",30,{min:0,max:200}),$t=N("Nearly Straight Iterations",8,{min:0,max:10}),at=N("Stop At Pass",30,{min:0,max:30}),Mt=1.5,Et=.01,Ct=1,Ht=.1,z=40;function _(a){return a.mir.attributes.includes("loopheader")}function Dt(a){return a.loopHeight!==void 0}function $(a){if(k(a),Dt(a))return a;throw new Error(`Block ${a.id} is not a pseudo LoopHeader`)}var j=1,lt=2,U=4,O=0,W=1,w=new Proxy(console,{get(a,t){let e=a[t];return typeof e!="function"?e:+dt?e.bind(a):()=>{}}}),V=class{constructor(t,e,s={}){this.viewport=t;let i=t.getBoundingClientRect();this.viewportSize={x:i.width,y:i.height},this.graphContainer=document.createElement("div"),this.graphContainer.classList.add("ig-graph"),this.graphContainer.style.transformOrigin="top left",this.viewport.appendChild(this.graphContainer),this.sampleCounts=s.sampleCounts,this.maxSampleCounts=[0,0],this.heatmapMode=O;for(let[n,r]of this.sampleCounts?.totalLineHits??[])this.maxSampleCounts[O]=Math.max(this.maxSampleCounts[O],r);for(let[n,r]of this.sampleCounts?.selfLineHits??[])this.maxSampleCounts[W]=Math.max(this.maxSampleCounts[W],r);this.size={x:0,y:0},this.numLayers=0,this.zoom=1,this.translation={x:0,y:0},this.animating=!1,this.targetZoom=1,this.targetTranslation={x:0,y:0},this.startMousePos={x:0,y:0},this.lastMousePos={x:0,y:0},this.selectedBlockPtrs=new Set,this.lastSelectedBlockPtr=0,this.nav={visited:[],currentIndex:-1,siblings:[]},this.highlightedInstructions=[],this.instructionPalette=s.instructionPalette??[0,1,2,3,4].map(n=>`var(--ig-highlight-${n})`),this.blocks=e.mir.blocks.map(n=>{let r={ptr:n.ptr,id:n.id,mir:n,lir:e.lir.blocks.find(d=>d.id===n.id)??null,preds:[],succs:[],el:void 0,size:{x:0,y:0},layer:-1,loopID:-1,layoutNode:void 0};if(r.mir.attributes.includes("loopheader")){let d=r;d.loopHeight=0,d.parentLoop=null,d.outgoingEdges=[]}return k(r.ptr,"blocks must always have non-null ptrs"),r}),this.blocksByID=new Map,this.blocksByPtr=new Map,this.insPtrsByID=new Map,this.insIDsByPtr=new Map,this.loops=[];for(let n of this.blocks){this.blocksByID.set(n.id,n),this.blocksByPtr.set(n.ptr,n);for(let r of n.mir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id);if(n.lir)for(let r of n.lir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id)}for(let n of this.blocks)if(n.preds=n.mir.predecessors.map(r=>L(this.blocksByID.get(r))),n.succs=n.mir.successors.map(r=>L(this.blocksByID.get(r))),_(n)){let r=n.preds.filter(d=>d.mir.attributes.includes("backedge"));k(r.length===1),n.backedge=r[0]}for(let n of this.blocks)n.el=this.renderBlock(n);for(let n of this.blocks)n.size={x:n.el.clientWidth,y:n.el.clientHeight};let[c,l,o]=this.layout();this.render(c,l,o),this.addEventListeners()}layout(){let[t,e]=this.findLayoutRoots();w.log("Layout roots:",t.map(l=>l.id));for(let l of[...t,...e]){let o=l;o.loopHeight=0,o.parentLoop=null,o.outgoingEdges=[],Object.defineProperty(o,"backedge",{get(){throw new Error("Accessed .backedge on a pseudo loop header! Don't do that.")},configurable:!0})}for(let l of t)w.group("findLoops"),this.findLoops(l),w.groupEnd();for(let l of t)w.group("layer"),this.layer(l),w.groupEnd();for(let l of e)l.layer=0,l.loopID=l.id;let s=this.makeLayoutNodes();this.straightenEdges(s);let i=this.finagleJoints(s),c=this.verticalize(s,i);return[s,c,i]}findLayoutRoots(){let t=[],e=[],s=this.blocks.filter(i=>i.preds.length===0);for(let i of s){let c=i;if(i.mir.attributes.includes("osr")){k(i.succs.length>0);let l=i.succs[0];c=l;for(let o=0;;o++){if(o>=1e7)throw new Error("likely infinite loop");let n=c.preds.filter(r=>!r.mir.attributes.includes("osr")&&!r.mir.attributes.includes("backedge"));if(n.length===0)break;c=n[0]}c!==l?e.push(i):c=i}t.includes(c)||t.push(c)}return[t,e]}findLoops(t,e=null){if(e===null&&(e=[t.id]),!(t.loopID>=0)){if(w.log("block:",t.id,t.mir.loopDepth,"loopIDsByDepth:",e),w.log(t.mir.attributes),_(t)){let s=e[e.length-1],i=$(this.blocksByID.get(s));t.parentLoop=i,e=[...e,t.id],w.log("Block",t.id,"is true loop header, loopIDsByDepth is now",e)}if(t.mir.loopDepth>e.length-1&&(t.mir.loopDepth=e.length-1,w.log("Block",t.id,"has been forced back to loop depth",t.mir.loopDepth)),t.mir.loopDepth<e.length-1&&(e=e.slice(0,t.mir.loopDepth+1),w.log("Block",t.id,"has low loop depth, therefore we exited a loop. loopIDsByDepth:",e)),t.loopID=e[t.mir.loopDepth],!t.mir.attributes.includes("backedge"))for(let s of t.succs)this.findLoops(s,e)}}layer(t,e=0){if(w.log("block",t.id,"layer",e),t.mir.attributes.includes("backedge")){t.layer=t.succs[0].layer;return}if(e<=t.layer)return;t.layer=Math.max(t.layer,e),this.numLayers=Math.max(t.layer+1,this.numLayers);let s=$(this.blocksByID.get(t.loopID));for(;s;)s.loopHeight=Math.max(s.loopHeight,t.layer-s.layer+1),s=s.parentLoop;for(let i of t.succs)i.mir.loopDepth<t.mir.loopDepth?$(this.blocksByID.get(t.loopID)).outgoingEdges.push(i):this.layer(i,e+1);if(_(t))for(let i of t.outgoingEdges)this.layer(i,e+t.loopHeight)}makeLayoutNodes(){w.group("makeLayoutNodes");function t(o,n,r){o.dstNodes[n]=r,r.srcNodes.includes(o)||r.srcNodes.push(o),w.log("connected",o.id,"to",r.id)}let e;{let o={};for(let n of this.blocks)o[n.layer]||(o[n.layer]=[]),o[n.layer].push(n);e=Object.entries(o).map(([n,r])=>[Number(n),r]).sort((n,r)=>n[0]-r[0]).map(([n,r])=>r)}let s=0,i=e.map(()=>[]),c=[],l=new Map;for(let[o,n]of e.entries()){w.group("layer",o,"blocks",n.map(u=>u.id));let r=[];for(let u of n)for(let h=c.length-1;h>=0;h--){let p=c[h];p.dstBlock===u&&(r.unshift(p),c.splice(h,1))}let d=new Map;for(let u of c){let h,p=d.get(u.dstBlock.id);if(p)t(u.src,u.srcPort,p),h=p;else{let m={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:u.dstBlock,jointOffsets:[],flags:0};t(u.src,u.srcPort,m),i[o].push(m),d.set(u.dstBlock.id,m),h=m,w.log("Created dummy",h.id,"on the way to block",u.dstBlock.id)}u.src=h,u.srcPort=0}let b=[];for(let u of n){let h=$(this.blocksByID.get(u.loopID));for(;_(h);){let p=b.find(f=>f.loopID===h.id);p?p.block=u:b.push({loopID:h.id,block:u});let m=h.parentLoop;if(!m)break;h=m}}let g=[];for(let u of n){let h={id:s++,pos:{x:P,y:P},size:u.size,block:u,srcNodes:[],dstNodes:[],jointOffsets:[],flags:0};for(let p of r)p.dstBlock===u&&t(p.src,p.srcPort,h);i[o].push(h),u.layoutNode=h;for(let p of b.filter(m=>m.block===u)){let m=$(this.blocksByID.get(p.loopID)).backedge,f={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:m,jointOffsets:[],flags:0},y=l.get(m);y?t(f,0,y):(f.flags|=U,t(f,0,m.layoutNode)),i[o].push(f),l.set(m,f)}if(u.mir.attributes.includes("backedge"))t(u.layoutNode,0,u.succs[0].layoutNode);else for(let[p,m]of u.succs.entries())m.mir.attributes.includes("backedge")?g.push({src:h,srcPort:p,dstBlock:m}):c.push({src:h,srcPort:p,dstBlock:m})}for(let u of g){let h=L(l.get(u.dstBlock));t(u.src,u.srcPort,h)}w.groupEnd()}w.log("Pruning backedge dummies");{let o=[];for(let r of At(i))r.srcNodes.length===0&&o.push(r);let n=new Set;for(let r of o){let d=r;for(;d.block===null&&d.srcNodes.length===0;)Ot(d),n.add(d),k(d.dstNodes.length===1),d=d.dstNodes[0]}for(let r of i)for(let d=r.length-1;d>=0;d--)n.has(r[d])&&r.splice(d,1)}w.log("Marking leftmost and rightmost dummies");for(let o of i){for(let n=0;n<o.length&&o[n].block===null;n++)o[n].flags|=j;for(let n=o.length-1;n>=0&&o[n].block===null;n--)o[n].flags|=lt}w.log("Verifying integrity of all nodes");for(let o of i)for(let n of o){n.block?k(n.dstNodes.length===n.block.succs.length,`expected node ${n.id} for block ${n.block.id} to have ${n.block.succs.length} destination nodes, but got ${n.dstNodes.length} instead`):k(n.dstNodes.length===1,`expected dummy node ${n.id} to have only one destination node, but got ${n.dstNodes.length} instead`);for(let r=0;r<n.dstNodes.length;r++)k(n.dstNodes[r]!==void 0,`dst slot ${r} of node ${n.id} was undefined`)}return w.groupEnd(),i}straightenEdges(t){let e=g=>{for(let u=0;u<g.length-1;u++){let h=g[u],p=g[u+1],m=h.block===null&&p.block!==null,f=h.pos.x+h.size.x+(m?I:0)+C;p.pos.x=Math.max(p.pos.x,f)}},s=()=>{for(let g of t)for(let u of g){if(u.block===null)continue;let h=u.block.loopID!==null?$(this.blocksByID.get(u.block.loopID)):null;if(h){let p=h.layoutNode;u.pos.x=Math.max(u.pos.x,p.pos.x)}}},i=()=>{let g=new Map;for(let u of Q(t)){let h=u.dstBlock,p=u.pos.x;g.set(h,Math.max(g.get(h)??0,p))}for(let u of Q(t)){let h=u.dstBlock,p=g.get(h);k(p,`no position for backedge ${h.id}`),u.pos.x=p}for(let u of t)e(u)},c=()=>{let g=new Map;for(let u of t){let h=0,p=0;for(;h<u.length;h++)if(!(u[h].flags&j)){p=u[h].pos.x;break}for(h-=1,p-=C+I;h>=0;h--){let m=u[h];k(m.block===null&&m.flags&j);let f=p;for(let y of m.srcNodes){let v=y.pos.x+y.dstNodes.indexOf(m)*S;v<f&&(f=v)}m.pos.x=f,p=m.pos.x-C,g.set(m.dstBlock,Math.min(g.get(m.dstBlock)??1/0,f))}}for(let u of Q(t)){if(!(u.flags&j))continue;let h=g.get(u.dstBlock);k(h,`no position for run to block ${u.dstBlock.id}`),u.pos.x=h}},l=()=>{for(let g=0;g<t.length-1;g++){let u=t[g];e(u);let h=-1;for(let p of u)for(let[m,f]of p.dstNodes.entries()){let y=t[g+1].indexOf(f);if(y>h&&f.srcNodes[0]===p){let v=I+S*m,B=I,Z=f.pos.x;f.pos.x=Math.max(f.pos.x,p.pos.x+v-B),f.pos.x!==Z&&(h=y)}}}},o=()=>{for(let g of t){for(let u=g.length-1;u>=0;u--){let h=g[u];if(!h.block||h.block.mir.attributes.includes("backedge"))continue;let p=[];for(let m of h.srcNodes){let f=I+m.dstNodes.indexOf(h)*S,y=I;p.push(m.pos.x+f-(h.pos.x+y))}for(let[m,f]of h.dstNodes.entries()){if(f.block===null&&f.dstBlock.mir.attributes.includes("backedge"))continue;let y=I+m*S,v=I;p.push(f.pos.x+v-(h.pos.x+y))}if(!p.includes(0)){p=p.filter(m=>m>0).sort((m,f)=>m-f);for(let m of p){let f=!1;for(let y=u+1;y<g.length;y++){let v=g[y];if(v.flags<)continue;let B=h.pos.x+m,Z=h.pos.x+m+h.size.x,ht=v.pos.x-C,pt=v.pos.x+v.size.x+C;Z>=ht&&B<=pt&&(f=!0)}if(!f){h.pos.x+=m;break}}}}e(g)}},n=()=>{for(let g=t.length-1;g>=0;g--){let u=t[g];e(u);for(let h of u)for(let p of h.srcNodes){if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}},r=()=>{for(let g=0;g<t.length;g++){let u=t[g];e(u);for(let h of u){if(h.dstNodes.length===0)continue;let p=h.dstNodes[0];if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}};function d(g,u){let h=[];for(let p=0;p<u;p++)for(let m of g)h.push(m);return h}let b=[...d([l,s,i],Tt),i,...d([n,r],$t),o,i,c];k(b.length<=(at.initial??1/0),`STOP_AT_PASS was too small - should be at least ${b.length}`),w.group("Running passes");for(let[g,u]of b.entries())g<at&&(w.log(u.name??u.toString()),u());w.groupEnd()}finagleJoints(t){let e=[];for(let s of t){let i=[];for(let r of s)if(r.jointOffsets=new Array(r.dstNodes.length).fill(0),!r.block?.mir.attributes.includes("backedge"))for(let[d,b]of r.dstNodes.entries()){let g=r.pos.x+I+S*d,u=b.pos.x+I;Math.abs(u-g)<2*T||i.push({x1:g,x2:u,src:r,srcPort:d,dst:b})}i.sort((r,d)=>r.x1-d.x1);let c=[],l=[];t:for(let r of i){let d=r.x2-r.x1>=0?c:l,b=null;for(let g=d.length-1;g>=0;g--){let u=d[g],h=!1;for(let p of u){if(r.dst===p.dst){u.push(r);continue t}let m=Math.min(r.x1,r.x2),f=Math.max(r.x1,r.x2),y=Math.min(p.x1,p.x2),v=Math.max(p.x1,p.x2);if(f>=y&&m<=v){h=!0;break}}if(h)break;b=u}b?b.push(r):d.push([r])}let o=Math.max(0,c.length+l.length-1)*it,n=-o/2;for(let r of[...c.reverse(),...l]){for(let d of r)d.src.jointOffsets[d.srcPort]=n;n+=it}e.push(o)}return k(e.length===t.length),e}verticalize(t,e){let s=new Array(t.length),i=P;for(let c=0;c<t.length;c++){let l=t[c],o=0;for(let n of l)n.pos.y=i,o=Math.max(o,n.size.y);s[c]=o,i+=o+H+e[c]+H}return s}renderBlock(t){let e=document.createElement("div");this.graphContainer.appendChild(e),e.classList.add("ig-block","ig-bg-white");for(let o of t.mir.attributes)e.classList.add(`ig-block-att-${o}`);e.setAttribute("data-ig-block-ptr",`${t.ptr}`),e.setAttribute("data-ig-block-id",`${t.id}`);let s="";t.mir.attributes.includes("loopheader")?s=" (loop header)":t.mir.attributes.includes("backedge")?s=" (backedge)":t.mir.attributes.includes("splitedge")&&(s=" (split edge)");let i=document.createElement("div");i.classList.add("ig-block-header"),i.innerText=`Block ${t.id}${s}`,e.appendChild(i);let c=document.createElement("div");c.classList.add("ig-instructions"),e.appendChild(c);let l=document.createElement("table");if(t.lir){l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + ${this.sampleCounts?` + <col style="width: 1px"> + <col style="width: 1px"> + `:""} + </colgroup> + ${this.sampleCounts?` + <thead> + <tr> + <th></th> + <th></th> + <th class="ig-f6">Total</th> + <th class="ig-f6">Self</th> + </tr> + </thead> + `:""} + `;for(let o of t.lir.instructions)l.appendChild(this.renderLIRInstruction(o))}else{l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + <col style="width: 1px"> + </colgroup> + `;for(let o of t.mir.instructions)l.appendChild(this.renderMIRInstruction(o))}if(c.appendChild(l),t.succs.length===2)for(let[o,n]of[1,0].entries()){let r=document.createElement("div");r.innerText=`${n}`,r.classList.add("ig-edge-label"),r.style.left=`${I+S*o}px`,e.appendChild(r)}return i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",o=>{o.stopPropagation(),o.shiftKey||this.selectedBlockPtrs.clear(),this.setSelection([],t.ptr)}),e}render(t,e,s){for(let n of t)for(let r of n)if(r.block!==null){let d=r.block;d.el.style.left=`${r.pos.x}px`,d.el.style.top=`${r.pos.y}px`}let i=0,c=0;for(let n of t)for(let r of n)i=Math.max(i,r.pos.x+r.size.x+P),c=Math.max(c,r.pos.y+r.size.y+P);let l=document.createElementNS("http://www.w3.org/2000/svg","svg");this.graphContainer.appendChild(l);let o=(n,r)=>{for(let d of n)i=Math.max(i,d+P);for(let d of r)c=Math.max(c,d+P)};for(let n=0;n<t.length;n++){let r=t[n];for(let d of r){d.block||k(d.dstNodes.length===1,`dummy nodes must have exactly one destination, but dummy ${d.id} had ${d.dstNodes.length}`),k(d.dstNodes.length===d.jointOffsets.length,"must have a joint offset for each destination");for(let[b,g]of d.dstNodes.entries()){let u=d.pos.x+I+S*b,h=d.pos.y+d.size.y;if(d.block?.mir.attributes.includes("backedge")){let p=d.block.succs[0],m=d.pos.x,f=d.pos.y+D,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=jt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(d.flags&U){let p=L(g.block),m=d.pos.x+I,f=d.pos.y+D+T,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=zt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(g.block===null&&g.dstBlock.mir.attributes.includes("backedge")){let p=g.pos.x+I,m=g.pos.y+(g.flags&U?D+T:0);if(d.block===null){let f=h-H,y=Ft(u,h,p,m,f,!1);l.appendChild(y),o([u,p],[h,m,f])}else{let f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=_t(u,h,p,m,f);l.appendChild(y),o([u,p],[h,m,f])}}else{let p=g.pos.x+I,m=g.pos.y,f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=Rt(u,h,p,m,f,g.block!==null);l.appendChild(y),o([u,p],[h,m,f])}}}}if(l.setAttribute("width",`${i}`),l.setAttribute("height",`${c}`),this.size={x:i,y:c},+dt)for(let n of t)for(let r of n){let d=document.createElement("div");d.innerHTML=`${r.id}<br><- ${r.srcNodes.map(b=>b.id)}<br>-> ${r.dstNodes.map(b=>b.id)}<br>${r.flags}`,d.style.position="absolute",d.style.border="1px solid black",d.style.backgroundColor="white",d.style.left=`${r.pos.x}px`,d.style.top=`${r.pos.y}px`,d.style.whiteSpace="nowrap",this.graphContainer.appendChild(d)}this.updateHighlightedInstructions(),this.updateHotness()}renderMIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-mir","ig-can-flash",...t.attributes.map(o=>`ig-ins-att-${o}`)),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=`v${t.id}`,s.appendChild(i);let c=document.createElement("td");c.innerHTML=e.replace(/(v)(\d+)/g,(o,n,r)=>`<span class="ig-use ig-highlightable" data-ig-use="${r}">${n}${r}</span>`),s.appendChild(c);let l=document.createElement("td");return l.classList.add("ig-ins-type"),l.innerText=t.type==="None"?"":t.type,s.appendChild(l),i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),c.querySelectorAll(".ig-use").forEach(o=>{o.addEventListener("pointerdown",n=>{n.preventDefault(),n.stopPropagation()}),o.addEventListener("click",n=>{let r=parseInt(L(o.getAttribute("data-ig-use")),10);this.jumpToInstruction(r,{zoom:1})})}),s}renderLIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-lir","ig-hotness"),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=String(t.id),s.appendChild(i);let c=document.createElement("td");if(c.innerText=e,s.appendChild(c),this.sampleCounts){let l=this.sampleCounts?.totalLineHits.get(t.id)??0,o=this.sampleCounts?.selfLineHits.get(t.id)??0,n=document.createElement("td");n.classList.add("ig-ins-samples"),n.classList.toggle("ig-text-dim",l===0),n.innerText=`${l}`,n.title="Color by total count",s.appendChild(n);let r=document.createElement("td");r.classList.add("ig-ins-samples"),r.classList.toggle("ig-text-dim",o===0),r.innerText=`${o}`,r.title="Color by self count",s.appendChild(r);for(let[d,b]of[n,r].entries())b.addEventListener("pointerdown",g=>{g.preventDefault(),g.stopPropagation()}),b.addEventListener("click",()=>{k(d===O||d===W),this.heatmapMode=d,this.updateHotness()})}return i.addEventListener("pointerdown",l=>{l.preventDefault(),l.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),s}renderSelection(){this.graphContainer.querySelectorAll(".ig-block").forEach(t=>{let e=parseInt(L(t.getAttribute("data-ig-block-ptr")),10);t.classList.toggle("ig-selected",this.selectedBlockPtrs.has(e)),t.classList.toggle("ig-last-selected",this.lastSelectedBlockPtr===e)})}removeNonexistentHighlights(){this.highlightedInstructions=this.highlightedInstructions.filter(t=>this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`))}updateHighlightedInstructions(){for(let t of this.highlightedInstructions)k(this.highlightedInstructions.filter(e=>e.ptr===t.ptr).length===1,`instruction ${t.ptr} was highlighted more than once`);this.graphContainer.querySelectorAll(".ig-ins, .ig-use").forEach(t=>{Vt(t)});for(let t of this.highlightedInstructions){let e=this.instructionPalette[t.paletteColor%this.instructionPalette.length],s=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`);if(s){ct(s,e);let i=this.insIDsByPtr.get(t.ptr);this.graphContainer.querySelectorAll(`.ig-use[data-ig-use="${i}"]`).forEach(c=>{ct(c,e)})}}}updateHotness(){this.graphContainer.querySelectorAll(".ig-ins-lir").forEach(t=>{k(t.classList.contains("ig-hotness"));let e=parseInt(L(t.getAttribute("data-ig-ins-id")),10),s=0;this.sampleCounts&&(s=((this.heatmapMode===O?this.sampleCounts.totalLineHits:this.sampleCounts.selfLineHits).get(e)??0)/this.maxSampleCounts[this.heatmapMode]),t.style.setProperty("--ig-hotness",`${s}`)})}addEventListeners(){this.viewport.addEventListener("wheel",e=>{e.preventDefault();let s=this.zoom;if(e.ctrlKey){s=Math.max(Ht,Math.min(Ct,this.zoom*Math.pow(Mt,-e.deltaY*Et)));let c=s/this.zoom-1;this.zoom=s;let{x:l,y:o}=this.viewport.getBoundingClientRect(),n=e.clientX-l-this.translation.x,r=e.clientY-o-this.translation.y;this.translation.x-=n*c,this.translation.y-=r*c}else this.translation.x-=e.deltaX,this.translation.y-=e.deltaY;let i=this.clampTranslation(this.translation,s);this.translation.x=i.x,this.translation.y=i.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerdown",e=>{e.pointerType==="mouse"&&!(e.button===0||e.button===1)||(e.preventDefault(),this.viewport.setPointerCapture(e.pointerId),this.startMousePos={x:e.clientX,y:e.clientY},this.lastMousePos={x:e.clientX,y:e.clientY},this.animating=!1)}),this.viewport.addEventListener("pointermove",e=>{if(!this.viewport.hasPointerCapture(e.pointerId))return;let s=e.clientX-this.lastMousePos.x,i=e.clientY-this.lastMousePos.y;this.translation.x+=s,this.translation.y+=i,this.lastMousePos={x:e.clientX,y:e.clientY};let c=this.clampTranslation(this.translation,this.zoom);this.translation.x=c.x,this.translation.y=c.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerup",e=>{this.viewport.releasePointerCapture(e.pointerId);let s=2,i=this.startMousePos.x-e.clientX,c=this.startMousePos.y-e.clientY;Math.abs(i)<=s&&Math.abs(c)<=s&&this.setSelection([]),this.animating=!1}),new ResizeObserver(e=>{k(e.length===1);let s=e[0].contentRect;this.viewportSize.x=s.width,this.viewportSize.y=s.height}).observe(this.viewport)}setSelection(t,e=0){this.setSelectionRaw(t,e),e?this.nav={visited:[e],currentIndex:0,siblings:[e]}:this.nav={visited:[],currentIndex:-1,siblings:[]}}setSelectionRaw(t,e){this.selectedBlockPtrs.clear();for(let s of[...t,e])this.blocksByPtr.has(s)&&this.selectedBlockPtrs.add(s);this.lastSelectedBlockPtr=this.blocksByPtr.has(e)?e:0,this.renderSelection()}navigate(t){let e=this.lastSelectedBlockPtr;if(t==="down"||t==="up")if(e){let s=L(this.blocksByPtr.get(e)),i=(t==="down"?s.succs:s.preds).map(l=>l.ptr);s.ptr!==this.nav.visited[this.nav.currentIndex]&&(this.nav.visited=[s.ptr],this.nav.currentIndex=0);let c=this.nav.currentIndex+(t==="down"?1:-1);if(0<=c&&c<this.nav.visited.length)this.nav.currentIndex=c,this.nav.siblings=i;else{let l=i[0];l!==void 0&&(t==="down"?(this.nav.visited.push(l),this.nav.currentIndex+=1,k(this.nav.currentIndex===this.nav.visited.length-1)):(this.nav.visited.unshift(l),k(this.nav.currentIndex===0)),this.nav.siblings=i)}this.setSelectionRaw([],this.nav.visited[this.nav.currentIndex])}else{let s=[...this.blocks].sort((n,r)=>n.id-r.id),i=s.filter(n=>n.preds.length===0),c=s.filter(n=>n.succs.length===0),l=t==="down"?i:c,o=l[0];k(o),this.setSelectionRaw([],o.ptr),this.nav={visited:[o.ptr],currentIndex:0,siblings:l.map(n=>n.ptr)}}else if(e!==void 0){let s=this.nav.siblings.indexOf(e);k(s>=0,"currently selected node should be in siblings array");let i=s+(t==="right"?1:-1);0<=i&&i<this.nav.siblings.length&&this.setSelectionRaw([],this.nav.siblings[i])}k(this.nav.visited.length===0||this.nav.siblings.includes(this.nav.visited[this.nav.currentIndex]),"expected currently visited node to be in the siblings array"),k(this.lastSelectedBlockPtr===0||this.nav.siblings.includes(this.lastSelectedBlockPtr),"expected currently selected block to be in siblings array")}toggleInstructionHighlight(t,e){this.removeNonexistentHighlights();let s=this.highlightedInstructions.findIndex(c=>c.ptr===t),i=s>=0;if(e!==void 0&&(i=!e),i)s>=0&&this.highlightedInstructions.splice(s,1);else if(s<0){let c=0;for(;;){if(this.highlightedInstructions.find(l=>l.paletteColor===c)){c+=1;continue}break}this.highlightedInstructions.push({ptr:t,paletteColor:c})}this.updateHighlightedInstructions()}clampTranslation(t,e){let s=z-this.size.x*e,i=this.viewportSize.x-z,c=z-this.size.y*e,l=this.viewportSize.y-z,o=q(t.x,s,i),n=q(t.y,c,l);return{x:o,y:n}}updatePanAndZoom(){let t=this.clampTranslation(this.translation,this.zoom);this.graphContainer.style.transform=`translate(${t.x}px, ${t.y}px) scale(${this.zoom})`}graph2viewport(t,e=this.translation,s=this.zoom){return{x:t.x*s+e.x,y:t.y*s+e.y}}viewport2graph(t,e=this.translation,s=this.zoom){return{x:(t.x-e.x)/s,y:(t.y-e.y)/s}}async goToGraphCoordinates(t,{zoom:e=this.zoom,animate:s=!0}){let i={x:-t.x*e,y:-t.y*e};if(!s){this.animating=!1,this.translation.x=i.x,this.translation.y=i.y,this.zoom=e,this.updatePanAndZoom(),await new Promise(l=>setTimeout(l,0));return}if(this.targetTranslation=i,this.targetZoom=e,this.animating)return;this.animating=!0;let c=performance.now();for(;this.animating;){let l=await new Promise(h=>requestAnimationFrame(h)),o=(l-c)/1e3;c=l;let n=1,r=.01,d=1e-6,b=this.targetTranslation.x-this.translation.x,g=this.targetTranslation.y-this.translation.y,u=this.targetZoom-this.zoom;if(this.translation.x=R(this.translation.x,this.targetTranslation.x,d,o),this.translation.y=R(this.translation.y,this.targetTranslation.y,d,o),this.zoom=R(this.zoom,this.targetZoom,d,o),this.updatePanAndZoom(),Math.abs(b)<=n&&Math.abs(g)<=n&&Math.abs(u)<=r){this.translation.x=this.targetTranslation.x,this.translation.y=this.targetTranslation.y,this.zoom=this.targetZoom,this.animating=!1,this.updatePanAndZoom();break}}await new Promise(l=>setTimeout(l,0))}jumpToBlock(t,{zoom:e=this.zoom,animate:s=!0,viewportPos:i}={}){let c=this.blocksByPtr.get(t);if(!c)return Promise.resolve();let l;return i?l={x:c.layoutNode.pos.x-i.x/e,y:c.layoutNode.pos.y-i.y/e}:l=this.graphPosToCenterRect(c.layoutNode.pos,c.layoutNode.size,e),this.goToGraphCoordinates(l,{zoom:e,animate:s})}async jumpToInstruction(t,{zoom:e=this.zoom,animate:s=!0}){let i=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-id="${t}"]`);if(!i)return;let c=i.getBoundingClientRect(),l=this.graphContainer.getBoundingClientRect(),o=(c.x-l.x)/this.zoom,n=(c.y-l.y)/this.zoom,r=c.width/this.zoom,d=c.height/this.zoom,b=this.graphPosToCenterRect({x:o,y:n},{x:r,y:d},e);i.classList.add("ig-flash"),await this.goToGraphCoordinates(b,{zoom:e,animate:s}),i.classList.remove("ig-flash")}graphPosToCenterRect(t,e,s){let i=this.viewportSize.x/s,c=this.viewportSize.y/s,l=Math.max(20/s,(i-e.x)/2),o=Math.max(20/s,(c-e.y)/2),n=t.x-l,r=t.y-o;return{x:n,y:r}}exportState(){let t={translation:this.translation,zoom:this.zoom,heatmapMode:this.heatmapMode,highlightedInstructions:this.highlightedInstructions,selectedBlockPtrs:this.selectedBlockPtrs,lastSelectedBlockPtr:this.lastSelectedBlockPtr,viewportPosOfSelectedBlock:void 0};return this.lastSelectedBlockPtr&&(t.viewportPosOfSelectedBlock=this.graph2viewport(L(this.blocksByPtr.get(this.lastSelectedBlockPtr)).layoutNode.pos)),t}restoreState(t,e){this.translation.x=t.translation.x,this.translation.y=t.translation.y,this.zoom=t.zoom,this.heatmapMode=t.heatmapMode,this.highlightedInstructions=t.highlightedInstructions,this.setSelection(Array.from(t.selectedBlockPtrs),t.lastSelectedBlockPtr),this.updatePanAndZoom(),this.updateHotness(),this.updateHighlightedInstructions(),e.preserveSelectedBlockPosition&&this.jumpToBlock(this.lastSelectedBlockPtr,{zoom:this.zoom,animate:!1,viewportPos:t.viewportPosOfSelectedBlock})}};function Ot(a){for(let t of a.dstNodes){let e=t.srcNodes.indexOf(a);k(e!==-1),t.srcNodes.splice(e,1)}}function*Q(a){for(let t of a)for(let e of t)e.block===null&&(yield e)}function*At(a){for(let t of a)for(let e of t)e.block===null&&e.dstBlock.mir.attributes.includes("backedge")&&(yield e)}function Rt(a,t,e,s,i,c,l=1){let o=T;k(t+o<=i&&i<s-o,`downward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i-o} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${e} ${i+o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,180);r.appendChild(b)}return r}function Ft(a,t,e,s,i,c,l=1){let o=T;k(s+o<=i&&i<=t-o,`upward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i+o} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${e} ${i-o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,0);r.appendChild(b)}return r}function zt(a,t,e,s,i=1){let c=T;k(t-c>=s&&a-c>=e,`to backedge: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, r = ${c} `,!0),i%2===1&&(a+=.5,s+=.5);let l="";l+=`M ${a} ${t} `,l+=`A ${c} ${c} 0 0 0 ${a-c} ${s} `,l+=`L ${e} ${s} `;let o=document.createElementNS("http://www.w3.org/2000/svg","g"),n=document.createElementNS("http://www.w3.org/2000/svg","path");n.setAttribute("d",l),n.setAttribute("fill","none"),n.setAttribute("stroke","black"),n.setAttribute("stroke-width",`${i} `),o.appendChild(n);let r=G(e,s,270);return o.appendChild(r),o}function _t(a,t,e,s,i,c=1){let l=T;k(t+l<=i&&a<=e&&s<=t,`block to backedge dummy: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${l} `,!0),c%2===1&&(a+=.5,e+=.5,i+=.5);let o="";o+=`M ${a} ${t} `,o+=`L ${a} ${i-l} `,o+=`A ${l} ${l} 0 0 0 ${a+l} ${i} `,o+=`L ${e-l} ${i} `,o+=`A ${l} ${l} 0 0 0 ${e} ${i-l} `,o+=`L ${e} ${s} `;let n=document.createElementNS("http://www.w3.org/2000/svg","g"),r=document.createElementNS("http://www.w3.org/2000/svg","path");return r.setAttribute("d",o),r.setAttribute("fill","none"),r.setAttribute("stroke","black"),r.setAttribute("stroke-width",`${c} `),n.appendChild(r),n}function jt(a,t,e,s,i=1){k(e<a&&s===t,`x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s} `,!0),i%2===1&&(t+=.5,s+=.5);let c="";c+=`M ${a} ${t} `,c+=`L ${e} ${s} `;let l=document.createElementNS("http://www.w3.org/2000/svg","g"),o=document.createElementNS("http://www.w3.org/2000/svg","path");o.setAttribute("d",c),o.setAttribute("fill","none"),o.setAttribute("stroke","black"),o.setAttribute("stroke-width",`${i} `),l.appendChild(o);let n=G(e,s,270);return l.appendChild(n),l}function G(a,t,e,s=5){let i=document.createElementNS("http://www.w3.org/2000/svg","path");return i.setAttribute("d",`M 0 0 L ${-s} ${s*1.5} L ${s} ${s*1.5} Z`),i.setAttribute("transform",`translate(${a}, ${t}) rotate(${e})`),i}function ct(a,t){a.classList.add("ig-highlight"),a.style.setProperty("--ig-highlight-color",t)}function Vt(a){a.classList.remove("ig-highlight"),a.style.setProperty("--ig-highlight-color","transparent")}var A=class{constructor(t,{func:e,pass:s=0,sampleCounts:i}){this.graph=null,this.func=e,this.passNumber=s,this.sampleCounts=i,this.keyPasses=[null,null,null,null];{let c=null;for(let[l,o]of e.passes.entries())o.mir.blocks.length>0&&(this.keyPasses[0]===null&&(this.keyPasses[0]=l),o.lir.blocks.length===0&&(this.keyPasses[1]=l)),o.lir.blocks.length>0&&(c?.lir.blocks.length===0&&(this.keyPasses[2]=l),this.keyPasses[3]=l),c=o}this.redundantPasses=[];{let c=null;for(let[l,o]of e.passes.entries()){if(c===null){c=o;continue}E(c.mir,o.mir)&&E(c.lir,o.lir)&&this.redundantPasses.push(l),c=o}}this.viewport=x("div",["ig-flex-grow-1","ig-overflow-hidden"],c=>{c.style.position="relative"}),this.sidebarLinks=e.passes.map((c,l)=>x("a",["ig-link-normal","ig-pv1","ig-ph2","ig-flex","ig-g2"],o=>{o.href="#",o.addEventListener("click",n=>{n.preventDefault(),this.switchPass(l)})},[x("div",["ig-w1","ig-tr","ig-f6","ig-text-dim"],o=>{o.style.paddingTop="0.08rem"},[`${l}`]),x("div",[this.redundantPasses.includes(l)&&"ig-text-dim"],()=>{},[c.name])])),this.container=x("div",["ig-absolute","ig-absolute-fill","ig-flex"],()=>{},[x("div",["ig-w5","ig-br","ig-flex-shrink-0","ig-overflow-y-auto","ig-bg-white"],()=>{},[...this.sidebarLinks]),this.viewport]),t.appendChild(this.container),this.keydownHandler=this.keydownHandler.bind(this),this.tweakHandler=this.tweakHandler.bind(this),window.addEventListener("keydown",this.keydownHandler),window.addEventListener("tweak",this.tweakHandler),this.update()}destroy(){this.container.remove(),window.removeEventListener("keydown",this.keydownHandler),window.removeEventListener("tweak",this.tweakHandler)}update(){for(let[s,i]of this.sidebarLinks.entries())i.classList.toggle("ig-bg-primary",this.passNumber===s);let t=this.graph?.exportState();this.viewport.innerHTML="",this.graph=null;let e=this.func.passes[this.passNumber];if(e)try{this.graph=new V(this.viewport,e,{sampleCounts:this.sampleCounts}),t&&this.graph.restoreState(t,{preserveSelectedBlockPosition:!0})}catch(s){this.viewport.innerHTML="An error occurred while laying out the graph. See console.",console.error(s)}}switchPass(t){this.passNumber=t,this.update()}keydownHandler(t){switch(t.key){case"w":case"s":this.graph?.navigate(t.key==="s"?"down":"up"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"a":case"d":this.graph?.navigate(t.key==="d"?"right":"left"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"f":for(let e=this.passNumber+1;e<this.func.passes.length;e++)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"r":for(let e=this.passNumber-1;e>=0;e--)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"1":case"2":case"3":case"4":{let e=["1","2","3","4"].indexOf(t.key),s=this.keyPasses[e];typeof s=="number"&&this.switchPass(s)}break;case"c":{let e=this.graph?.blocksByPtr.get(this.graph?.lastSelectedBlockPtr??-1);e&&this.graph?.jumpToBlock(e.ptr,{zoom:1})}break}}tweakHandler(){this.update()}};var M=new URL(window.location.toString()).searchParams,Gt=M.has("func")?parseInt(M.get("func"),10):void 0,ut=M.has("pass")?parseInt(M.get("pass"),10):void 0,Y=class{constructor(t){this.exportButton=null,this.ionjson=null,this.funcIndex=Gt??0,this.funcSelected=t.funcSelected,this.funcSelector=x("div",[],()=>{},["Function",x("input",["ig-w3"],e=>{e.type="number",e.min="1",e.addEventListener("input",()=>{this.switchFunc(parseInt(e.value,10)-1)})},[])," / ",x("span",["num-functions"])]),this.funcSelectorNone=x("div",[],()=>{},["No functions to display."]),this.funcName=x("div"),this.root=x("div",["ig-bb","ig-flex","ig-bg-white"],()=>{},[x("div",["ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center","ig-br","ig-hide-if-empty"],()=>{},[t.browse&&x("div",[],()=>{},[x("input",[],e=>{e.type="file",e.addEventListener("change",s=>{let i=s.target;i.files?.length&&this.fileSelected(i.files[0])})})]),this.funcSelector,this.funcSelectorNone]),x("div",["ig-flex-grow-1","ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center"],()=>{},[this.funcName,x("div",["ig-flex-grow-1"]),t.export&&x("div",[],()=>{},[x("button",[],e=>{this.exportButton=e,e.addEventListener("click",()=>{this.exportStandalone()})},["Export"])])])]),this.update()}async fileSelected(t){let e=JSON.parse(await t.text());this.ionjson=X(e),this.switchFunc(0),this.update()}switchIonJSON(t){this.ionjson=t,this.switchFunc(this.funcIndex)}switchFunc(t){t=Math.max(0,Math.min(this.numFunctions()-1,t)),this.funcIndex=isNaN(t)?0:t,this.funcSelected(this.ionjson?.functions[this.funcIndex]??null),this.update()}numFunctions(){return this.ionjson?.functions.length??0}update(){let t=0<=this.funcIndex&&this.funcIndex<this.numFunctions();this.funcSelector.hidden=this.numFunctions()<=1,this.funcSelectorNone.hidden=!(this.ionjson&&this.numFunctions()===0);let e=this.funcSelector.querySelector("input");e.max=`${this.numFunctions()}`,e.value=`${this.funcIndex+1}`,this.funcSelector.querySelector(".num-functions").innerHTML=`${this.numFunctions()}`,this.funcName.hidden=!t,this.funcName.innerText=`${this.ionjson?.functions[this.funcIndex].name??""}`,this.exportButton&&(this.exportButton.disabled=!this.ionjson||!t)}async exportStandalone(){let t=L(this.ionjson),e=t.functions[this.funcIndex].name,s={version:1,functions:[t.functions[this.funcIndex]]},c=(await(await fetch("./standalone.html")).text()).replace(/\{\{\s*IONJSON\s*\}\}/,JSON.stringify(s)),l=URL.createObjectURL(new Blob([c],{type:"text/html;charset=utf-8"})),o=document.createElement("a");o.href=l,o.download=`iongraph-${e}.html`,document.body.appendChild(o),o.click(),o.remove(),URL.revokeObjectURL(l)}},tt=class{constructor(){this.menuBar=new Y({browse:!0,export:!0,funcSelected:t=>this.switchFunc(t)}),this.func=null,this.sampleCountsFromFile=void 0,this.graph=null,this.loadStuffFromQueryParams(),this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer]),this.update()}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut,sampleCounts:this.sampleCountsFromFile}))}loadStuffFromQueryParams(){(async()=>{let t=M.get("file");if(t){let s=await(await fetch(t)).json(),i=X(s);this.menuBar.switchIonJSON(i)}})(),(async()=>{let t=M.get("sampleCounts");if(t){let s=await(await fetch(t)).json();this.sampleCountsFromFile={selfLineHits:new Map(s.selfLineHits),totalLineHits:new Map(s.totalLineHits)},this.update()}})()}switchFunc(t){this.func=t,this.update()}},et=class{constructor(){this.menuBar=new Y({funcSelected:t=>this.switchFunc(t)}),this.func=null,this.graph=null,this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer])}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut}))}setIonJSON(t){this.menuBar.switchIonJSON(t)}switchFunc(t){this.func=t,this.update()}};return yt(Yt);})(); +</script> + <script>window.__exportedIonJSON = {{ IONJSON }}</script> + <script> + const ui = new iongraph.StandaloneUI(); + document.body.appendChild(ui.root); + ui.setIonJSON(window.__exportedIonJSON); + </script> +</body> diff --git a/tool/zjit_iongraph.rb b/tool/zjit_iongraph.rb new file mode 100755 index 0000000000..0cb7701614 --- /dev/null +++ b/tool/zjit_iongraph.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +require 'json' +require 'logger' + +LOGGER = Logger.new($stderr) + +def run_ruby *cmd + # Find the first --zjit* option and add --zjit-dump-hir-iongraph after it + zjit_index = cmd.find_index { |arg| arg.start_with?("--zjit") } + raise "No --zjit option found in command" unless zjit_index + cmd.insert(zjit_index + 1, "--zjit-dump-hir-iongraph") + pid = Process.spawn(*cmd) + _, status = Process.wait2(pid) + if status.exitstatus != 0 + LOGGER.warn("Command failed with exit status #{status.exitstatus}") + end + pid +end + +usage = "Usage: zjit_iongraph.rb <path_to_ruby> <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +pid = run_ruby(RUBY, *OPTIONS) +functions = Dir["/tmp/zjit-iongraph-#{pid}/fun*.json"].map do |path| + JSON.parse(File.read(path)) +end + +if functions.empty? + LOGGER.warn("No iongraph functions found for PID #{pid}") +end + +json = JSON.dump({version: 1, functions: functions}) +# Get zjit_iongraph.html from the sibling file next to this script +html = File.read(File.join(File.dirname(__FILE__), "zjit_iongraph.html")) +html.sub!("{{ IONJSON }}", json) +output_path = "zjit_iongraph_#{pid}.html" +File.write(output_path, html) +puts "Wrote iongraph to #{output_path}" |