diff options
Diffstat (limited to 'tool')
348 files changed, 37771 insertions, 3494 deletions
diff --git a/tool/annocheck/Dockerfile b/tool/annocheck/Dockerfile new file mode 100644 index 0000000000..d1fb1839c9 --- /dev/null +++ b/tool/annocheck/Dockerfile @@ -0,0 +1,4 @@ +FROM ghcr.io/ruby/fedora:latest + +RUN dnf -y install annobin-annocheck +WORKDIR /work diff --git a/tool/annocheck/Dockerfile-copy b/tool/annocheck/Dockerfile-copy new file mode 100644 index 0000000000..d437f27387 --- /dev/null +++ b/tool/annocheck/Dockerfile-copy @@ -0,0 +1,6 @@ +FROM ghcr.io/ruby/fedora:latest +ARG IN_DIR + +RUN dnf -y install annobin-annocheck +COPY ${IN_DIR} /work +WORKDIR /work diff --git a/tool/asm_parse.rb b/tool/asm_parse.rb index e39580c1a1..32882be3ad 100644 --- a/tool/asm_parse.rb +++ b/tool/asm_parse.rb @@ -1,3 +1,5 @@ +# YARV tool to parse assembly output. + stat = {} while line = ARGF.gets diff --git a/tool/auto-style.rb b/tool/auto-style.rb new file mode 100755 index 0000000000..3b93c8c317 --- /dev/null +++ b/tool/auto-style.rb @@ -0,0 +1,284 @@ +#!/usr/bin/env ruby +# Usage: +# auto-style.rb oldrev newrev [pushref] + +require 'shellwords' +require 'tmpdir' +ENV['LC_ALL'] = 'C' + +class Git + attr_reader :depth + + def initialize(oldrev, newrev, branch = nil) + @oldrev = oldrev + @newrev = !newrev || newrev.empty? ? 'HEAD' : newrev + @branch = branch + + return unless oldrev + + # GitHub may not fetch github.event.pull_request.base.sha at checkout + git('log', '--format=%H', '-1', @oldrev, out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=1', 'origin', @oldrev) + git('log', '--format=%H', '-1', "#@newrev~99", out: IO::NULL, err: [:child, :out]) or + git('fetch', '--depth=100', 'origin', @newrev) + + with_clean_env do + @revs = {} + IO.popen(['git', 'log', '--format=%H %s', "#{@oldrev}..#{@newrev}"]) do |f| + f.each do |line| + line.chomp! + rev, subj = line.split(' ', 2) + @revs[rev] = subj + end + end + @depth = @revs.size + end + end + + # ["foo/bar.c", "baz.h", ...] + def updated_paths + with_clean_env do + IO.popen(['git', 'diff', '--name-only', @oldrev, @newrev], &:readlines).each(&:chomp!) + end + end + + # [0, 1, 4, ...] + def updated_lines(file) # NOTE: This doesn't work well on pull requests, so not used anymore + lines = [] + revs = @revs.map {|rev, subj| rev unless subj.start_with?("Revert ")}.compact + revs_pattern = /\A(?:#{revs.join('|')}) / + with_clean_env { IO.popen(['git', 'blame', '-l', '--', file], &:readlines) }.each_with_index do |line, index| + if revs_pattern =~ line + lines << index + end + end + lines + end + + def commit(log, *files) + git('add', *files) + git('commit', '-m', log) + end + + def push + git('push', 'origin', @branch) if @branch + end + + def diff + git('--no-pager', 'diff') + end + + private + + def git(*args, **opts) + cmd = ['git', *args] + puts "+ #{cmd.shelljoin}" + ret = with_clean_env { system(*cmd, **opts) } + unless ret or opts[:err] + abort "Failed to run: #{cmd}" + end + ret + end + + def with_clean_env + git_dir = ENV.delete('GIT_DIR') # this overcomes '-C' or pwd + yield + ensure + ENV['GIT_DIR'] = git_dir if git_dir + end +end + +DEFAULT_GEM_LIBS = %w[ + bundler + cmath + csv + e2mmap + fileutils + forwardable + ipaddr + irb + logger + matrix + mutex_m + ostruct + prime + rdoc + rexml + rss + scanf + shell + sync + thwait + tracer + webrick +] + +DEFAULT_GEM_EXTS = %w[ + bigdecimal + date + dbm + digest + etc + fcntl + fiddle + gdbm + io/console + io/nonblock + json + openssl + psych + racc + sdbm + stringio + strscan + zlib +] + +IGNORED_FILES = [ + # default gems whose master is GitHub + %r{\Abin/(?!erb)\w+\z}, + *(DEFAULT_GEM_LIBS + DEFAULT_GEM_EXTS).flat_map { |lib| + [ + %r{\Alib/#{lib}/}, + %r{\Alib/#{lib}\.gemspec\z}, + %r{\Alib/#{lib}\.rb\z}, + %r{\Atest/#{lib}/}, + ] + }, + *DEFAULT_GEM_EXTS.flat_map { |ext| + [ + %r{\Aext/#{ext}/}, + %r{\Atest/#{ext}/}, + ] + }, + + # vendoring (ccan) + %r{\Accan/}, + + # vendoring (io/) + %r{\Aext/io/}, + + # vendoring (nkf) + %r{\Aext/nkf/nkf-utf8/}, + + # vendoring (onigmo) + %r{\Aenc/}, + %r{\Ainclude/ruby/onigmo\.h\z}, + %r{\Areg.+\.(c|h)\z}, + + # explicit or implicit `c-file-style: "linux"` + %r{\Aaddr2line\.c\z}, + %r{\Amissing/}, + %r{\Astrftime\.c\z}, + %r{\Avsnprintf\.c\z}, + + # to respect the original statements of licenses + %r{\ALEGAL\z}, + + # trailing spaces could be intentional in TRICK code + %r{\Asample/trick[^/]*/}, +] + +DIFFERENT_STYLE_FILES = %w[ + addr2line.c io_buffer.c prism*.c scheduler.c +] + +def adjust_styles(files) + trailing = eofnewline = expandtab = indent = false + + edited_files = files.select do |f| + src = File.binread(f) rescue next + eofnewline = eofnewline0 = true if src.sub!(/(?<!\A|\n)\z/, "\n") + + trailing0 = false + expandtab0 = false + indent0 = false + + src.gsub!(/^.*$/).with_index do |line, lineno| + trailing = trailing0 = true if line.sub!(/[ \t]+$/, '') + line + end + + if f.end_with?('.c') || f.end_with?('.h') || f == 'insns.def' + # If and only if unedited lines did not have tab indentation, prevent introducing tab indentation to the file. + expandtab_allowed = src.each_line.with_index.all? do |line, lineno| + !line.start_with?("\t") + end + + if expandtab_allowed + src.gsub!(/^.*$/).with_index do |line, lineno| + if line.start_with?("\t") # last-committed line with hard tabs + expandtab = expandtab0 = true + line.sub(/\A\t+/) { |tabs| ' ' * (8 * tabs.length) } + else + line + end + end + end + end + + if File.fnmatch?("*.[chy]", f, File::FNM_PATHNAME) && + !DIFFERENT_STYLE_FILES.any? {|pat| File.fnmatch?(pat, f, File::FNM_PATHNAME)} + indent0 = true if src.gsub!(/^\w+\([^\n]*?\)\K[ \t]*(?=\{( *\\)?$)/, '\1' "\n") + indent0 = true if src.gsub!(/^([ \t]*)\}\K[ \t]*(?=else\b.*?( *\\)?$)/, '\2' "\n" '\1') + indent0 = true if src.gsub!(/^[ \t]*\}\n\K\n+(?=[ \t]*else\b)/, '') + indent ||= indent0 + end + + if trailing0 or eofnewline0 or expandtab0 or indent0 + File.binwrite(f, src) + true + end + end + if edited_files.empty? + return + else + msg = [('remove trailing spaces' if trailing), + ('append newline at EOF' if eofnewline), + ('expand tabs' if expandtab), + ('adjust indents' if indent), + ].compact + message = "* #{msg.join(', ')}. [ci skip]" + if expandtab + message += "\nPlease consider using misc/expand_tabs.rb as a pre-commit hook." + end + return message, edited_files + end +end + +oldrev, newrev, pushref = ARGV +if (dry_run = oldrev == '-n') or oldrev == '--' + _, *updated_files = ARGV + git = Git.new(nil, nil) +else + unless dry_run = pushref.nil? + branch = IO.popen(['git', 'rev-parse', '--symbolic', '--abbrev-ref', pushref], &:read).strip + end + git = Git.new(oldrev, newrev, branch) + + updated_files = git.updated_paths +end + +files = updated_files.select {|l| + /^\d/ !~ l and /\.bat\z/ !~ l and + (/\A(?:config|[Mm]akefile|GNUmakefile|README)/ =~ File.basename(l) or + /\A\z|\.(?:[chsy]|\d+|e?rb|tmpl|bas[eh]|z?sh|in|ma?k|def|src|trans|rdoc|ja|en|el|sed|awk|p[ly]|scm|mspec|html|rs)\z/ =~ File.extname(l)) +} +files.select! {|n| File.file?(n) } +files.reject! do |f| + IGNORED_FILES.any? { |re| f.match(re) } +end + +if files.empty? + puts "No files are an auto-style target:\n#{updated_files.join("\n")}" +elsif !(message, edited_files = adjust_styles(files)) + puts "All edited lines are formatted well:\n#{files.join("\n")}" +else + if dry_run + git.diff + abort message + else + git.commit(message, *edited_files) + git.push + end +end diff --git a/tool/auto_review_pr.rb b/tool/auto_review_pr.rb new file mode 100755 index 0000000000..07c98c7e0a --- /dev/null +++ b/tool/auto_review_pr.rb @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'uri' +require_relative './sync_default_gems' + +class GitHubAPIClient + def initialize(token) + @token = token + end + + def get(path) + response = Net::HTTP.get_response(URI("https://api.github.com#{path}"), { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end + + def post(path, body = {}) + body = JSON.dump(body) + response = Net::HTTP.post(URI("https://api.github.com#{path}"), body, { + 'Authorization' => "token #{@token}", + 'Accept' => 'application/vnd.github.v3+json', + 'Content-Type' => 'application/json', + }).tap(&:value) + JSON.parse(response.body, symbolize_names: true) + end +end + +class AutoReviewPR + REPO = 'ruby/ruby' + + COMMENT_USER = 'github-actions[bot]' + COMMENT_PREFIX = 'The following files are maintained in the following upstream repositories:' + COMMENT_SUFFIX = 'Please file a pull request to the above instead. Thank you!' + + def initialize(client) + @client = client + end + + def review(pr_number) + # Fetch the list of files changed by the PR + changed_files = @client.get("/repos/#{REPO}/pulls/#{pr_number}/files").map { it.fetch(:filename) } + + # Build a Hash: { upstream_repo => files, ... } + upstream_repos = SyncDefaultGems::Repository.group(changed_files) + upstream_repos.delete(nil) # exclude no-upstream files + upstream_repos.delete('prism') if changed_files.include?('prism_compile.c') # allow prism changes in this case + if upstream_repos.empty? + puts "Skipped: The PR ##{pr_number} doesn't have upstream repositories." + return + end + + # Check if the PR is already reviewed + existing_comments = @client.get("/repos/#{REPO}/issues/#{pr_number}/comments") + existing_comments.map! { [it.fetch(:user).fetch(:login), it.fetch(:body)] } + if existing_comments.any? { |user, comment| user == COMMENT_USER && comment.start_with?(COMMENT_PREFIX) } + puts "Skipped: The PR ##{pr_number} already has an automated review comment." + return + end + + # Post a comment + comment = format_comment(upstream_repos) + result = @client.post("/repos/#{REPO}/issues/#{pr_number}/comments", { body: comment }) + puts "Success: #{JSON.pretty_generate(result)}" + end + + private + + # upstream_repos: { upstream_repo => files, ... } + def format_comment(upstream_repos) + comment = +'' + comment << "#{COMMENT_PREFIX}\n\n" + + upstream_repos.each do |upstream_repo, files| + comment << "* https://github.com/ruby/#{upstream_repo}\n" + files.each do |file| + comment << " * #{file}\n" + end + end + + comment << "\n#{COMMENT_SUFFIX}" + comment + end +end + +pr_number = ARGV[0] || abort("Usage: #{$0} <pr_number>") +client = GitHubAPIClient.new(ENV.fetch('GITHUB_TOKEN')) + +AutoReviewPR.new(client).review(pr_number) diff --git a/tool/bisect.sh b/tool/bisect.sh index d47bd988b4..dfc3a64041 100755 --- a/tool/bisect.sh +++ b/tool/bisect.sh @@ -1,7 +1,7 @@ #!/bin/sh # usage: # edit $(srcdir)/test.rb -# git bisect start `git svn find-rev <rBADREV>` `git svn find-rev <rGOODREV>` +# git bisect start <bad> <good> # cd <builddir> # make bisect (or bisect-ruby for full ruby) @@ -11,32 +11,55 @@ fi case $1 in miniruby | ruby ) # (miniruby|ruby) <srcdir> - srcdir=$2 + srcdir="$2" builddir=`pwd` # assume pwd is builddir - path=$builddir/_bisect.sh + path="$builddir/_bisect.sh" echo "path: $path" - cp $0 $path - cd $srcdir - echo "git bisect run $path run-$1" - git bisect run $path run-$1 + cp "$0" "$path" + cd "$srcdir" + set -x + exec git bisect run "$path" "run-$1" ;; run-miniruby ) - cd ${0%/*} # assume a copy of this script is in builddir - $MAKE Makefile - $MAKE mini || exit 125 - $MAKE run || exit 1 + prep=mini + run=run ;; run-ruby ) - cd ${0%/*} # assume a copy of this script is in builddir - $MAKE Makefile - $MAKE program || exit 125 - $MAKE runruby || exit 1 + prep=program + run=runruby ;; "" ) - echo foo bar + echo missing command 1>&2 + exit 1 ;; * ) - echo unknown command "'$cmd'" + echo unknown command "'$1'" 1>&2 + exit 1 ;; esac -exit 0 + +# Apply $(srcdir)/bisect.patch to build if exists +# e.g., needs 5c2508060b~2..5c2508060b to use Bison 3.5.91. +if [ -f bisect.patch ]; then + if ! patch -p1 -N < bisect.patch || git diff --no-patch --exit-code; then + exit 125 + fi + git status + exec= +else + exec=exec +fi + +case "$0" in +*/*) + # assume a copy of this script is in builddir + cd `echo "$0" | sed 's:\(.*\)/.*:\1:'` || exit 125 + ;; +esac +for target in srcs Makefile $prep; do + $MAKE $target || exit 125 +done +$exec $MAKE $run +status=$? +git checkout -f HEAD +exit $status diff --git a/tool/bundler/dev_gems.rb b/tool/bundler/dev_gems.rb new file mode 100644 index 0000000000..91ac5628f1 --- /dev/null +++ b/tool/bundler/dev_gems.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "test-unit", "~> 3.0" +gem "test-unit-ruby-core" +gem "rake", "~> 13.1" +gem "rb_sys" + +gem "turbo_tests", "~> 2.2.3" +gem "parallel_tests", "~> 4.10.1" +gem "parallel", "~> 1.19" +gem "rspec-core", "~> 3.12" +gem "rspec-expectations", "~> 3.12" +gem "rspec-mocks", "~> 3.12" +gem "rubygems-generate_index", "~> 1.1" + +group :doc do + gem "ronn-ng", "~> 0.10.1", platform: :ruby +end diff --git a/tool/bundler/dev_gems.rb.lock b/tool/bundler/dev_gems.rb.lock new file mode 100644 index 0000000000..832127fb4c --- /dev/null +++ b/tool/bundler/dev_gems.rb.lock @@ -0,0 +1,132 @@ +GEM + remote: https://rubygems.org/ + specs: + compact_index (0.15.0) + diff-lcs (1.6.2) + kramdown (2.5.1) + rexml (>= 3.3.9) + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + mini_portile2 (2.8.9) + mustache (1.1.1) + nokogiri (1.19.0) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.19.0-aarch64-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm-linux-gnu) + racc (~> 1.4) + nokogiri (1.19.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-java) + racc (~> 1.4) + nokogiri (1.19.0-x64-mingw-ucrt) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-gnu) + racc (~> 1.4) + parallel (1.27.0) + parallel_tests (4.10.1) + parallel + power_assert (3.0.1) + racc (1.8.1) + racc (1.8.1-java) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rexml (3.4.4) + ronn-ng (0.10.1) + kramdown (~> 2, >= 2.1) + kramdown-parser-gfm (~> 1, >= 1.0.1) + mustache (~> 1) + nokogiri (~> 1, >= 1.14.3) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + test-unit (3.7.7) + power_assert + test-unit-ruby-core (1.0.14) + test-unit (>= 3.7.2) + turbo_tests (2.2.5) + parallel_tests (>= 3.3.0, < 5) + rspec (>= 3.10) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm-linux + arm64-darwin + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86-linux + x86_64-darwin + x86_64-linux + +DEPENDENCIES + parallel (~> 1.19) + parallel_tests (~> 4.10.1) + rake (~> 13.1) + rb_sys + ronn-ng (~> 0.10.1) + rspec-core (~> 3.12) + rspec-expectations (~> 3.12) + rspec-mocks (~> 3.12) + rubygems-generate_index (~> 1.1) + test-unit (~> 3.0) + test-unit-ruby-core + turbo_tests (~> 2.2.3) + +CHECKSUMS + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + kramdown (2.5.1) sha256=87bbb6abd9d3cebe4fc1f33e367c392b4500e6f8fa19dd61c0972cf4afe7368c + kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 + mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289 + mustache (1.1.1) sha256=90891fdd50b53919ca334c8c1031eada1215e78d226d5795e523d6123a2717d0 + nokogiri (1.19.0) sha256=e304d21865f62518e04f2bf59f93bd3a97ca7b07e7f03952946d8e1c05f45695 + nokogiri (1.19.0-aarch64-linux-gnu) sha256=11a97ecc3c0e7e5edcf395720b10860ef493b768f6aa80c539573530bc933767 + nokogiri (1.19.0-arm-linux-gnu) sha256=572a259026b2c8b7c161fdb6469fa2d0edd2b61cd599db4bbda93289abefbfe5 + nokogiri (1.19.0-arm64-darwin) sha256=0811dfd936d5f6dd3f6d32ef790568bf29b2b7bead9ba68866847b33c9cf5810 + nokogiri (1.19.0-java) sha256=5f3a70e252be641d8a4099f7fb4cc25c81c632cb594eec9b4b8f2ca8be4374f3 + nokogiri (1.19.0-x64-mingw-ucrt) sha256=05d7ed2d95731edc9bef2811522dc396df3e476ef0d9c76793a9fca81cab056b + nokogiri (1.19.0-x86_64-darwin) sha256=1dad56220b603a8edb9750cd95798bffa2b8dd9dd9aa47f664009ee5b43e3067 + nokogiri (1.19.0-x86_64-linux-gnu) sha256=f482b95c713d60031d48c44ce14562f8d2ce31e3a9e8dd0ccb131e9e5a68b58c + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parallel_tests (4.10.1) sha256=df05458c691462b210f7a41fc2651d4e4e8a881e8190e6d1e122c92c07735d70 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + ronn-ng (0.10.1) sha256=4eeb0185c0fbfa889efed923b5b50e949cd869e7d82ac74138acd0c9c7165ec0 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + test-unit-ruby-core (1.0.14) sha256=d2e997796c9c5c5e8e31ac014f83a473ff5c2523a67cfa491b08893e12d43d22 + turbo_tests (2.2.5) sha256=3fa31497d12976d11ccc298add29107b92bda94a90d8a0a5783f06f05102509f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/rubocop_gems.rb b/tool/bundler/rubocop_gems.rb new file mode 100644 index 0000000000..a9b6fda11b --- /dev/null +++ b/tool/bundler/rubocop_gems.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rubocop", ">= 1.52.1", "< 2" + +gem "minitest", "~> 5.1" +gem "irb" +gem "rake" +gem "rake-compiler" +gem "rspec" +gem "test-unit" +gem "rb_sys" diff --git a/tool/bundler/rubocop_gems.rb.lock b/tool/bundler/rubocop_gems.rb.lock new file mode 100644 index 0000000000..d0c3120e11 --- /dev/null +++ b/tool/bundler/rubocop_gems.rb.lock @@ -0,0 +1,159 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) + ast (~> 2.4.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) + rake + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.82.1) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.48.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + stringio (3.2.0) + test-unit (3.7.7) + power_assert + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + irb + minitest (~> 5.1) + rake + rake-compiler + rb_sys + rspec + rubocop (>= 1.52.1, < 2) + test-unit + +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.82.1) sha256=09f1a6a654a960eda767aebea33e47603080f8e9c9a3f019bf9b94c9cab5e273 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/standard_gems.rb b/tool/bundler/standard_gems.rb new file mode 100644 index 0000000000..f7bc34cf5e --- /dev/null +++ b/tool/bundler/standard_gems.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "standard", "~> 1.0" + +gem "minitest", "~> 5.1" +gem "irb" +gem "rake" +gem "rake-compiler" +gem "rspec" +gem "test-unit" +gem "rb_sys" diff --git a/tool/bundler/standard_gems.rb.lock b/tool/bundler/standard_gems.rb.lock new file mode 100644 index 0000000000..f3792f8611 --- /dev/null +++ b/tool/bundler/standard_gems.rb.lock @@ -0,0 +1,179 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + date (3.5.1) + date (3.5.1-java) + diff-lcs (1.6.2) + erb (6.0.1) + erb (6.0.1-java) + io-console (0.8.2) + io-console (0.8.2-java) + irb (1.16.0) + pp (>= 0.6.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + jar-dependencies (0.5.5) + json (2.18.0) + json (2.18.0-java) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + minitest (5.27.0) + parallel (1.27.0) + parser (3.3.10.0) + ast (~> 2.4.1) + racc + power_assert (3.0.1) + pp (0.6.3) + prettyprint + prettyprint (0.2.0) + prism (1.7.0) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + racc (1.8.1) + racc (1.8.1-java) + rainbow (3.1.1) + rake (13.3.1) + rake-compiler (1.3.1) + rake + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + rdoc (7.0.3) + erb + psych (>= 4.0.0) + tsort + regexp_parser (2.11.3) + reline (0.6.3) + io-console (~> 0.5) + rspec (3.13.2) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.6) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.7) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.6) + rubocop (1.81.7) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.49.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + rubocop-performance (1.26.1) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.47.1, < 2.0) + ruby-progressbar (1.13.0) + standard (1.52.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.81.7) + standard-custom (~> 1.0.0) + standard-performance (~> 1.8) + standard-custom (1.0.2) + lint_roller (~> 1.0) + rubocop (~> 1.50) + standard-performance (1.9.0) + lint_roller (~> 1.1) + rubocop-performance (~> 1.26.0) + stringio (3.2.0) + test-unit (3.7.7) + power_assert + tsort (0.2.0) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) + +PLATFORMS + aarch64-darwin + aarch64-linux + arm64-darwin + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + irb + minitest (~> 5.1) + rake + rake-compiler + rb_sys + rspec + standard (~> 1.0) + test-unit + +CHECKSUMS + ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962 + erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5 + erb (6.0.1-java) sha256=5c6b8d885fb0220d4a8ad158f70430d805845939dd44827e5130ef7fdbaed8ba + io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc + io-console (0.8.2-java) sha256=837efefe96084c13ae91114917986ae6c6d1cf063b27b8419cc564a722a38af8 + irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806 + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505 + json (2.18.0-java) sha256=74706f684baeb1a40351ed26fc8fe6e958afa861320d1c28ff4eb7073b29c7aa + language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc + lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5 + parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parser (3.3.10.0) sha256=ce3587fa5cc55a88c4ba5b2b37621b3329aadf5728f9eafa36bbd121462aabd6 + power_assert (3.0.1) sha256=8ce9876716cc74e863fcd4cdcdc52d792bd983598d1af3447083a3a9a4d34103 + pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6 + prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 + prism (1.7.0) sha256=10062f734bf7985c8424c44fac382ac04a58124ea3d220ec3ba9fe4f2da65103 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f + racc (1.8.1-java) sha256=54f2e6d1e1b91c154013277d986f52a90e5ececbe91465d29172e49342732b98 + rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler (1.3.1) sha256=6b351612b6e2d73ddd5563ee799bb58685176e05363db6758504bd11573d670a + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + rdoc (7.0.3) sha256=dfe3d0981d19b7bba71d9dbaeb57c9f4e3a7a4103162148a559c4fc687ea81f9 + regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4 + reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835 + rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 + rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d + rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836 + rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c + rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2 + rubocop (1.81.7) sha256=6fb5cc298c731691e2a414fe0041a13eb1beed7bab23aec131da1bcc527af094 + rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd + rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834 + ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 + standard (1.52.0) sha256=ec050e63228e31fabe40da3ef96da7edda476f7acdf3e7c2ad47b6e153f6a076 + standard-custom (1.0.2) sha256=424adc84179a074f1a2a309bb9cf7cd6bfdb2b6541f20c6bf9436c0ba22a652b + standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + test-unit (3.7.7) sha256=3c89d5ff0690a16bef9946156c4624390402b9d54dfcf4ce9cbd5b06bead1e45 + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42 + unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb new file mode 100644 index 0000000000..ddc19e2939 --- /dev/null +++ b/tool/bundler/test_gems.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rack", "~> 3.1" +gem "rack-test", "~> 2.1" +gem "compact_index", "~> 0.15.0" +gem "sinatra", "~> 4.1" +gem "rake", "~> 13.1" +gem "builder", "~> 3.2" +gem "rb_sys" +gem "fiddle" +gem "rubygems-generate_index", "~> 1.1" +gem "concurrent-ruby" +gem "psych" +gem "etc", platforms: [:ruby, :windows] +gem "open3" +gem "shellwords" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock new file mode 100644 index 0000000000..fdffc1f09d --- /dev/null +++ b/tool/bundler/test_gems.rb.lock @@ -0,0 +1,106 @@ +GEM + remote: https://rubygems.org/ + specs: + base64 (0.3.0) + builder (3.3.0) + compact_index (0.15.0) + concurrent-ruby (1.3.6) + date (3.5.1) + date (3.5.1-java) + etc (1.4.6) + fiddle (1.1.8) + jar-dependencies (0.5.5) + logger (1.7.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + open3 (0.2.1) + psych (5.3.1) + date + stringio + psych (5.3.1-java) + date + jar-dependencies (>= 0.1.7) + rack (3.2.4) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.1) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rake (13.3.1) + rake-compiler-dock (1.10.0) + rb_sys (0.9.123) + rake-compiler-dock (= 1.10.0) + ruby2_keywords (0.0.5) + rubygems-generate_index (1.1.3) + compact_index (~> 0.15.0) + shellwords (0.2.2) + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) + tilt (~> 2.0) + stringio (3.2.0) + tilt (2.6.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + builder (~> 3.2) + compact_index (~> 0.15.0) + concurrent-ruby + etc + fiddle + open3 + psych + rack (~> 3.1) + rack-test (~> 2.1) + rake (~> 13.1) + rb_sys + rubygems-generate_index (~> 1.1) + shellwords + sinatra (~> 4.1) + +CHECKSUMS + base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b + builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f + compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b + concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab + date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0 + date (3.5.1-java) sha256=12e09477dc932afe45bf768cd362bf73026804e0db1e6c314186d6cd0bee3344 + etc (1.4.6) sha256=0f7e9e7842ea5e3c3bd9bc81746ebb8c65ea29e4c42a93520a0d638129c7de01 + fiddle (1.1.8) sha256=7fa8ee3627271497f3add5503acdbc3f40b32f610fc1cf49634f083ef3f32eee + jar-dependencies (0.5.5) sha256=2972b9fcba4b014e6446a84b5c09674a3e8648b95b71768e729f0e8e40568059 + logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 + mustermann (3.0.4) sha256=85fadcb6b3c6493a8b511b42426f904b7f27b282835502233dd154daab13aa22 + open3 (0.2.1) sha256=8e2d7d2113526351201438c1aa35c8139f0141c9e8913baa007c898973bf3952 + psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974 + psych (5.3.1-java) sha256=20a4a81ad01479ef060f604ed75ba42fe673169e67d923b1bae5aa4e13cc5820 + rack (3.2.4) sha256=5d74b6f75082a643f43c1e76b419c40f0e5527fcfee1e669ac1e6b73c0ccb6f6 + rack-protection (4.2.1) sha256=cf6e2842df8c55f5e4d1a4be015e603e19e9bc3a7178bae58949ccbb58558bac + rack-session (2.1.1) sha256=0b6dc07dea7e4b583f58a48e8b806d4c9f1c6c9214ebc202ec94562cbea2e4e9 + rack-test (2.2.0) sha256=005a36692c306ac0b4a9350355ee080fd09ddef1148a5f8b2ac636c720f5c463 + rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c + rake-compiler-dock (1.10.0) sha256=dd62ee19df2a185a3315697e560cfa8cc9129901332152851e023fab0e94bf11 + rb_sys (0.9.123) sha256=c22ae84d1bca3eec0f13a45ae4ca9ba6eace93d5be270a40a9c0a9a5b92a34e5 + ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef + rubygems-generate_index (1.1.3) sha256=3571424322666598e9586a906485e1543b617f87644913eaf137d986a3393f5c + shellwords (0.2.2) sha256=b8695a791de2f71472de5abdc3f4332f6535a4177f55d8f99e7e44266cd32f94 + sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 + stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1 + tilt (2.6.1) sha256=35a99bba2adf7c1e362f5b48f9b581cce4edfba98117e34696dde6d308d84770 + +BUNDLED WITH + 4.1.0.dev diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb new file mode 100644 index 0000000000..8d12c5adde --- /dev/null +++ b/tool/bundler/vendor_gems.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "fileutils", "1.8.0" +gem "molinillo", github: "cocoapods/molinillo", ref: "1d62d7d5f448e79418716dc779a4909509ccda2a" +gem "net-http", "0.7.0" # net-http-0.8.0 is broken with JRuby +gem "net-http-persistent", "4.0.6" +gem "net-protocol", "0.2.2" +gem "optparse", "0.8.0" +gem "pub_grub", github: "jhawthorn/pub_grub", ref: "df6add45d1b4d122daff2f959c9bd1ca93d14261" +gem "resolv", "0.6.2" +gem "securerandom", "0.4.1" +gem "timeout", "0.4.4" +gem "thor", "1.4.0" +gem "tsort", "0.2.0" +gem "uri", "1.1.1" diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock new file mode 100644 index 0000000000..cc7886e60b --- /dev/null +++ b/tool/bundler/vendor_gems.rb.lock @@ -0,0 +1,75 @@ +GIT + remote: https://github.com/cocoapods/molinillo.git + revision: 1d62d7d5f448e79418716dc779a4909509ccda2a + ref: 1d62d7d5f448e79418716dc779a4909509ccda2a + specs: + molinillo (0.8.0) + +GIT + remote: https://github.com/jhawthorn/pub_grub.git + revision: df6add45d1b4d122daff2f959c9bd1ca93d14261 + ref: df6add45d1b4d122daff2f959c9bd1ca93d14261 + specs: + pub_grub (0.5.0) + +GEM + remote: https://rubygems.org/ + specs: + connection_pool (2.5.4) + fileutils (1.8.0) + net-http (0.7.0) + uri + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + net-protocol (0.2.2) + timeout + optparse (0.8.0) + resolv (0.6.2) + securerandom (0.4.1) + thor (1.4.0) + timeout (0.4.4) + tsort (0.2.0) + uri (1.1.1) + +PLATFORMS + java + ruby + universal-java + x64-mingw-ucrt + x64-mswin64-140 + x86_64-darwin + x86_64-linux + +DEPENDENCIES + fileutils (= 1.8.0) + molinillo! + net-http (= 0.7.0) + net-http-persistent (= 4.0.6) + net-protocol (= 0.2.2) + optparse (= 0.8.0) + pub_grub! + resolv (= 0.6.2) + securerandom (= 0.4.1) + thor (= 1.4.0) + timeout (= 0.4.4) + tsort (= 0.2.0) + uri (= 1.1.1) + +CHECKSUMS + connection_pool (2.5.4) sha256=e9e1922327416091f3f6542f5f4446c2a20745276b9aa796dd0bb2fd0ea1e70a + fileutils (1.8.0) sha256=8c6b1df54e2540bdb2f39258f08af78853aa70bad52b4d394bbc6424593c6e02 + molinillo (0.8.0) + net-http (0.7.0) sha256=4db7d9f558f8ffd4dcf832d0aefd02320c569c7d4f857def49e585069673a425 + net-http-persistent (4.0.6) sha256=2abb3a04438edf6cb9e0e7e505969605f709eda3e3c5211beadd621a2c84dd5d + net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 + optparse (0.8.0) sha256=ef6b7fbaf7ec331474f325bc08dd5622e6e1e651007a5341330ee4b08ce734f0 + pub_grub (0.5.0) + resolv (0.6.2) sha256=61efe545cedddeb1b14f77e51f85c85ca66af5098fdbf567fadf32c34590fb14 + securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 + thor (1.4.0) sha256=8763e822ccb0f1d7bee88cde131b19a65606657b847cc7b7b4b82e772bcd8a3d + timeout (0.4.4) sha256=f0f6f970104b82427cd990680f539b6bbb8b1e55efa913a55c6492935e4e0edb + tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f + uri (1.1.1) sha256=379fa58d27ffb1387eaada68c749d1426738bd0f654d812fcc07e7568f5c57c6 + +BUNDLED WITH + 4.0.0.dev diff --git a/tool/change_maker.rb b/tool/change_maker.rb deleted file mode 100755 index 2bbc275d93..0000000000 --- a/tool/change_maker.rb +++ /dev/null @@ -1,34 +0,0 @@ -#! ./miniruby - -def diff2index(cmd, *argv) - lines = [] - path = nil - output = `#{cmd} #{argv.join(" ")}` - if defined? Encoding::BINARY - output.force_encoding Encoding::BINARY - end - output.each_line do |line| - case line - when /^Index: (\S*)/, /^diff --git [a-z]\/(\S*) [a-z]\/\1/ - path = $1 - when /^@@\s*-[,\d]+ +\+(\d+)[,\d]*\s*@@(?: +([A-Za-z_][A-Za-z_0-9 ]*[A-Za-z_0-9]))?/ - line = $1.to_i - ent = "\t* #{path}" - ent << " (#{$2})" if $2 - lines << "#{ent}:" - end - end - lines.uniq! - lines.empty? ? nil : lines -end - -if `svnversion` =~ /^\d+/ - cmd = "svn diff --diff-cmd=diff -x-pU0" - change = diff2index(cmd, ARGV) -elsif File.directory?(".git") - cmd = "git diff -U0" - change = diff2index(cmd, ARGV) || diff2index(cmd, "--cached", ARGV) -else - abort "does not seem to be under a vcs" -end -puts change if change diff --git a/tool/checksum.rb b/tool/checksum.rb index 0de54a314d..8f2d1d97d0 100755 --- a/tool/checksum.rb +++ b/tool/checksum.rb @@ -1,6 +1,6 @@ #!ruby -require_relative 'vpath' +require_relative 'lib/vpath' class Checksum def initialize(vpath) @@ -36,9 +36,7 @@ class Checksum end def update! - open(@checksum, "wb") {|f| - f.puts("src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") - } + File.binwrite(@checksum, "src=\"#{@source}\", len=#{@len}, checksum=#{@sum}") end def update @@ -67,6 +65,6 @@ class Checksum def self.update(argv) k = new(VPath.new) k.source, k.target, *argv = k.def_options.parse(*argv) - k.update {|k| yield(k, *argv)} + k.update {|_| yield(_, *argv)} end end diff --git a/tool/colors b/tool/colors new file mode 100644 index 0000000000..a65c326ade --- /dev/null +++ b/tool/colors @@ -0,0 +1,3 @@ +pass=36;7 +fail=31;1;7 +skip=33;1 diff --git a/tool/commit-email.rb b/tool/commit-email.rb new file mode 100755 index 0000000000..c887f8783e --- /dev/null +++ b/tool/commit-email.rb @@ -0,0 +1,372 @@ +#!/usr/bin/env ruby + +require "optparse" +require "nkf" +require "shellwords" + +CommitEmailInfo = Struct.new( + :author, + :author_email, + :revision, + :entire_sha256, + :date, + :log, + :branch, + :diffs, + :added_files, :deleted_files, :updated_files, + :added_dirs, :deleted_dirs, :updated_dirs, +) + +class GitInfoBuilder + GitCommandFailure = Class.new(RuntimeError) + + def initialize(repo_path) + @repo_path = repo_path + end + + def build(oldrev, newrev, refname) + diffs = build_diffs(oldrev, newrev) + + info = CommitEmailInfo.new + info.author = git_show(newrev, format: '%an') + info.author_email = normalize_email(git_show(newrev, format: '%aE')) + info.revision = newrev[0...10] + info.entire_sha256 = newrev + info.date = Time.at(Integer(git_show(newrev, format: '%at'))) + info.log = git_show(newrev, format: '%B') + info.branch = git('rev-parse', '--symbolic', '--abbrev-ref', refname).strip + info.diffs = diffs + info.added_files = find_files(diffs, status: :added) + info.deleted_files = find_files(diffs, status: :deleted) + info.updated_files = find_files(diffs, status: :modified) + info.added_dirs = [] # git does not deal with directory + info.deleted_dirs = [] # git does not deal with directory + info.updated_dirs = [] # git does not deal with directory + info + end + + private + + # Force git-svn email address to @ruby-lang.org to avoid email bounce by invalid email address. + def normalize_email(email) + if email.match(/\A[^@]+@\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/) # git-svn + svn_user, _ = email.split('@', 2) + "#{svn_user}@ruby-lang.org" + else + email + end + end + + def find_files(diffs, status:) + files = [] + diffs.each do |path, values| + if values.keys.first == status + files << path + end + end + files + end + + # SVN version: + # { + # "filename" => { + # "[modified|added|deleted|copied|property_changed]" => { + # type: "[modified|added|deleted|copied|property_changed]", + # body: "diff body", # not implemented because not used + # added: Integer, + # deleted: Integer, + # } + # } + # } + def build_diffs(oldrev, newrev) + diffs = {} + + numstats = git('diff', '--numstat', oldrev, newrev).lines.map { |l| l.strip.split("\t", 3) } + git('diff', '--name-status', oldrev, newrev).each_line do |line| + status, path, _newpath = line.strip.split("\t", 3) + diff = build_diff(path, numstats) + + case status + when 'A' + diffs[path] = { added: { type: :added, **diff } } + when 'M' + diffs[path] = { modified: { type: :modified, **diff } } + when 'C' + diffs[path] = { copied: { type: :copied, **diff } } + when 'D' + diffs[path] = { deleted: { type: :deleted, **diff } } + when /\AR/ # R100 (which does not exist in git.ruby-lang.org's git 2.1.4) + # TODO: implement something + else + $stderr.puts "unexpected git diff status: #{status}" + end + end + + diffs + end + + def build_diff(path, numstats) + diff = { added: 0, deleted: 0 } # :body not implemented because not used + line = numstats.find { |(_added, _deleted, file, *)| file == path } + return diff if line.nil? + + added, deleted, _ = line + if added + diff[:added] = Integer(added) + end + if deleted + diff[:deleted] = Integer(deleted) + end + diff + end + + def git_show(revision, format:) + git('show', '--no-show-signature', "--pretty=#{format}", '--no-patch', revision).strip + end + + def git(*args) + command = ['git', '-C', @repo_path, *args] + output = with_gitenv { IO.popen(command, external_encoding: 'UTF-8', &:read) } + unless $?.success? + raise GitCommandFailure, "failed to execute '#{command.join(' ')}':\n#{output}" + end + output + end + + def with_gitenv + orig = ENV.to_h.dup + begin + ENV.delete('GIT_DIR') + yield + ensure + ENV.replace(orig) + end + end +end + +CommitEmailOptions = Struct.new(:error_to, :viewer_uri) + +CommitEmail = Module.new +class << CommitEmail + SENDMAIL = ENV.fetch('SENDMAIL', '/usr/sbin/sendmail') + private_constant :SENDMAIL + + def parse(args) + options = CommitEmailOptions.new + + opts = OptionParser.new do |opts| + opts.separator('') + + opts.on('-e', '--error-to [TO]', + 'Add [TO] to to address when error is occurred') do |to| + options.error_to = to + end + + opts.on('--viewer-uri [URI]', + 'Use [URI] as URI of revision viewer') do |uri| + options.viewer_uri = uri + end + + opts.on_tail('--help', 'Show this message') do + puts opts + exit + end + end + + return opts.parse(args), options + end + + def main(repo_path, to, rest) + args, options = parse(rest) + + infos = args.each_slice(3).flat_map do |oldrev, newrev, refname| + revisions = IO.popen(['git', 'log', '--no-show-signature', '--reverse', '--pretty=%H', "#{oldrev}^..#{newrev}"], &:read).lines.map(&:strip) + revisions[0..-2].zip(revisions[1..-1]).map do |old, new| + GitInfoBuilder.new(repo_path).build(old, new, refname) + end + end + + infos.each do |info| + next if info.branch.start_with?('notes/') + puts "#{info.branch}: #{info.revision} (#{info.author})" + + from = make_from(name: info.author, email: "noreply@ruby-lang.org") + sendmail(to, from, make_mail(to, from, info, viewer_uri: options.viewer_uri)) + end + end + + def sendmail(to, from, mail) + IO.popen([*SENDMAIL.shellsplit, to], 'w') do |f| + f.print(mail) + end + unless $?.success? + raise "Failed to run `#{SENDMAIL} #{to}` with: '#{mail}'" + end + end + + private + + def b_encode(str) + NKF.nkf('-WwM', str) + end + + def make_body(info, viewer_uri:) + body = +'' + body << "#{info.author}\t#{format_time(info.date)}\n" + body << "\n" + body << " New Revision: #{info.revision}\n" + body << "\n" + body << " #{viewer_uri}#{info.revision}\n" + body << "\n" + body << " Log:\n" + body << info.log.lstrip.gsub(/^\t*/, ' ').rstrip + body << "\n\n" + body << added_dirs(info) + body << added_files(info) + body << deleted_dirs(info) + body << deleted_files(info) + body << modified_dirs(info) + body << modified_files(info) + [body.rstrip].pack('M') + end + + def format_time(time) + time.strftime('%Y-%m-%d %X %z (%a, %d %b %Y)') + end + + def changed_items(title, type, items) + rv = '' + unless items.empty? + rv << " #{title} #{type}:\n" + rv << items.collect {|item| " #{item}\n"}.join('') + end + rv + end + + def changed_files(title, files) + changed_items(title, 'files', files) + end + + def added_files(info) + changed_files('Added', info.added_files) + end + + def deleted_files(info) + changed_files('Removed', info.deleted_files) + end + + def modified_files(info) + changed_files('Modified', info.updated_files) + end + + def changed_dirs(title, files) + changed_items(title, 'directories', files) + end + + def added_dirs(info) + changed_dirs('Added', info.added_dirs) + end + + def deleted_dirs(info) + changed_dirs('Removed', info.deleted_dirs) + end + + def modified_dirs(info) + changed_dirs('Modified', info.updated_dirs) + end + + def changed_dirs_info(info, uri) + (info.added_dirs.collect do |dir| + " Added: #{dir}\n" + end + info.deleted_dirs.collect do |dir| + " Deleted: #{dir}\n" + end + info.updated_dirs.collect do |dir| + " Modified: #{dir}\n" + end).join("\n") + end + + def diff_info(info, uri) + info.diffs.collect do |key, values| + [ + key, + values.collect do |type, value| + case type + when :added + rev = "?revision=#{info.revision}&view=markup" + when :modified, :property_changed + prev_revision = (info.revision.is_a?(Integer) ? info.revision - 1 : "#{info.revision}^") + rev = "?r1=#{info.revision}&r2=#{prev_revision}&diff_format=u" + when :deleted, :copied + rev = '' + else + raise "unknown diff type: #{value[:type]}" + end + + link = [uri, key.sub(/ .+/, '') || ''].join('/') + rev + + desc = '' + + [desc, link] + end + ] + end + end + + def make_header(to, from, info) + <<~EOS + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: #{from} + To: #{to} + Subject: #{make_subject(info)} + EOS + end + + def make_subject(info) + subject = +'' + subject << "#{info.revision}" + subject << " (#{info.branch})" + subject << ': ' + subject << info.log.lstrip.lines.first.to_s.strip + b_encode(subject) + end + + # https://tools.ietf.org/html/rfc822#section-4.1 + # https://tools.ietf.org/html/rfc822#section-6.1 + # https://tools.ietf.org/html/rfc822#appendix-D + # https://tools.ietf.org/html/rfc2047 + def make_from(name:, email:) + if name.ascii_only? + escaped_name = name.gsub(/["\\\n]/) { |c| "\\#{c}" } + %Q["#{escaped_name}" <#{email}>] + else + escaped_name = "=?UTF-8?B?#{NKF.nkf('-WwMB', name)}?=" + %Q[#{escaped_name} <#{email}>] + end + end + + def make_mail(to, from, info, viewer_uri:) + make_header(to, from, info) + make_body(info, viewer_uri: viewer_uri) + end +end + +repo_path, to, *rest = ARGV +begin + CommitEmail.main(repo_path, to, rest) +rescue StandardError => e + $stderr.puts "#{e.class}: #{e.message}" + $stderr.puts e.backtrace + + _, options = CommitEmail.parse(rest) + to = options.error_to + CommitEmail.sendmail(to, to, <<-MAIL) +From: #{to} +To: #{to} +Subject: Error + +#{$!.class}: #{$!.message} +#{$@.join("\n")} +MAIL + exit 1 +end diff --git a/tool/darwin-ar b/tool/darwin-ar new file mode 100755 index 0000000000..8b25425cfe --- /dev/null +++ b/tool/darwin-ar @@ -0,0 +1,6 @@ +#!/bin/bash +export LANG=C LC_ALL=C # Suppress localication +exec 2> >(exec grep -v \ + -e ' no symbols$' \ + >&2) +exec "$@" diff --git a/tool/darwin-cc b/tool/darwin-cc new file mode 100755 index 0000000000..42637022a4 --- /dev/null +++ b/tool/darwin-cc @@ -0,0 +1,9 @@ +#!/bin/bash +exec 2> >(exec grep -v \ + -e '^ld: warning: The [a-z0-9_][a-z0-9_]* architecture is deprecated for macOS' \ + -e '^ld: warning: text-based stub file /System/Library/Frameworks/' \ + -e '^ld: warning: ignoring duplicate libraries:' \ + -e "warning: '\.debug_macinfo' is not currently supported:" \ + -e "note: while processing" \ + >&2) +exec "$@" diff --git a/tool/disable_ipv6.sh b/tool/disable_ipv6.sh new file mode 100755 index 0000000000..ce1cc0da68 --- /dev/null +++ b/tool/disable_ipv6.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex +sysctl -w net.ipv6.conf.all.disable_ipv6=1 +sysctl -w net.ipv6.conf.default.disable_ipv6=1 +sysctl -w net.ipv6.conf.lo.disable_ipv6=1 + +cat /etc/hosts +ruby -e "hosts = File.read('/etc/hosts').sub(/^::1\s*localhost.*$/, ''); File.write('/etc/hosts', hosts)" +cat /etc/hosts diff --git a/tool/downloader.rb b/tool/downloader.rb index 1bad317ae5..39ebf44a83 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -1,76 +1,102 @@ +# Used by configure and make to download or update mirrored Ruby and GCC +# files. + +# -*- frozen-string-literal: true -*- + +require 'fileutils' require 'open-uri' -begin - require 'net/https' - $rubygems_schema = 'https' - - # open-uri of ruby 2.2.0 accept an array of PEMs as ssl_ca_cert, but old - # versions are not. so, patching OpenSSL::X509::Store#add_file instead. - class OpenSSL::X509::Store - alias orig_add_file add_file - def add_file(pems) - Array(pems).each do |pem| - if File.directory?(pem) - add_path pem - else - orig_add_file pem - end - end +require 'pathname' +verbose, $VERBOSE = $VERBOSE, nil +require 'net/https' +$VERBOSE = verbose + +class Downloader + def self.find(dlname) + constants.find do |name| + return const_get(name) if dlname.casecmp(name.to_s) == 0 end end - # since open-uri internally checks ssl_ca_cert by File.directory?, to allow - # accept an array. - class <<File - alias orig_directory? directory? - def File.directory? files - files.is_a?(Array) ? false : orig_directory?(files) - end + + def self.get_option(argv, options) + false end -rescue LoadError - $rubygems_schema = 'http' -end -class Downloader class GNU < self - def self.download(name, *rest) - super("http://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=#{name};hb=master", name, *rest) + Mirrors = %w[ + https://raw.githubusercontent.com/autotools-mirror/autoconf/refs/heads/master/build-aux/ + https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master + ] + + def self.download(name, *rest, **options) + Mirrors.each_with_index do |url, i| + super("#{url}/#{name}", name, *rest, **options) + rescue => e + raise if i + 1 == Mirrors.size # no more URLs + m1, m2 = e.message.split("\n", 2) + STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}" + else + return + end end end class RubyGems < self - def self.download(name, dir = nil, ims = true, options = {}) + def self.download(name, dir = nil, since = true, **options) require 'rubygems' - require 'rubygems/package' - options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/*.pem", File.dirname(__FILE__))) - if $rubygems_schema != 'https' - warn "*** using http instead of https ***" - end - file = under(dir, name) - super("#{$rubygems_schema}://rubygems.org/downloads/#{name}", file, nil, ims, options) or - return false - policy = Gem::Security::LowSecurity - (policy = policy.dup).ui = Gem::SilentUI.new if policy.respond_to?(:'ui=') - pkg = Gem::Package.new(file) - pkg.security_policy = policy - begin - pkg.verify - rescue Gem::Security::Exception => e - $stderr.puts e.message - File.unlink(file) - false - else - true + options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true end - end - - def self.verify(pkg) + super("https://rubygems.org/downloads/#{name}", name, dir, since, **options) end end Gems = RubyGems class Unicode < self - def self.download(name, *rest) - super("http://www.unicode.org/Public/#{name}", name, *rest) + INDEX = {} # cache index file information across files in the same directory + UNICODE_PUBLIC = "https://www.unicode.org/Public/" + + def self.get_option(argv, options) + case argv[0] + when '--unicode-beta' + options[:unicode_beta] = argv[1] + argv.shift(2) + true + when /\A--unicode-beta=(.*)/m + options[:unicode_beta] = $1 + argv.shift + true + else + super + end + end + + def self.download(name, dir = nil, since = true, unicode_beta: nil, **options) + name_dir_part = name.sub(/[^\/]+$/, '') + if unicode_beta == 'YES' + if INDEX.size == 0 + cache_save = false # TODO: make sure caching really doesn't work for index file + index_data = File.read(under(dir, "index.html")) rescue nil + index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, cache_save: cache_save, **options) + INDEX[:index] = File.read(index_file) + since = true unless INDEX[:index] == index_data + end + file_base = File.basename(name, '.txt') + return if file_base == '.' # Use pre-generated headers and tables + beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] + # make sure we always check for new versions of files, + # because they can easily change in the beta period + super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, **options) + else + index_file = Pathname.new(under(dir, name_dir_part+'index.html')) + if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ + raise "Although Unicode is not in beta, file #{index_file} exists. " + + "Remove all files in this directory and in .downloaded-cache/ " + + "because they may be leftovers from the beta period." + end + super(UNICODE_PUBLIC+name, name, dir, since, **options) + end end end @@ -91,113 +117,322 @@ class Downloader options['If-Modified-Since'] = since end end - options['Accept-Encoding'] = '*' # to disable Net::HTTP::GenericRequest#decode_content + options['Accept-Encoding'] = 'identity' # to disable Net::HTTP::GenericRequest#decode_content options end - # Downloader.download(url, name, [dir, [ims]]) + def self.httpdate(date) + Time.httpdate(date) + rescue ArgumentError => e + # Some hosts (e.g., zlib.net) return similar to RFC 850 but 4 + # digit year, sometimes. + /\A\s* + (?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day,\x20 + (\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4})\x20 + (\d\d):(\d\d):(\d\d)\x20 + GMT + \s*\z/ix =~ date or raise + warn e.message + Time.utc($3, $2, $1, $4, $5, $6) + end + + # Downloader.download(url, name, [dir, [since]]) # # Update a file from url if newer version is available. # Creates the file if the file doesn't yet exist; however, the # directory where the file is being created has to exist already. - # If +ims+ is false, always download url regardless of its last - # modified time. + # The +since+ parameter can take the following values, with associated meanings: + # true :: + # Take the last-modified time of the current file on disk, and only download + # if the server has a file that was modified later. Download unconditionally + # if we don't have the file yet. Default. + # +some time value+ :: + # Use this time value instead of the time of modification of the file on disk. + # nil :: + # Only download the file if it doesn't exist yet. + # false :: + # always download url regardless of whether we already have a file, + # and regardless of modification times. (This is essentially just a waste of + # network resources, except in the case that the file we have is somehow damaged. + # Please note that using this recurringly might create or be seen as a + # denial of service attack.) # # Example usage: # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', # 'UnicodeData.txt', 'enc/unicode/data' - def self.download(url, name, dir = nil, ims = true, options = {}) - file = under(dir, name) - if ims.nil? and File.exist?(file) - if $VERBOSE - $stdout.puts "#{name} already exists" + def self.download(url, name, dir = nil, since = true, + cache_save: ENV["CACHE_SAVE"] != "no", cache_dir: nil, + ignore_http_client_errors: nil, + dryrun: nil, verbose: false, **options) + url = URI(url) + if name + file = Pathname.new(under(dir, name)) + else + name = File.basename(url.path) + end + cache = cache_file(url, name, cache_dir) + file ||= cache + if since.nil? and file.exist? + if verbose + $stdout.puts "#{file} already exists" $stdout.flush end - return true + return file.to_path end - url = URI(url) - if $VERBOSE + if dryrun + puts "Download #{url} into #{file}" + return + end + if link_cache(cache, file, name, verbose: verbose) + return file.to_path + end + if verbose $stdout.print "downloading #{name} ... " $stdout.flush end + mtime = nil + options = options.merge(http_options(file, since.nil? ? true : since)) begin - data = url.read(options.merge(http_options(file, ims.nil? ? true : ims))) + data = with_retry(10) {url.read(options)} rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified - if $VERBOSE - $stdout.puts "not modified" + case http_error.message + when /^304 / # 304 Not Modified + if verbose + $stdout.puts "#{name} not modified" $stdout.flush end - return true + return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error - if ims.nil? and File.exist?(file) + if since.nil? and file.exist? puts "Request for #{url} timed out, using old version." - return true + return file.to_path end raise rescue SocketError - if ims.nil? and File.exist?(file) + if since.nil? and file.exist? puts "No network connection, unable to download #{url}, using old version." - return true + return file.to_path end raise + else + if mtime = data.meta["last-modified"] + mtime = Time.httpdate(mtime) + end end - mtime = nil - open(file, "wb", 0600) do |f| + dest = (cache_save && cache && !cache.exist? ? cache : file) + dest.parent.mkpath + dest.unlink if dest.symlink? && !dest.exist? + dest.open("wb", 0600) do |f| f.write(data) f.chmod(mode_for(data)) - mtime = data.meta["last-modified"] end if mtime - mtime = Time.httpdate(mtime) - File.utime(mtime, mtime, file) + dest.utime(mtime, mtime) end - if $VERBOSE + if verbose $stdout.puts "done" $stdout.flush end - true + if dest.eql?(cache) + link_cache(cache, file, name) + elsif cache_save + save_cache(cache, file, name) + end + return file.to_path rescue => e - raise "failed to download #{name}\n#{e.message}: #{url}" + raise "failed to download #{name}\n#{e.class}: #{e.message}: #{url}" end def self.under(dir, name) dir ? File.join(dir, File.basename(name)) : name end + + def self.default_cache_dir + if cache_dir = ENV['CACHE_DIR'] + return cache_dir unless cache_dir.empty? + end + ".downloaded-cache" + end + + def self.cache_file(url, name, cache_dir = nil) + case cache_dir + when false + return nil + when nil + cache_dir = default_cache_dir + end + Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) + end + + def self.link_cache(cache, file, name, verbose: false) + return false unless cache and cache.exist? + return true if cache.eql?(file) + if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] + begin + link = cache.relative_path_from(file.parent) + rescue ArgumentError + abs = cache.expand_path + link = abs.relative_path_from(file.parent.expand_path) + if link.to_s.count("/") > abs.to_s.count("/") + link = abs + end + end + begin + file.make_symlink(link) + rescue SystemCallError + else + if verbose + $stdout.puts "made symlink #{name} to #{cache}" + $stdout.flush + end + return true + end + end + begin + file.make_link(cache) + rescue SystemCallError + else + if verbose + $stdout.puts "made link #{name} to #{cache}" + $stdout.flush + end + return true + end + end + + def self.save_cache(cache, file, name) + return unless cache or cache.eql?(file) + begin + st = cache.stat + rescue + begin + file.rename(cache) + rescue + return + end + else + return unless st.mtime > file.lstat.mtime + file.unlink + end + link_cache(cache, file, name) + end + + def self.with_retry(max_times, &block) + times = 0 + begin + block.call + rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e + raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error + times += 1 + if times <= max_times + $stderr.puts "retrying #{e.class} (#{e.message}) after #{times ** 2} seconds..." + sleep(times ** 2) + retry + else + raise + end + end + end + private_class_method :with_retry end if $0 == __FILE__ - ims = true + since = true + options = {} + dl = nil + (args = []).singleton_class.__send__(:define_method, :downloader?) do |arg| + !dl and args.empty? and (dl = Downloader.find(arg)) + end until ARGV.empty? + if ARGV[0] == '--' + ARGV.shift + break if ARGV.empty? + ARGV.shift if args.downloader? ARGV[0] + args.concat(ARGV) + break + end + + if dl and dl.get_option(ARGV, options) + # the downloader dealt with the arguments, and should be removed + # from ARGV. + next + end + case ARGV[0] - when '-d' + when '-d', '--destdir' + ## -d, --destdir DIRECTORY Download into the directory destdir = ARGV[1] ARGV.shift - when '-e' - ims = nil - when '-a' - ims = true + when '-p', '--prefix' + ## -p, --prefix Strip directory names from the name to download, + ## and add the prefix instead. + prefix = ARGV[1] + ARGV.shift + when '-e', '--exist', '--non-existent-only' + ## -e, --exist, --non-existent-only Skip already existent files. + since = nil + when '-a', '--always' + ## -a, --always Download all files. + since = false + when '-u', '--update', '--if-modified' + ## -u, --update, --if-modified Download newer files only. + since = true + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Do not download actually. + options[:dryrun] = true + when '--cache-dir' + ## --cache-dir DIRECTORY Cache downloaded files in the directory. + options[:cache_dir] = ARGV[1] + ARGV.shift + when /\A--cache-dir=(.*)/m + options[:cache_dir] = $1 + when /\A--help\z/ + ## --help Print this message + puts "Usage: #$0 [options] relative-url..." + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit when /\A-/ abort "#{$0}: unknown option #{ARGV[0]}" else - break + args << ARGV[0] unless args.downloader? ARGV[0] end ARGV.shift end - dl = Downloader.constants.find do |name| - ARGV[0].casecmp(name.to_s) == 0 - end unless ARGV.empty? - $VERBOSE = true + options[:verbose] = true if dl - dl = Downloader.const_get(dl) - ARGV.shift - ARGV.each do |name| - dl.download(name, destdir, ims) + args.each do |name| + dir = destdir + if prefix + name = name.sub(/\A\.\//, '') + destdir2 = destdir.sub(/\A\.\//, '') + if name.start_with?(destdir2+"/") + name = name[(destdir2.size+1)..-1] + if (dir = File.dirname(name)) == '.' + dir = destdir + else + dir = File.join(destdir, dir) + end + else + name = File.basename(name) + end + name = "#{prefix}/#{name}" + end + dl.download(name, dir, since, **options) end else - abort "usage: #{$0} url name" unless ARGV.size == 2 - Downloader.download(ARGV[0], ARGV[1], destdir, ims) + abort "usage: #{$0} url name" unless args.size == 2 + Downloader.download(args[0], args[1], destdir, since, **options) end end diff --git a/tool/enc-case-folding.rb b/tool/enc-case-folding.rb new file mode 100755 index 0000000000..82fec7b625 --- /dev/null +++ b/tool/enc-case-folding.rb @@ -0,0 +1,416 @@ +#!/usr/bin/ruby +require 'stringio' + +# Usage (for case folding only): +# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +# $ ruby enc-case-folding.rb CaseFolding.txt -o casefold.h +# or (for case folding and case mapping): +# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +# $ wget http://www.unicode.org/Public/UNIDATA/UnicodeData.txt +# $ wget http://www.unicode.org/Public/UNIDATA/SpecialCasing.txt +# $ ruby enc-case-folding.rb -m . -o casefold.h +# using -d or --debug will include UTF-8 characters in comments for debugging + +class CaseFolding + module Util + module_function + + def hex_seq(v) + v.map { |i| "0x%04x" % i }.join(", ") + end + + def print_table_1(dest, type, mapping_data, data) + for k, v in data = data.sort + sk = (Array === k and k.length > 1) ? "{#{hex_seq(k)}}" : ("0x%04x" % k) + if type=='CaseUnfold_11' and v.length>1 + # reorder CaseUnfold_11 entries to avoid special treatment for U+03B9/U+03BC/U+A64B + item = mapping_data.map("%04X" % k[0]) + upper = item.upper if item + v = v.sort_by { |i| ("%04X"%i) == upper ? 0 : 1 } + end + ck = @debug ? ' /* ' + Array(k).pack("U*") + ' */' : '' + cv = @debug ? ' /* ' + Array(v).map{|c|[c].pack("U*")}.join(", ") + ' */' : '' + dest.print(" {#{sk}#{ck}, {#{v.length}#{mapping_data.flags(k, type, v)}, {#{hex_seq(v)}#{cv}}}},\n") + end + data + end + + def print_table(dest, type, mapping_data, data) + dest.print("static const #{type}_Type #{type}_Table[] = {\n") + i = 0 + ret = data.inject([]) do |a, (n, d)| + dest.print("#define #{n} (*(#{type}_Type (*)[#{d.size}])(#{type}_Table+#{i}))\n") + i += d.size + a.concat(print_table_1(dest, type, mapping_data, d)) + end + dest.print("};\n\n") + ret + end + end + + include Util + + attr_reader :fold, :fold_locale, :unfold, :unfold_locale, :version + + def load(filename) + pattern = /([0-9A-F]{4,6}); ([CFT]); ([0-9A-F]{4,6})(?: ([0-9A-F]{4,6}))?(?: ([0-9A-F]{4,6}))?;/ + + @fold = fold = {} + @unfold = unfold = [{}, {}, {}] + @debug = false + @version = nil + turkic = [] + + File.foreach(filename, mode: "rb") do |line| + @version ||= line[/-([0-9.]+).txt/, 1] + next unless res = pattern.match(line) + ch_from = res[1].to_i(16) + + if res[2] == 'T' + # Turkic case folding + turkic << ch_from + next + end + + # store folding data + ch_to = res[3..6].inject([]) do |a, i| + break a unless i + a << i.to_i(16) + end + fold[ch_from] = ch_to + + # store unfolding data + i = ch_to.length - 1 + (unfold[i][ch_to] ||= []) << ch_from + end + + # move locale dependent data to (un)fold_locale + @fold_locale = fold_locale = {} + @unfold_locale = unfold_locale = [{}, {}] + for ch_from in turkic + key = fold[ch_from] + i = key.length - 1 + unfold_locale[i][i == 0 ? key[0] : key] = unfold[i].delete(key) + fold_locale[ch_from] = fold.delete(ch_from) + end + self + end + + def range_check(code) + "#{code} <= MAX_CODE_VALUE && #{code} >= MIN_CODE_VALUE" + end + + def lookup_hash(key, type, data) + hash = "onigenc_unicode_#{key}_hash" + lookup = "onigenc_unicode_#{key}_lookup" + arity = Array(data[0][0]).size + gperf = %W"gperf -7 -k#{[*1..(arity*3)].join(',')} -F,-1 -c -j1 -i1 -t -T -E -C -H #{hash} -N #{lookup} -n" + argname = arity > 1 ? "codes" : "code" + argdecl = "const OnigCodePoint #{arity > 1 ? "*": ""}#{argname}" + n = 7 + m = (1 << n) - 1 + min, max = data.map {|c, *|c}.flatten.minmax + src = IO.popen(gperf, "r+") {|f| + f << "short\n%%\n" + data.each_with_index {|(k, _), i| + k = Array(k) + ks = k.map {|j| [(j >> n*2) & m, (j >> n) & m, (j) & m]}.flatten.map {|c| "\\x%.2x" % c}.join("") + f.printf "\"%s\", ::::/*%s*/ %d\n", ks, k.map {|c| "0x%.4x" % c}.join(","), i + } + f << "%%\n" + f.close_write + f.read + } + src.sub!(/^(#{hash})\s*\(.*?\).*?\n\{\n(.*)^\}/m) { + name = $1 + body = $2 + body.gsub!(/\(unsigned char\)str\[(\d+)\]/, "bits_#{arity > 1 ? 'at' : 'of'}(#{argname}, \\1)") + "#{name}(#{argdecl})\n{\n#{body}}" + } + src.sub!(/const short *\*\n^(#{lookup})\s*\(.*?\).*?\n\{\n(.*)^\}/m) { + name = $1 + body = $2 + body.sub!(/\benum\s+\{(\n[ \t]+)/, "\\&MIN_CODE_VALUE = 0x#{min.to_s(16)},\\1""MAX_CODE_VALUE = 0x#{max.to_s(16)},\\1") + body.gsub!(/(#{hash})\s*\(.*?\)/, "\\1(#{argname})") + body.gsub!(/\{"",-1}/, "-1") + body.gsub!(/\{"(?:[^"]|\\")+", *::::(.*)\}/, '\1') + body.sub!(/(\s+if\s)\(len\b.*\)/) do + "#$1(" << + (arity > 1 ? (0...arity).map {|i| range_check("#{argname}[#{i}]")}.join(" &&\n ") : range_check(argname)) << + ")" + end + v = nil + body.sub!(/(if\s*\(.*MAX_HASH_VALUE.*\)\n([ \t]*))\{(.*?)\n\2\}/m) { + pre = $1 + indent = $2 + s = $3 + s.sub!(/const char *\* *(\w+)( *= *wordlist\[\w+\]).\w+/, 'short \1 = wordlist[key]') + v = $1 + s.sub!(/\bif *\(.*\)/, "if (#{v} >= 0 && code#{arity}_equal(#{argname}, #{key}_Table[#{v}].from))") + "#{pre}{#{s}\n#{indent}}" + } + body.sub!(/\b(return\s+&)([^;]+);/, '\1'"#{key}_Table[#{v}].to;") + "static const #{type} *\n#{name}(#{argdecl})\n{\n#{body}}" + } + src + end + + def display(dest, mapping_data) + # print the header + dest.print("/* DO NOT EDIT THIS FILE. */\n") + dest.print("/* Generated by enc-case-folding.rb */\n\n") + + versions = version.scan(/\d+/) + dest.print("#if defined ONIG_UNICODE_VERSION_STRING && !( \\\n") + %w[MAJOR MINOR TEENY].zip(versions) do |n, v| + dest.print(" ONIG_UNICODE_VERSION_#{n} == #{v} && \\\n") + end + dest.print(" 1)\n") + dest.print("# error ONIG_UNICODE_VERSION_STRING mismatch\n") + dest.print("#endif\n") + dest.print("#define ONIG_UNICODE_VERSION_STRING #{version.dump}\n") + %w[MAJOR MINOR TEENY].zip(versions) do |n, v| + dest.print("#define ONIG_UNICODE_VERSION_#{n} #{v}\n") + end + dest.print("\n") + + # print folding data + + # CaseFold + CaseFold_Locale + name = "CaseFold_11" + data = print_table(dest, name, mapping_data, "CaseFold"=>fold, "CaseFold_Locale"=>fold_locale) + dest.print lookup_hash(name, "CodePointList3", data) + + # print unfolding data + + # CaseUnfold_11 + CaseUnfold_11_Locale + name = "CaseUnfold_11" + data = print_table(dest, name, mapping_data, name=>unfold[0], "#{name}_Locale"=>unfold_locale[0]) + dest.print lookup_hash(name, "CodePointList3", data) + + # CaseUnfold_12 + CaseUnfold_12_Locale + name = "CaseUnfold_12" + data = print_table(dest, name, mapping_data, name=>unfold[1], "#{name}_Locale"=>unfold_locale[1]) + dest.print lookup_hash(name, "CodePointList2", data) + + # CaseUnfold_13 + name = "CaseUnfold_13" + data = print_table(dest, name, mapping_data, name=>unfold[2]) + dest.print lookup_hash(name, "CodePointList2", data) + + # TitleCase + dest.print mapping_data.specials_output + end + + def debug! + @debug = true + end + + def self.load(*args) + new.load(*args) + end +end + +class MapItem + attr_accessor :upper, :lower, :title, :code + + def initialize(code, upper, lower, title) + @code = code + @upper = upper unless upper == '' + @lower = lower unless lower == '' + @title = title unless title == '' + end +end + +class CaseMapping + attr_reader :filename, :version + + def initialize(mapping_directory) + @mappings = {} + @specials = [] + @specials_length = 0 + @version = nil + File.foreach(File.join(mapping_directory, 'UnicodeData.txt'), mode: "rb") do |line| + next if line =~ /^</ + code, _, _, _, _, _, _, _, _, _, _, _, upper, lower, title = line.chomp.split ';' + unless upper and lower and title and (upper+lower+title)=='' + @mappings[code] = MapItem.new(code, upper, lower, title) + end + end + + @filename = File.join(mapping_directory, 'SpecialCasing.txt') + File.foreach(@filename, mode: "rb") do |line| + @version ||= line[/-([0-9.]+).txt/, 1] + line.chomp! + line, comment = line.split(/ *#/) + next if not line or line == '' + code, lower, title, upper, conditions = line.split(/ *; */) + unless conditions + item = @mappings[code] + item.lower = lower + item.title = title + item.upper = upper + end + end + end + + def map (from) + @mappings[from] + end + + def flags(from, type, to) + # types: CaseFold_11, CaseUnfold_11, CaseUnfold_12, CaseUnfold_13 + flags = "" + from = Array(from).map {|i| "%04X" % i}.join(" ") + to = Array(to).map {|i| "%04X" % i}.join(" ") + item = map(from) + specials = [] + case type + when 'CaseFold_11' + flags += '|F' + if item + flags += '|U' if to==item.upper + flags += '|D' if to==item.lower + unless item.upper == item.title + if item.code == item.title + flags += '|IT' + swap = case item.code + when '01C5' then '0064 017D' + when '01C8' then '006C 004A' + when '01CB' then '006E 004A' + when '01F2' then '0064 005A' + else # Greek + to.split(' ').first + ' 0399' + end + specials << swap + else + flags += '|ST' + specials << item.title + end + end + unless item.lower.nil? or item.lower==from or item.lower==to + specials << item.lower + flags += '|SL' + end + unless item.upper.nil? or item.upper==from or item.upper==to + specials << item.upper + flags += '|SU' + end + end + when 'CaseUnfold_11' + to = to.split(/ /) + if item + case to.first + when item.upper then flags += '|U' + when item.lower then flags += '|D' + else + raise "Unpredicted case 0 in enc/unicode/case_folding.rb. Please contact https://bugs.ruby-lang.org/." + end + unless item.upper == item.title + if item.code == item.title + flags += '|IT' # was unpredicted case 1 + elsif item.title==to[1] + flags += '|ST' + else + raise "Unpredicted case 2 in enc/unicode/case_folding.rb. Please contact https://bugs.ruby-lang.org/." + end + end + end + end + unless specials.empty? + flags += "|I(#{@specials_length})" + @specials_length += specials.map { |s| s.split(/ /).length }.reduce(:+) + @specials << specials + end + flags + end + + def debug! + @debug = true + end + + def specials_output + "static const OnigCodePoint CaseMappingSpecials[] = {\n" + + @specials.map do |sps| + ' ' + sps.map do |sp| + chars = sp.split(/ /) + ct = ' /* ' + Array(chars).map{|c|[c.to_i(16)].pack("U*")}.join(", ") + ' */' if @debug + " L(#{chars.length})|#{chars.map {|c| "0x"+c }.join(', ')}#{ct}," + end.join + "\n" + end.join + "};\n" + end + + def self.load(*args) + new(*args) + end +end + +class CaseMappingDummy + def flags(from, type, to) + "" + end + + def titlecase_output() '' end + def debug!() end +end + +if $0 == __FILE__ + require 'optparse' + dest = nil + mapping_directory = nil + mapping_data = nil + debug = false + fold_1 = false + ARGV.options do |opt| + opt.banner << " [INPUT]" + opt.on("--output-file=FILE", "-o", "output to the FILE instead of STDOUT") {|output| + dest = (output unless output == '-') + } + opt.on('--mapping-data-directory=DIRECTORY', '-m', 'data DIRECTORY of mapping files') { |directory| + mapping_directory = directory + } + opt.on('--debug', '-d') { + debug = true + } + opt.parse! + abort(opt.to_s) if ARGV.size > 1 + end + if mapping_directory + if ARGV[0] + warn "Either specify directory or individual file, but not both." + exit + end + filename = File.join(mapping_directory, 'CaseFolding.txt') + mapping_data = CaseMapping.load(mapping_directory) + end + filename ||= ARGV[0] || 'CaseFolding.txt' + data = CaseFolding.load(filename) + if mapping_data and data.version != mapping_data.version + abort "Unicode data version mismatch\n" \ + " #{filename} = #{data.version}\n" \ + " #{mapping_data.filename} = #{mapping_data.version}" + end + mapping_data ||= CaseMappingDummy.new + + if debug + data.debug! + mapping_data.debug! + end + f = StringIO.new + begin + data.display(f, mapping_data) + rescue Errno::ENOENT => e + raise unless /gperf/ =~ e.message + warn e.message + abort unless dest + File.utime(nil, nil, dest) # assume existing file is OK + exit + else + s = f.string + end + if dest + File.binwrite(dest, s) + else + STDOUT.print(s) + end +end diff --git a/tool/enc-emoji-citrus-gen.rb b/tool/enc-emoji-citrus-gen.rb index 5037cbde1e..0b37e48d3f 100644 --- a/tool/enc-emoji-citrus-gen.rb +++ b/tool/enc-emoji-citrus-gen.rb @@ -1,4 +1,4 @@ -require File.expand_path('../jisx0208', __FILE__) +require File.expand_path('../lib/jisx0208', __FILE__) ENCODES = [ { @@ -71,7 +71,7 @@ end def generate_to_ucs(params, pairs) pairs.sort_by! {|u, c| c } name = "EMOJI_#{params[:name]}%UCS" - open("#{name}.src", "w") do |io| + File.open("#{name}.src", "w") do |io| io.print header(params.merge(name: name.tr('%', '/'))) io.puts io.puts "BEGIN_MAP" @@ -83,7 +83,7 @@ end def generate_from_ucs(params, pairs) pairs.sort_by! {|u, c| u } name = "UCS%EMOJI_#{params[:name]}" - open("#{name}.src", "w") do |io| + File.open("#{name}.src", "w") do |io| io.print header(params.merge(name: name.tr('%', '/'))) io.puts io.puts "BEGIN_MAP" @@ -93,7 +93,7 @@ def generate_from_ucs(params, pairs) end def make_pairs(code_map) - pairs = code_map.inject([]) {|acc, (range, ch)| + code_map.inject([]) {|acc, (range, ch)| acc += range.map{|uni| pair = [uni, Integer(ch)]; ch = ch.succ; next pair } } end diff --git a/tool/enc-unicode.rb b/tool/enc-unicode.rb index 38140ab292..a89390ad8f 100755 --- a/tool/enc-unicode.rb +++ b/tool/enc-unicode.rb @@ -1,21 +1,49 @@ #!/usr/bin/env ruby -# Creates the data structures needed by Onigurma to map Unicode codepoints to +# Creates the data structures needed by Oniguruma to map Unicode codepoints to # property names and POSIX character classes # # To use this, get UnicodeData.txt, Scripts.txt, PropList.txt, # PropertyAliases.txt, PropertyValueAliases.txt, DerivedCoreProperties.txt, -# DerivedAge.txt and Blocks.txt from unicode.org. +# DerivedAge.txt, Blocks.txt, emoji/emoji-data.txt, +# auxiliary/GraphemeBreakProperty.txt from unicode.org # (http://unicode.org/Public/UNIDATA/) And run following command. -# ruby1.9 tool/enc-unicode.rb data_dir > enc/unicode/name2ctype.kwd +# tool/enc-unicode.rb data_dir emoji_data_dir > enc/unicode/name2ctype.kwd # You can get source file for gperf. After this, simply make ruby. - -unless ARGV.size == 1 - $stderr.puts "Usage: #{$0} data_directory" - exit(1) +# Or directly run: +# tool/enc-unicode.rb --header data_dir emoji_data_dir > enc/unicode/<VERSION>/name2ctype.h +# +# There are Makefile rules that automate steps above: `make update-unicode` and +# `make enc/unicode/<VERSION>/name2ctype.h`. + +while arg = ARGV.shift + case arg + when "--" + break + when "--header" + header = true + when "--diff" + diff = ARGV.shift or abort "#{$0}: --diff=DIFF-COMMAND" + when /\A--diff=(.+)/m + diff = $1 + when /\A-/ + abort "#{$0}: unknown option #{arg}" + else + ARGV.unshift(arg) + break + end +end +unless ARGV.size == 2 + abort "Usage: #{$0} data_directory emoji_data_directory" end -POSIX_NAMES = %w[NEWLINE Alpha Blank Cntrl Digit Graph Lower Print Punct Space Upper XDigit Word Alnum ASCII] +pat = /(?:\A|\/)([.\d]+)\z/ +$versions = { + :Unicode => ARGV[0][pat, 1], + :Emoji => ARGV[1][pat, 1], +} + +POSIX_NAMES = %w[NEWLINE Alpha Blank Cntrl Digit Graph Lower Print XPosixPunct Space Upper XDigit Word Alnum ASCII Punct] def pair_codepoints(codepoints) @@ -50,7 +78,7 @@ def parse_unicode_data(file) data = {'Any' => (0x0000..0x10ffff).to_a, 'Assigned' => [], 'ASCII' => (0..0x007F).to_a, 'NEWLINE' => [0x0a], 'Cn' => []} beg_cp = nil - IO.foreach(file) do |line| + File.foreach(file) do |line| fields = line.split(';') cp = fields[0].to_i(16) @@ -110,6 +138,7 @@ def define_posix_props(data) data['Upper'] = data['Uppercase'] data['Lower'] = data['Lowercase'] data['Punct'] = data['Punctuation'] + data['XPosixPunct'] = data['Punctuation'] + [0x24, 0x2b, 0x3c, 0x3d, 0x3e, 0x5e, 0x60, 0x7c, 0x7e] data['Digit'] = data['Decimal_Number'] data['XDigit'] = (0x0030..0x0039).to_a + (0x0041..0x0046).to_a + (0x0061..0x0066).to_a @@ -117,7 +146,8 @@ def define_posix_props(data) data['Space'] = data['White_Space'] data['Blank'] = data['Space_Separator'] + [0x0009] data['Cntrl'] = data['Cc'] - data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation'] + data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + + data['Connector_Punctuation'] + data['Join_Control'] data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] - data['Surrogate'] - data['Unassigned'] data['Print'] = data['Graph'] + data['Space_Separator'] @@ -127,21 +157,32 @@ def parse_scripts(data, categories) files = [ {:fn => 'DerivedCoreProperties.txt', :title => 'Derived Property'}, {:fn => 'Scripts.txt', :title => 'Script'}, - {:fn => 'PropList.txt', :title => 'Binary Property'} + {:fn => 'PropList.txt', :title => 'Binary Property'}, + {:fn => 'emoji/emoji-data.txt', :title => 'Emoji'} ] current = nil cps = [] names = {} files.each do |file| - IO.foreach(get_file(file[:fn])) do |line| - if /^# Total code points: / =~ line + data_foreach(file[:fn]) do |line| + # Parse Unicode data files and store code points and properties. + if /^# Total (?:code points|elements): / =~ line data[current] = cps categories[current] = file[:title] (names[file[:title]] ||= []) << current cps = [] - elsif /^([0-9a-fA-F]+)(?:..([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line - current = $3 + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w(?:[\w\s;]*\w)?)/ =~ line + # $1: The first hexadecimal code point or the start of a range. + # $2: The end code point of the range, if present. + # If there's no range (just a single code point), $2 is nil. + # $3: The property or other info. + # Example: + # line = "0915..0939 ; InCB; Consonant # Lo [37] DEVANAGARI LETTER KA..DEVANAGARI LETTER HA" + # $1 = "0915" + # $2 = "0939" + # $3 = "InCB; Consonant" $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + current = $3.gsub(/\W+/, '_') end end end @@ -154,12 +195,12 @@ end def parse_aliases(data) kv = {} - IO.foreach(get_file('PropertyAliases.txt')) do |line| + data_foreach('PropertyAliases.txt') do |line| next unless /^(\w+)\s*; (\w+)/ =~ line data[$1] = data[$2] kv[normalize_propname($1)] = normalize_propname($2) end - IO.foreach(get_file('PropertyValueAliases.txt')) do |line| + data_foreach('PropertyValueAliases.txt') do |line| next unless /^(sc|gc)\s*; (\w+)\s*; (\w+)(?:\s*; (\w+))?/ =~ line if $1 == 'gc' data[$3] = data[$2] @@ -184,7 +225,7 @@ def parse_age(data) last_constname = nil cps = [] ages = [] - IO.foreach(get_file('DerivedAge.txt')) do |line| + data_foreach('DerivedAge.txt') do |line| if /^# Total code points: / =~ line constname = constantize_agename(current) # each version matches all previous versions @@ -194,7 +235,7 @@ def parse_age(data) ages << current last_constname = constname cps = [] - elsif /^([0-9a-fA-F]+)(?:..([0-9a-fA-F]+))?\s*;\s*(\d+\.\d+)/ =~ line + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\d+\.\d+)/ =~ line current = $3 $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) end @@ -202,13 +243,30 @@ def parse_age(data) ages end -def parse_block(data) +def parse_GraphemeBreakProperty(data) current = nil - last_constname = nil + cps = [] + ages = [] + data_foreach('auxiliary/GraphemeBreakProperty.txt') do |line| + if /^# Total code points: / =~ line + constname = constantize_Grapheme_Cluster_Break(current) + data[constname] = cps + make_const(constname, cps, "Grapheme_Cluster_Break=#{current}") + ages << current + cps = [] + elsif /^(\h+)(?:\.\.(\h+))?\s*;\s*(\w+)/ =~ line + current = $3 + $2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16)) + end + end + ages +end + +def parse_block(data) cps = [] blocks = [] - IO.foreach(get_file('Blocks.txt')) do |line| - if /^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+);\s*(.*)/ =~ line + data_foreach('Blocks.txt') do |line| + if /^(\h+)\.\.(\h+);\s*(.*)/ =~ line cps = ($1.to_i(16)..$2.to_i(16)).to_a constname = constantize_blockname($3) data[constname] = cps @@ -225,19 +283,12 @@ def parse_block(data) blocks << constname end -# shim for Ruby 1.8 -unless {}.respond_to?(:key) - class Hash - alias key index - end -end - $const_cache = {} # make_const(property, pairs, name): Prints a 'static const' structure for a # given property, group of paired codepoints, and a human-friendly name for # the group def make_const(prop, data, name) - puts "\n/* '#{prop}': #{name} */" + puts "\n/* '#{prop}': #{name} */" # comment used to generate documentation if origprop = $const_cache.key(data) puts "#define CR_#{prop} CR_#{origprop}" else @@ -264,65 +315,179 @@ def constantize_agename(name) "Age_#{name.sub(/\./, '_')}" end +def constantize_Grapheme_Cluster_Break(name) + "Grapheme_Cluster_Break_#{name}" +end + def constantize_blockname(name) "In_#{name.gsub(/\W/, '_')}" end def get_file(name) - File.join(ARGV[0], name) + File.join(ARGV[name.start_with?("emoji-[stz]") ? 1 : 0], name) end +def data_foreach(name, &block) + fn = get_file(name) + warn "Reading #{name}" + File.open(fn, 'rb') do |f| + if /^emoji/ =~ name + line = f.gets("") + # Headers till Emoji 13 or 15 + version = line[/^# #{Regexp.quote(File.basename(name))}.*(?:^# Version:|Emoji Version) ([\d.]+)/m, 1] + type = :Emoji + else + # Headers since Emoji 14 or other Unicode data + line = f.gets("\n") + type = :Unicode + end + version ||= line[/^# #{File.basename(name).sub(/\./, '-([\\d.]+)\\.')}/, 1] + unless version + raise ArgumentError, <<-ERROR +#{name}: no #{type} version +#{line.gsub(/^/, '> ')} + ERROR + end + if !(v = $versions[type]) + $versions[type] = version + elsif v != version and "#{v}.0" != version + raise ArgumentError, <<-ERROR +#{name}: #{type} version mismatch: #{version} to #{v} +#{line.gsub(/^/, '> ')} + ERROR + end + f.each(&block) + end +end # Write Data +class Unifdef + attr_accessor :output, :top, :stack, :stdout, :kwdonly + def initialize(out) + @top = @output = [] + @stack = [] + $stdout, @stdout = self, out + end + def restore + $stdout = @stdout + end + def ifdef(sym) + if @kwdonly + @stdout.puts "#ifdef #{sym}" + else + @stack << @top + @top << tmp = [sym] + @top = tmp + end + if block_given? + begin + return yield + ensure + endif(sym) + end + end + end + def endif(sym) + if @kwdonly + @stdout.puts "#endif /* #{sym} */" + else + unless sym == @top[0] + restore + raise ArgumentError, "#{sym} unmatch to #{@top[0]}" + end + @top = @stack.pop + end + end + def show(dest, *syms) + _show(dest, @output, syms) + end + def _show(dest, ary, syms) + if Symbol === (sym = ary[0]) + unless syms.include?(sym) + return + end + end + ary.each do |e| + case e + when Array + _show(dest, e, syms) + when String + dest.print e + end + end + end + def write(str) + if @kwdonly + @stdout.write(str) + else + @top << str + end + self + end + alias << write +end + +output = Unifdef.new($stdout) +output.kwdonly = !header + puts '%{' -puts '#define long size_t' props, data = parse_unicode_data(get_file('UnicodeData.txt')) categories = {} props.concat parse_scripts(data, categories) aliases = parse_aliases(data) +ages = blocks = graphemeBreaks = nil define_posix_props(data) POSIX_NAMES.each do |name| - make_const(name, data[name], "[[:#{name}:]]") + if name == 'XPosixPunct' + make_const(name, data[name], "[[:Punct:]]") + else + make_const(name, data[name], "[[:#{name}:]]") + end end -print "\n#ifdef USE_UNICODE_PROPERTIES" -props.each do |name| - category = categories[name] || - case name.size - when 1 then 'Major Category' - when 2 then 'General Category' - else '-' - end - make_const(name, data[name], category) +output.ifdef :USE_UNICODE_PROPERTIES do + props.each do |name| + category = categories[name] || + case name.size + when 1 then 'Major Category' + when 2 then 'General Category' + else '-' + end + make_const(name, data[name], category) + end + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages = parse_age(data) + end + graphemeBreaks = parse_GraphemeBreakProperty(data) + blocks = parse_block(data) end -print "\n#ifdef USE_UNICODE_AGE_PROPERTIES" -ages = parse_age(data) -puts "#endif /* USE_UNICODE_AGE_PROPERTIES */" -blocks = parse_block(data) -puts '#endif /* USE_UNICODE_PROPERTIES */' puts(<<'__HEREDOC') static const OnigCodePoint* const CodeRanges[] = { __HEREDOC POSIX_NAMES.each{|name|puts" CR_#{name},"} -puts "#ifdef USE_UNICODE_PROPERTIES" -props.each{|name| puts" CR_#{name},"} -puts "#ifdef USE_UNICODE_AGE_PROPERTIES" -ages.each{|name| puts" CR_#{constantize_agename(name)},"} -puts "#endif /* USE_UNICODE_AGE_PROPERTIES */" -blocks.each{|name|puts" CR_#{name},"} +output.ifdef :USE_UNICODE_PROPERTIES do + props.each{|name| puts" CR_#{name},"} + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages.each{|name| puts" CR_#{constantize_agename(name)},"} + end + graphemeBreaks.each{|name| puts" CR_#{constantize_Grapheme_Cluster_Break(name)},"} + blocks.each{|name|puts" CR_#{name},"} +end puts(<<'__HEREDOC') -#endif /* USE_UNICODE_PROPERTIES */ }; struct uniname2ctype_struct { - int name, ctype; + short name; + unsigned short ctype; }; +#define uniname2ctype_offset(str) offsetof(struct uniname2ctype_pool_t, uniname2ctype_pool_##str) -static const struct uniname2ctype_struct *uniname2ctype_p(const char *, unsigned int); +static const struct uniname2ctype_struct *uniname2ctype_p(register const char *str, register size_t len); %} struct uniname2ctype_struct; %% __HEREDOC + i = -1 name_to_index = {} POSIX_NAMES.each do |name| @@ -332,34 +497,44 @@ POSIX_NAMES.each do |name| name_to_index[name] = i puts"%-40s %3d" % [name + ',', i] end -puts "#ifdef USE_UNICODE_PROPERTIES" -props.each do |name| - i += 1 - name = normalize_propname(name) - name_to_index[name] = i - puts "%-40s %3d" % [name + ',', i] -end -aliases.each_pair do |k, v| - next if name_to_index[k] - next unless v = name_to_index[v] - puts "%-40s %3d" % [k + ',', v] -end -puts "#ifdef USE_UNICODE_AGE_PROPERTIES" -ages.each do |name| - i += 1 - name = "age=#{name}" - name_to_index[name] = i - puts "%-40s %3d" % [name + ',', i] -end -puts "#endif /* USE_UNICODE_AGE_PROPERTIES */" -blocks.each do |name| - i += 1 - name = normalize_propname(name) - name_to_index[name] = i - puts "%-40s %3d" % [name + ',', i] +output.ifdef :USE_UNICODE_PROPERTIES do + props.each do |name| + i += 1 + name = if name.start_with?('InCB') + name.downcase.gsub(/_/, '=') + else + normalize_propname(name) + end + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + aliases.each_pair do |k, v| + next if name_to_index[k] + next unless v = name_to_index[v] + puts "%-40s %3d" % [k + ',', v] + end + output.ifdef :USE_UNICODE_AGE_PROPERTIES do + ages.each do |name| + i += 1 + name = "age=#{name}" + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + end + graphemeBreaks.each do |name| + i += 1 + name = "graphemeclusterbreak=#{name.delete('_').downcase}" + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end + blocks.each do |name| + i += 1 + name = normalize_propname(name) + name_to_index[name] = i + puts "%-40s %3d" % [name + ',', i] + end end puts(<<'__HEREDOC') -#endif /* USE_UNICODE_PROPERTIES */ %% static int uniname2ctype(const UChar *name, unsigned int len) @@ -369,3 +544,92 @@ uniname2ctype(const UChar *name, unsigned int len) return -1; } __HEREDOC +$versions.each do |type, ver| + name = type == :Unicode ? "ONIG_UNICODE_VERSION" : "ONIG_UNICODE_EMOJI_VERSION" + versions = ver.scan(/\d+/) + print("#if defined #{name}_STRING && !( \\\n") + versions.zip(%w[MAJOR MINOR TEENY]) do |v, n| + print(" #{name}_#{n} == #{v} && \\\n") + end + print(" 1)\n") + print("# error #{name}_STRING mismatch\n") + print("#endif\n") + print("#define #{name}_STRING #{ver.dump}\n") + versions.zip(%w[MAJOR MINOR TEENY]) do |v, n| + print("#define #{name}_#{n} #{v}\n") + end +end + +output.restore + +if header + require 'tempfile' + + def diff_args(diff) + ok = IO.popen([diff, "-DDIFF_TEST", IO::NULL, "-"], "r+") do |f| + f.puts "Test for diffutils 3.8" + f.close_write + /^#if/ =~ f.read + end + if ok + proc {|macro, *inputs| + [diff, "-D#{macro}", *inputs] + } + else + IO.popen([diff, "--old-group-format=%<", "--new-group-format=%>", IO::NULL, IO::NULL], err: %i[child out], &:read) + unless $?.success? + abort "#{$0}: #{diff} -D does not work" + end + warn "Avoiding diffutils 3.8 bug#61193" + proc {|macro, *inputs| + [diff] + [ + "--old-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#endif /* ! @ */\n", + + "--new-group-format=" \ + "#ifdef @\n" \ + "%>" \ + "#endif /* @ */\n", + + "--changed-group-format=" \ + "#ifndef @\n" \ + "%<" \ + "#else /* @ */\n" \ + "%>" \ + "#endif /* @ */\n" + ].map {|opt| opt.gsub(/@/) {macro}} + inputs + } + end + end + + ifdef = diff_args(diff || "diff") + + NAME2CTYPE = %w[gperf -7 -c -j1 -i1 -t -C -P -T -H uniname2ctype_hash -Q uniname2ctype_pool -N uniname2ctype_p] + + fds = [] + syms = %i[USE_UNICODE_PROPERTIES USE_UNICODE_AGE_PROPERTIES] + begin + fds << (tmp = Tempfile.new(%w"name2ctype .h")) + IO.popen([*NAME2CTYPE, out: tmp], "w") {|f| output.show(f, *syms)} + end while syms.pop + fds.each(&:close) + IO.popen(ifdef["USE_UNICODE_AGE_PROPERTIES", fds[1].path, fds[0].path], "r") {|age| + IO.popen(ifdef["USE_UNICODE_PROPERTIES", fds[2].path, "-"], "r", in: age) {|f| + ansi = false + f.each {|line| + if /ANSI-C code produced by gperf/ =~ line + ansi = true + end + line.sub!(/\/\*ANSI\*\//, '1') if ansi + line.gsub!(/\(int\)\((?:long|size_t)\)&\(\(struct uniname2ctype_pool_t \*\)0\)->uniname2ctype_pool_(str\d+),\s+/, + 'uniname2ctype_offset(\1), ') + if line.start_with?("uniname2ctype_hash\s") ... line.start_with?("}") + line.sub!(/^( *(?:register\s+)?(.*\S)\s+hval\s*=\s*)(?=len;)/, '\1(\2)') + end + puts line + } + } + } +end diff --git a/tool/eval.rb b/tool/eval.rb index 18e645a314..9153573e6e 100644 --- a/tool/eval.rb +++ b/tool/eval.rb @@ -1,3 +1,4 @@ +# VM checking and benchmarking code require './rbconfig' require 'fileutils' @@ -106,7 +107,6 @@ def calc_each data end def calc_stat stats - stat = [] stats[0].each_with_index{|e, idx| bm = e[0] vals = stats.map{|st| @@ -133,8 +133,7 @@ def stat } # pp total total[0].each_with_index{|e, idx| - bm = e[0] - # print "#{bm}\t" + # print "#{e[0]}\t" total.each{|st| print st[idx][1], "\t" } diff --git a/tool/expand-config.rb b/tool/expand-config.rb index bb88865709..ac0ffbfd41 100755 --- a/tool/expand-config.rb +++ b/tool/expand-config.rb @@ -1,11 +1,23 @@ #!./miniruby -s +# Used to expand Ruby config entries for Win32 Makefiles. + config = File.read(conffile = $config) config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)$/, '\1true\2') rbconfig = Module.new {module_eval(config, conffile)}::RbConfig config = $expand ? rbconfig::CONFIG : rbconfig::MAKEFILE_CONFIG config["RUBY_RELEASE_DATE"] ||= - File.read(File.expand_path("../../version.h", __FILE__))[/^\s*#\s*define\s+RUBY_RELEASE_DATE\s+"(.*)"/, 1] + [ + ["revision.h"], + ["../../revision.h", __FILE__], + ["../../version.h", __FILE__], + ].find do |hdr, dir| + hdr = File.expand_path(hdr, dir) if dir + if date = File.read(hdr)[/^\s*#\s*define\s+RUBY_RELEASE_DATE(?:TIME)?\s+"([0-9-]*)/, 1] + break date + end +rescue +end while /\A(\w+)=(.*)/ =~ ARGV[0] config[$1] = $2 @@ -13,10 +25,8 @@ while /\A(\w+)=(.*)/ =~ ARGV[0] ARGV.shift end -re = /@(#{config.keys.map {|k| Regexp.quote(k)}.join('|')})@/ - if $output - output = open($output, "wb", $mode &&= $mode.oct) + output = File.open($output, "wb", $mode &&= $mode.oct) output.chmod($mode) if $mode else output = STDOUT diff --git a/tool/extlibs.rb b/tool/extlibs.rb index 6323d8fdcd..cef6712833 100755 --- a/tool/extlibs.rb +++ b/tool/extlibs.rb @@ -1,143 +1,285 @@ #!/usr/bin/ruby -require 'fileutils' + +# Used to download, extract and patch extension libraries (extlibs) +# for Ruby. See common.mk for Ruby's usage. + require 'digest' require_relative 'downloader' +begin + require_relative 'lib/colorize' +rescue LoadError +end -cache_dir = ".downloaded-cache" -FileUtils.mkdir_p(cache_dir) +class ExtLibs + unless defined?(Colorize) + class Colorize + def pass(str) str; end + def fail(str) str; end + end + end -def do_download(url, base, cache_dir) - Downloader.download(url, base, cache_dir, nil) -end + class Vars < Hash + def pattern + /\$\((#{Regexp.union(keys)})\)/ + end -def do_checksum(cache, chksums) - chksums.each do |sum| - name, sum = sum.split(/:/) - if $VERBOSE - $stdout.print "checking #{name} of #{cache} ..." - $stdout.flush + def expand(str) + if empty? + str + else + str.gsub(pattern) {self[$1]} + end end - hd = Digest(name.upcase).file(cache).hexdigest - if hd == sum + end + + def initialize(mode = :all, cache_dir: nil) + @mode = mode + @cache_dir = cache_dir + @colorize = Colorize.new + end + + def cache_file(url, cache_dir) + Downloader.cache_file(url, nil, cache_dir).to_path + end + + def do_download(url, cache_dir) + Downloader.download(url, nil, nil, nil, cache_dir: cache_dir) + end + + def do_checksum(cache, chksums) + chksums.each do |sum| + name, sum = sum.split(/:/) if $VERBOSE - $stdout.puts " OK" + $stdout.print "checking #{name} of #{cache} ..." $stdout.flush end - else + hd = Digest(name.upcase).file(cache).hexdigest if $VERBOSE - $stdout.puts " NG" + $stdout.print " " + $stdout.puts hd == sum ? @colorize.pass("OK") : @colorize.fail("NG") $stdout.flush end - raise "checksum mismatch: #{cache}, #{name}:#{hd}, expected #{sum}" + unless hd == sum + raise "checksum mismatch: #{cache}, #{name}:#{hd}, expected #{sum}" + end end end -end -def do_extract(cache, dir) - if $VERBOSE - $stdout.puts "extracting #{cache} into #{dir}" - $stdout.flush + def do_extract(cache, dir) + if $VERBOSE + $stdout.puts "extracting #{cache} into #{dir}" + $stdout.flush + end + ext = File.extname(cache) + case ext + when '.gz', '.tgz' + f = IO.popen(["gzip", "-dc", cache]) + cache = cache.chomp('.gz') + when '.bz2', '.tbz' + f = IO.popen(["bzip2", "-dc", cache]) + cache = cache.chomp('.bz2') + when '.xz', '.txz' + f = IO.popen(["xz", "-dc", cache]) + cache = cache.chomp('.xz') + else + inp = cache + end + inp ||= f.binmode + ext = File.extname(cache) + case ext + when '.tar', /\A\.t[gbx]z\z/ + pid = Process.spawn("tar", "xpf", "-", in: inp, chdir: dir) + when '.zip' + pid = Process.spawn("unzip", inp, "-d", dir) + end + f.close if f + Process.wait(pid) + $?.success? or raise "failed to extract #{cache}" end - ext = File.extname(cache) - case ext - when '.gz', '.tgz' - f = IO.popen(["gzip", "-dc", cache]) - cache = cache.chomp('.gz') - when '.bz2', '.tbz' - f = IO.popen(["bzip2", "-dc", cache]) - cache = cache.chomp('.bz2') - when '.xz', '.txz' - f = IO.popen(["xz", "-dc", cache]) - cache = cache.chomp('.xz') - else - inp = cache + + def do_patch(dest, patch, args) + if $VERBOSE + $stdout.puts "applying #{patch} under #{dest}" + $stdout.flush + end + Process.wait(Process.spawn(ENV.fetch("PATCH", "patch"), "-d", dest, "-i", patch, *args)) + $?.success? or raise "failed to patch #{patch}" end - inp ||= f.binmode - ext = File.extname(cache) - case ext - when '.tar', /\A\.t[gbx]z\z/ - pid = Process.spawn("tar", "xpf", "-", in: inp, chdir: dir) - when '.zip' - pid = Process.spawn("unzip", inp, "-d", dir) + + def do_link(file, src, dest) + file = File.join(dest, file) + if (target = src).start_with?("/") + target = File.join([".."] * file.count("/"), src) + end + return unless File.exist?(File.expand_path(target, File.dirname(file))) + File.unlink(file) rescue nil + begin + File.symlink(target, file) + rescue + else + if $VERBOSE + $stdout.puts "linked #{target} to #{file}" + $stdout.flush + end + return + end + begin + src = src.sub(/\A\//, '') + File.copy_stream(src, file) + rescue + if $VERBOSE + $stdout.puts "failed to link #{src} to #{file}: #{$!.message}" + end + else + if $VERBOSE + $stdout.puts "copied #{src} to #{file}" + end + end end - f.close if f - Process.wait(pid) - $?.success? or raise "failed to extract #{cache}" -end -def do_patch(dest, patch, args) - if $VERBOSE - $stdout.puts "applying #{patch} under #{dest}" - $stdout.flush + def do_exec(command, dir, dest) + dir = dir ? File.join(dest, dir) : dest + if $VERBOSE + $stdout.puts "running #{command.dump} under #{dir}" + $stdout.flush + end + system(command, chdir: dir) or raise "failed #{command.dump}" end - Process.wait(Process.spawn("patch", "-d", dest, "-i", patch, *args)) - $?.success? or raise "failed to patch #{patch}" -end -case ARGV[0] -when '--download' - mode = :download - ARGV.shift -when '--extract' - mode = :extract - ARGV.shift -when '--patch' - mode = :patch - ARGV.shift -when '--all' - mode = :all - ARGV.shift -else - mode = :all -end + def do_command(mode, dest, url, cache_dir, chksums) + extracted = false + base = /.*(?=\.tar(?:\.\w+)?\z)/ -success = true -ARGV.each do |dir| - Dir.glob("#{dir}/**/extlibs") do |list| + case mode + when :download + cache = do_download(url, cache_dir) + do_checksum(cache, chksums) + when :extract + cache = cache_file(url, cache_dir) + target = File.join(dest, File.basename(cache)[base]) + unless File.directory?(target) + do_checksum(cache, chksums) + extracted = do_extract(cache, dest) + end + when :all + cache = do_download(url, cache_dir) + target = File.join(dest, File.basename(cache)[base]) + unless File.directory?(target) + do_checksum(cache, chksums) + extracted = do_extract(cache, dest) + end + end + extracted + end + + def process(list) + mode = @mode + cache_dir = @cache_dir + after_extract = (mode == :all or mode == :patch) + success = true if $VERBOSE $stdout.puts "downloading for #{list}" $stdout.flush end + vars = Vars.new extracted = false dest = File.dirname(list) - IO.foreach(list) do |line| + url = chksums = nil + File.foreach(list) do |line| line.sub!(/\s*#.*/, '') - if /^\t/ =~ line - if extracted and (mode == :all or mode == :patch) - patch, *args = line.split + if /^(\w+)\s*=\s*(.*)/ =~ line + vars[$1] = vars.expand($2) + next + end + if chksums + chksums.concat(line.split) + elsif /^\t/ =~ line + if extracted and after_extract + patch, *args = line.split.map {|s| vars.expand(s)} do_patch(dest, patch, args) end next + elsif /^!\s*(?:chdir:\s*([^|\s]+)\|\s*)?(.*)/ =~ line + if extracted and after_extract + command = vars.expand($2.strip) + chdir = $1 and chdir = vars.expand(chdir) + do_exec(command, chdir, dest) + end + next + elsif /->/ =~ line + if extracted and after_extract + link, file = $`.strip, $'.strip + do_link(vars.expand(link), vars.expand(file), dest) + end + next + else + url, *chksums = line.split(' ') + end + if chksums.last == '\\' + chksums.pop + next + end + unless url + chksums = nil + next end - url, *chksums = line.split(' ') - next unless url - extracted = false - base = File.basename(url) - cache = File.join(cache_dir, base) - target = File.join(dest, base[/.*(?=\.tar(?:\.\w+)?\z)/]) + url = vars.expand(url) begin - case mode - when :download - do_download(url, base, cache_dir) - do_checksum(cache, chksums) - when :extract - unless File.directory?(target) - do_checksum(cache, chksums) - extracted = do_extract(cache, dest) - end - when :all - do_download(url, base, cache_dir) - unless File.directory?(target) - do_checksum(cache, chksums) - extracted = do_extract(cache, dest) - end - end + extracted = do_command(mode, dest, url, cache_dir, chksums) rescue => e - warn e.inspect + warn defined?(e.full_message) ? e.full_message : e.message success = false end + url = chksums = nil + end + success + end + + def process_under(dir) + success = true + Dir.glob("#{dir}/**/extlibs") do |list| + success &= process(list) + end + success + end + + def self.run(argv) + cache_dir = nil + mode = :all + until argv.empty? + case argv[0] + when '--download' + mode = :download + when '--extract' + mode = :extract + when '--patch' + mode = :patch + when '--all' + mode = :all + when '--cache' + argv.shift + cache_dir = argv[0] + when /\A--cache=/ + cache_dir = $' + when '--' + argv.shift + break + when /\A-/ + warn "unknown option: #{argv[0]}" + return false + else + break + end + argv.shift + end + + extlibs = new(mode, cache_dir: cache_dir) + argv.inject(true) do |success, dir| + success & extlibs.process_under(dir) end end end -exit(success) +if $0 == __FILE__ + exit ExtLibs.run(ARGV) +end diff --git a/tool/fake.rb b/tool/fake.rb index b2bdd086c4..2c458985d8 100644 --- a/tool/fake.rb +++ b/tool/fake.rb @@ -1,3 +1,6 @@ +# Used by Makefile and configure for building Ruby. +# See common.mk and Makefile.in for details. + class File sep = ("\\" if RUBY_PLATFORM =~ /mswin|bccwin|mingw/) if sep != ALT_SEPARATOR @@ -6,24 +9,21 @@ class File end end +[[libpathenv, "."], [preloadenv, libruby_so]].each do |env, path| + env or next + e = ENV[env] or next + e = e.split(File::PATH_SEPARATOR) + path = File.realpath(path, builddir) rescue next + e.delete(path) or next + ENV[env] = (e.join(File::PATH_SEPARATOR) unless e.empty?) +end + static = !!(defined?($static) && $static) $:.unshift(builddir) posthook = proc do - config = RbConfig::CONFIG - mkconfig = RbConfig::MAKEFILE_CONFIG - extout = File.expand_path(mkconfig["EXTOUT"], builddir) - [ - ["top_srcdir", $top_srcdir], - ["topdir", $topdir], - ].each do |var, val| - next unless val - mkconfig[var] = config[var] = val - t = /\A#{Regexp.quote(val)}(?=\/)/ - $hdrdir.sub!(t) {"$(#{var})"} - mkconfig.keys.grep(/dir\z/) do |k| - mkconfig[k] = "$(#{var})#$'" if t =~ mkconfig[k] - end - end + RbConfig.fire_update!("top_srcdir", $top_srcdir) + RbConfig.fire_update!("topdir", $topdir) + $hdrdir.sub!(/\A#{Regexp.quote($top_srcdir)}(?=\/)/, "$(top_srcdir)") if $extmk $ruby = "$(topdir)/miniruby -I'$(topdir)' -I'$(top_srcdir)/lib' -I'$(extout)/$(arch)' -I'$(extout)/common'" else @@ -33,6 +33,7 @@ posthook = proc do untrace_var(:$ruby, posthook) end prehook = proc do |extmk| +=begin pat = %r[(?:\A(?:\w:|//[^/]+)|\G)/[^/]*] dir = builddir.scan(pat) pwd = Dir.pwd.scan(pat) @@ -41,22 +42,30 @@ prehook = proc do |extmk| dir.shift pwd.shift end - builddir = File.join([".."]*pwd.size + dir) + builddir = File.join((pwd.empty? ? ["."] : [".."]*pwd.size) + dir) builddir = "." if builddir.empty? end +=end join = proc {|*args| File.join(*args).sub!(/\A(?:\.\/)*/, '')} $topdir ||= builddir - $top_srcdir ||= join[$topdir, srcdir] + $top_srcdir ||= (File.identical?(top_srcdir, dir = join[$topdir, srcdir]) ? + dir : top_srcdir) $extout = '$(topdir)/.ext' $extout_prefix = '$(extout)$(target_prefix)/' config = RbConfig::CONFIG mkconfig = RbConfig::MAKEFILE_CONFIG - mkconfig["builddir"] = config["builddir"] = builddir - mkconfig["top_srcdir"] = $top_srcdir if $top_srcdir - config["top_srcdir"] = File.expand_path($top_srcdir ||= top_srcdir) - config["rubyhdrdir"] = join[$top_srcdir, "include"] - config["rubyarchhdrdir"] = join[builddir, config["EXTOUT"], "include", config["arch"]] - mkconfig["libdirname"] = "builddir" + $builtruby ||= File.join(builddir, config['RUBY_INSTALL_NAME'] + config['EXEEXT']) + RbConfig.fire_update!("builddir", builddir) + RbConfig.fire_update!("buildlibdir", builddir) + RbConfig.fire_update!("libdir", builddir) + RbConfig.fire_update!("prefix", $topdir) + RbConfig.fire_update!("top_srcdir", $top_srcdir ||= top_srcdir) + RbConfig.fire_update!("extout", $extout) + RbConfig.fire_update!("rubyhdrdir", "$(top_srcdir)/include") + RbConfig.fire_update!("rubyarchhdrdir", "$(extout)/include/$(arch)") + RbConfig.fire_update!("rubyarchdir", "$(extout)/$(arch)") + RbConfig.fire_update!("rubylibdir", "$(extout)/common") + RbConfig.fire_update!("libdirname", "buildlibdir") trace_var(:$ruby, posthook) untrace_var(:$extmk, prehook) end diff --git a/tool/fetch-bundled_gems.rb b/tool/fetch-bundled_gems.rb new file mode 100755 index 0000000000..127ea236f3 --- /dev/null +++ b/tool/fetch-bundled_gems.rb @@ -0,0 +1,54 @@ +#!ruby -alnF\s+|#.* +BEGIN { + require 'fileutils' + require_relative 'lib/colorize' + + color = Colorize.new + + if ARGV.first.start_with?("BUNDLED_GEMS=") + bundled_gems = ARGV.shift[13..-1] + sep = bundled_gems.include?(",") ? "," : " " + bundled_gems = bundled_gems.split(sep) + bundled_gems = nil if bundled_gems.empty? + end + + dir = ARGV.shift + ARGF.eof? + FileUtils.mkdir_p(dir) + Dir.chdir(dir) +} + +n, v, u, r = $F + +next unless n +next if bundled_gems&.all? {|pat| !File.fnmatch?(pat, n)} + +unless File.exist?("#{n}/.git") + puts "retrieving #{color.notice(n)} ..." + system(*%W"git clone --depth=1 --no-tags #{u} #{n}") or abort +end + +if r + puts "fetching #{color.notice(r)} ..." + system("git", "fetch", "origin", r, chdir: n) or abort + c = r +else + c = ["v#{v}", v].find do |c| + puts "fetching #{color.notice(c)} ..." + system("git", "fetch", "origin", "refs/tags/#{c}:refs/tags/#{c}", chdir: n) + end or abort +end + +checkout = %w"git -c advice.detachedHead=false checkout" +info = %[, r=#{color.info(r)}] if r +puts "checking out #{color.notice(c)} (v=#{color.info(v)}#{info}) ..." +unless system(*checkout, c, "--", chdir: n) + abort if r or !system(*checkout, v, "--", chdir: n) +end + +if r + unless File.exist? "#{n}/#{n}.gemspec" + require_relative "lib/bundled_gem" + BundledGem.dummy_gemspec("#{n}/#{n}.gemspec") + end +end diff --git a/tool/file2lastrev.rb b/tool/file2lastrev.rb index 13f9a3bd1a..2de8379606 100755 --- a/tool/file2lastrev.rb +++ b/tool/file2lastrev.rb @@ -1,94 +1,106 @@ #!/usr/bin/env ruby +# Gets the most recent revision of a file in a VCS-agnostic way. +# Used by Doxygen, Makefiles and merger.rb. + require 'optparse' # this file run with BASERUBY, which may be older than 1.9, so no # require_relative -require File.expand_path('../vcs', __FILE__) +require File.expand_path('../lib/vcs', __FILE__) +require File.expand_path('../lib/output', __FILE__) Program = $0 -@output = nil -def self.output=(output) - if @output and @output != output +@format = nil +def self.format=(format) + if @format and @format != format raise "you can specify only one of --changed, --revision.h and --doxygen" end - @output = output + @format = format end @suppress_not_found = false +@limit = 20 +@output = Output.new -format = '%Y-%m-%dT%H:%M:%S%z' -srcdir = nil -parser = OptionParser.new {|opts| +time_format = '%Y-%m-%dT%H:%M:%S%z' +vcs = nil +create_only = false +OptionParser.new {|opts| + opts.banner << " paths..." + vcs_options = VCS.define_options(opts) + opts.new {@output.def_options(opts)} + srcdir = nil + opts.new opts.on("--srcdir=PATH", "use PATH as source directory") do |path| + abort "#{File.basename(Program)}: srcdir is already set" if srcdir srcdir = path + @output.vpath.add(srcdir) end opts.on("--changed", "changed rev") do - self.output = :changed + self.format = :changed end opts.on("--revision.h", "RUBY_REVISION macro") do - self.output = :revision_h + self.format = :revision_h end opts.on("--doxygen", "Doxygen format") do - self.output = :doxygen + self.format = :doxygen end opts.on("--modified[=FORMAT]", "modified time") do |fmt| - self.output = :modified - format = fmt if fmt + self.format = :modified + time_format = fmt if fmt + end + opts.on("--limit=NUM", "limit branch name length (#@limit)", Integer) do |n| + @limit = n end opts.on("-q", "--suppress_not_found") do @suppress_not_found = true end + opts.order! rescue abort "#{File.basename(Program)}: #{$!}\n#{opts}" + begin + vcs = VCS.detect(srcdir || ".", vcs_options, opts.new) + rescue VCS::NotFoundError => e + abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found + opts.remove + (vcs = VCS::Null.new(nil)).set_options(vcs_options) + if @format == :revision_h + create_only = true # don't overwrite existing revision.h when .git doesn't exist + end + end } -parser.parse! rescue abort "#{File.basename(Program)}: #{$!}\n#{parser}" -@output = - case @output +formatter = + case @format when :changed, nil - proc {|last, changed| - changed + Proc.new {|last, changed| + changed || "" } when :revision_h - proc {|last, changed, modified, branch, title| - [ - "#define RUBY_REVISION #{changed || 0}", - if branch - e = '..' - limit = 16 - name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} - "#define RUBY_BRANCH_NAME #{name.dump}" - end, - if title - "#define RUBY_LAST_COMMIT_TITLE #{title.dump}" - end, - ].compact + Proc.new {|last, changed, modified, branch, title| + vcs.revision_header(last, modified, modified, branch, title, limit: @limit).join("\n") } when :doxygen - proc {|last, changed| + Proc.new {|last, changed| "r#{changed}/r#{last}" } when :modified - proc {|last, changed, modified| - modified.strftime(format) + Proc.new {|last, changed, modified| + modified.strftime(time_format) } else - raise "unknown output format `#{@output}'" + raise "unknown output format `#{@format}'" end -srcdir ||= File.dirname(File.dirname(Program)) -begin - vcs = VCS.detect(srcdir) -rescue VCS::NotFoundError => e - abort "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found -else - ok = true - (ARGV.empty? ? [nil] : ARGV).each do |arg| - begin - puts @output[*vcs.get_revisions(arg)] - rescue => e - warn "#{File.basename(Program)}: #{e.message}" unless @suppress_not_found - ok = false - end +ok = true +(ARGV.empty? ? [nil] : ARGV).each do |arg| + begin + data = formatter[*vcs.get_revisions(arg)] + data.sub!(/(?<!\A|\n)\z/, "\n") + @output.write(data, overwrite: true, create_only: create_only) + rescue => e + next if @suppress_not_found and VCS::NotFoundError === e + warn "#{File.basename(Program)}: #{e.message}" + ok = false end - exit ok end +exit ok diff --git a/tool/format-release b/tool/format-release new file mode 100755 index 0000000000..8bb6154243 --- /dev/null +++ b/tool/format-release @@ -0,0 +1,245 @@ +#!/usr/bin/env ruby + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "diffy" +end + +require "open-uri" +require "yaml" +require_relative "./ruby-version" + +Diffy::Diff.default_options.merge!( + include_diff_info: true, + context: 1, +) + +class Tarball + attr_reader :version, :size, :sha1, :sha256, :sha512 + + def initialize(version, url, size, sha1, sha256, sha512) + @url = url + @size = size + @sha1 = sha1 + @sha256 = sha256 + @sha512 = sha512 + @version = version + @xy = version[/\A\d+\.\d+/] + end + + def gz?; @url.end_with?('.gz'); end + def zip?; @url.end_with?('.zip'); end + def xz?; @url.end_with?('.xz'); end + + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end + + def to_md + <<eom +* <https://cache.ruby-lang.org/pub/ruby/#{@xy}/ruby-#{@version}.#{ext}> + + SIZE: #{@size} bytes + SHA1: #{@sha1} + SHA256: #{@sha256} + SHA512: #{@sha512} +eom + end + + # * /home/naruse/obj/ruby-trunk/tmp/ruby-2.6.0-preview3.tar.gz + # SIZE: 17116009 bytes + # SHA1: 21f62c369661a2ab1b521fd2fa8191a4273e12a1 + # SHA256: 97cea8aa63dfa250ba6902b658a7aa066daf817b22f82b7ee28f44aec7c2e394 + # SHA512: 1e2042324821bb4e110af7067f52891606dcfc71e640c194ab1c117f0b941550e0b3ac36ad3511214ac80c536b9e5cfaf8789eec74cf56971a832ea8fc4e6d94 + def self.parse(wwwdir, version, rubydir) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + teeny = Integer($3) + + uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" + info = YAML.unsafe_load(URI(uri).read) + if info.size != 1 + raise "unexpected info.yml '#{uri}'" + end + tarballs = [] + info[0]["size"].each_key do |ext| + url = info[0]["url"][ext] + size = info[0]["size"][ext] + sha1 = info[0]["sha1"][ext] + sha256 = info[0]["sha256"][ext] + sha512 = info[0]["sha512"][ext] + tarball = Tarball.new(version, url, size, sha1, sha256, sha512) + tarballs << tarball + end + + if teeny == 0 + # show diff shortstat + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) + stat = `git -C #{rubydir} diff -l0 --shortstat #{prev_tag}..#{tag}` + files_changed, insertions, deletions = stat.scan(/\d+/) + end + + xy = version[/\A\d+\.\d+/] + #puts "## Download\n\n" + #tarballs.each do |tarball| + # puts tarball.to_md + #end + update_branches_yml(version, xy, wwwdir) + update_downloads_yml(version, xy, wwwdir) + update_releases_yml(version, xy, tarballs, wwwdir, files_changed, insertions, deletions) + tarballs + end + + def self.update_branches_yml(ver, xy, wwwdir) + filename = "_data/branches.yml" + orig_data = File.read(File.join(wwwdir, filename)) + data = orig_data.dup + if data.include?("\n- name: #{xy}\n") + data.sub!(/\n- name: #{Regexp.escape(xy)}\n(?: .*\n)*/) do |node| + unless ver.include?("-") + # assume this is X.Y.0 release + node.sub!(/^ status: preview\n/, " status: normal maintenance\n") + node.sub!(/^ date:\n/, " date: #{Time.now.year}-12-25\n") + end + node + end + else + if ver.include?("-") + status = "preview" + year = nil + else + status = "normal maintenance" + year = Time.now.year + end + entry = <<eom +- name: #{xy} + status: #{status} + date:#{ year && " #{year}-12-25" } + eol_date: + +eom + data.sub!(/(?=^- name)/, entry) + end + if data != orig_data + diff = Diffy::Diff.new(orig_data, data) + show_diff(filename, diff) + end + end + + def self.update_downloads_yml(ver, xy, wwwdir) + filename = "_data/downloads.yml" + orig_data = File.read(File.join(wwwdir, filename)) + data = orig_data.dup + + if /^preview:\n\n(?: .*\n)* - #{Regexp.escape(xy)}\./ =~ data + if ver.include?("-") + data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + else + data.sub!(/^ - #{Regexp.escape(xy)}\..*\n/, "") + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + else + unless data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + if ver.include?("-") + data.sub!(/(?<=^preview:\n\n)/, " - #{ver}\n") + else + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + end + end + if data != orig_data + diff = Diffy::Diff.new(orig_data, data) + show_diff(filename, diff) + end + end + + def self.update_releases_yml(ver, xy, ary, wwwdir, files_changed, insertions, deletions) + filename = "_data/releases.yml" + orig_data = File.read(File.join(wwwdir, filename)) + data = orig_data.dup + + date = Time.now.utc # use utc to use previous day in midnight + entry = <<eom +- version: #{ver} + date: #{date.strftime("%Y-%m-%d")} + post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ +eom + + if /\.0(?:-\w+)?\z/ =~ ver + # preview, rc, or first release + entry <<= <<eom + tag: #{RubyVersion.tag(ver)} + stats: + files_changed: #{files_changed} + insertions: #{insertions} + deletions: #{deletions} +eom + end + + entry <<= <<eom + url: + gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz + zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip + xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz + size: + gz: #{ary.find{|x|x.gz? }.size} + zip: #{ary.find{|x|x.zip?}.size} + xz: #{ary.find{|x|x.xz? }.size} + sha1: + gz: #{ary.find{|x|x.gz? }.sha1} + zip: #{ary.find{|x|x.zip?}.sha1} + xz: #{ary.find{|x|x.xz? }.sha1} + sha256: + gz: #{ary.find{|x|x.gz? }.sha256} + zip: #{ary.find{|x|x.zip?}.sha256} + xz: #{ary.find{|x|x.xz? }.sha256} + sha512: + gz: #{ary.find{|x|x.gz? }.sha512} + zip: #{ary.find{|x|x.zip?}.sha512} + xz: #{ary.find{|x|x.xz? }.sha512} +eom + + if data.include?("\n- version: #{ver}\n") + # update existing entry + data.sub!(/\n- version: #{ver}\n(^ .*\n)*\n/, "\n#{entry}\n") + elsif data.sub!(/\n# #{Regexp.escape(xy)} series\n/, "\\&\n#{entry}") + else + data.sub!(/^$/, "\n# #{xy} series\n\n#{entry}") + end + if data != orig_data + diff = Diffy::Diff.new(orig_data, data) + show_diff(filename, diff) + end + end + + def self.show_diff(filename, diff) + diff.each_with_index do |line, index| + case index + when 0 + line.sub!(/\A--- (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do + "--- a/#{filename}\t#{$2}" + end + when 1 + line.sub!(/\A\+\+\+ (.*)\t(\d+-\d+-\d+ [0-9:.]+(?: [\-+]\d+)?)\Z/) do + "+++ b/#{filename}\t#{$2}" + end + end + puts line + end + end +end + +def main + wwwdir = ARGV.shift + version = ARGV.shift + rubydir = ARGV.shift + unless rubydir + STDERR.puts "usage: format-release <dir-of-w.r-l.o> <version> <ruby-dir>" + exit + end + Tarball.parse(wwwdir, version, rubydir) +end + +main diff --git a/tool/gem-unpack.rb b/tool/gem-unpack.rb deleted file mode 100755 index 7f84126677..0000000000 --- a/tool/gem-unpack.rb +++ /dev/null @@ -1,18 +0,0 @@ -require 'rubygems' -require 'rubygems/package' - -def Gem.unpack(file, dir = nil) - policy = Gem::Security::LowSecurity - (policy = policy.dup).ui = Gem::SilentUI.new - pkg = Gem::Package.new(file) - pkg.security_policy = policy - spec = pkg.spec - target = spec.full_name - target = File.join(dir, target) if dir - pkg.extract_files target - spec_file = File.join(target, "#{spec.name}.gemspec") - open(spec_file, 'wb') do |f| - f.print spec.to_ruby - end - puts "Unpacked #{file}" -end diff --git a/tool/gen-github-release.rb b/tool/gen-github-release.rb new file mode 100755 index 0000000000..cdb66080d9 --- /dev/null +++ b/tool/gen-github-release.rb @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby + +if ARGV.size < 2 + puts "Usage: #{$0} <from version tag> <to version tag> [--no-dry-run]" + puts " : if --no-dry-run is specified, it will create a release on GitHub" + exit 1 +end + +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "octokit" + gem "faraday-retry" + gem "nokogiri" +end + +require "open-uri" + +Octokit.configure do |c| + c.access_token = ENV['GITHUB_TOKEN'] + c.auto_paginate = true + c.per_page = 100 +end + +client = Octokit::Client.new + +note = "## What's Changed\n\n" + +notes = [] + +diff = client.compare("ruby/ruby", ARGV[0], ARGV[1]) +diff[:commits].each do |c| + if c[:commit][:message] =~ /\[(Backport|Feature|Bug) #(\d*)\]/ + url = "https://bugs.ruby-lang.org/issues/#{$2}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ - Ruby master - Ruby Issue Tracking System/, "") + elsif c[:commit][:message] =~ /\(#(\d*)\)/ + url = "https://github.com/ruby/ruby/pull/#{$1}" + title = Nokogiri::HTML(URI.open(url)).title + title.gsub!(/ · ruby\/ruby · GitHub/, "") + else + next + end + notes << "* [#{title}](#{url})" +rescue OpenURI::HTTPError + puts "Error: #{url}" +end + +notes.uniq! + +note << notes.join("\n") + +note << "\n\n" +note << "Note: This list is automatically generated by tool/gen-github-release.rb. Because of this, some commits may be missing.\n\n" +note << "## Full Changelog\n\n" +note << "https://github.com/ruby/ruby/compare/#{ARGV[0]}...#{ARGV[1]}\n\n" + +if ARGV[2] == "--no-dry-run" + name = ARGV[1].gsub(/^v/, "").gsub(/_/, ".") + prerelease = ARGV[1].match?(/rc|preview/) ? true : false + client.create_release("ruby/ruby", ARGV[1], name: name, body: note, make_latest: "false", prerelease: prerelease) + puts "Created a release: https://github.com/ruby/ruby/releases/tag/#{ARGV[1]}" +else + puts note +end diff --git a/tool/gen-mailmap.rb b/tool/gen-mailmap.rb new file mode 100755 index 0000000000..0cdedf1e7b --- /dev/null +++ b/tool/gen-mailmap.rb @@ -0,0 +1,47 @@ +#!/usr/bin/env ruby + +require "open-uri" +require "yaml" + +EMAIL_YML_URL = "https://cdn.jsdelivr.net/gh/ruby/git.ruby-lang.org/config/email.yml" + +email_yml = URI(EMAIL_YML_URL).read.sub(/\A(?:#.*\n)+/, "").gsub(/^# +(.+)$/) { $1 + ": []" } + +email = YAML.load(email_yml) +YAML.load(DATA.read).each do |name, mails| + email[name] ||= [] + email[name] |= mails +end + +File.open(File.join(__dir__, "../.mailmap"), "w") do |f| + email.each do |name, mails| + canonical = "#{ name }@ruby-lang.org" + mails.delete(canonical) + svn = "#{ name }@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" + ((mails | [canonical]) + [svn]).each do |mail| + f.puts "#{ name } <#{ canonical }> <#{ mail }>" + end + end +end + +puts "You'll see canonical names (SVN account names) by the following commands:" +puts +puts " git shortlog -ce" +puts " git log --pretty=format:'%cN <%cE>'" +puts " git log --use-mailmap --pretty=full" + +__END__ +git: +- svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e +- "(no author)@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" +kazu: +- znz@users.noreply.github.com +marcandre: +- github@marc-andre.ca +mrkn: +- mrkn@users.noreply.github.com +- muraken@b2dd03c8-39d4-4d8f-98ff-823fe69b080e +naruse: +- nurse@users.noreply.github.com +tenderlove: +- tenderlove@github.com diff --git a/tool/gen_dummy_probes.rb b/tool/gen_dummy_probes.rb index c19f9d53cd..45222830f3 100755 --- a/tool/gen_dummy_probes.rb +++ b/tool/gen_dummy_probes.rb @@ -1,28 +1,32 @@ #!/usr/bin/ruby # -*- coding: us-ascii -*- +# Used to create dummy probes (as for systemtap and DTrace) by Makefiles. +# See common.mk. + text = ARGF.read -text.gsub!(/^(?!#)(.*)/){$1.upcase} # remove comments text.gsub!(%r'(?:^ *)?/\*.*?\*/\n?'m, '') -# remove the pragma declarations -text.gsub!(/^#pragma.*\n/, '') +# remove the pragma declarations and ifdefs +text.gsub!(/^#(?:pragma|include|if|endif).*\n/, '') # replace the provider section with the start of the header file -text.gsub!(/PROVIDER RUBY \{/, "#ifndef\t_PROBES_H\n#define\t_PROBES_H\n#define DTRACE_PROBES_DISABLED 1\n") +text.gsub!(/provider ruby \{/, "#ifndef\t_PROBES_H\n#define\t_PROBES_H\n#define DTRACE_PROBES_DISABLED 1\n") # finish up the #ifndef sandwich text.gsub!(/\};/, "\n#endif\t/* _PROBES_H */") -text.gsub!(/__/, '_') - -text.gsub!(/\((.+?)(?=\);)/) { - "(arg" << (0..$1.count(',')).to_a.join(", arg") +# expand probes to DTRACE macros +text.gsub!(/^ *probe ([^\(]*)\(([^\)]*)\);/) { + name, args = $1, $2 + name.upcase! + name.gsub!(/__/, '_') + args.gsub!(/(\A|, *)[^,]*\b(?=\w+(?=,|\z))/, '\1') + "#define RUBY_DTRACE_#{name}_ENABLED() 0\n" \ + "#define RUBY_DTRACE_#{name}(#{args}) do {} while (0)" } -text.gsub!(/ *PROBE ([^\(]*)(\([^\)]*\));/, "#define RUBY_DTRACE_\\1_ENABLED() 0\n#define RUBY_DTRACE_\\1\\2\ do \{ \} while\(0\)") puts "/* -*- c -*- */" print text - diff --git a/tool/gen_ruby_tapset.rb b/tool/gen_ruby_tapset.rb index c34fb88611..ae3c1eccd2 100755 --- a/tool/gen_ruby_tapset.rb +++ b/tool/gen_ruby_tapset.rb @@ -1,10 +1,11 @@ #!/usr/bin/ruby # -*- coding: us-ascii -*- +# Create a tapset for systemtap and DTrace # usage: ./ruby gen_ruby_tapset.rb --ruby-path=/path/to/ruby probes.d > output require "optparse" -def set_argument (argname, nth) +def set_argument(argname, nth) # remove C style type info argname.gsub!(/.+ (.+)/, '\1') # e.g. char *hoge -> *hoge argname.gsub!(/^\*/, '') # e.g. *filename -> filename @@ -30,7 +31,7 @@ text.gsub!(/^\};/, "") # probename() text.gsub!(/probe (.+)\( *\);/) { probe_name = $1 - probe = <<-End + <<-End probe #{probe_name} = process("ruby").provider("ruby").mark("#{probe_name}") { } @@ -42,7 +43,7 @@ text.gsub!(/ *probe (.+)\(([^,)]+)\);/) { probe_name = $1 arg1 = $2 - probe = <<-End + <<-End probe #{probe_name} = process("ruby").provider("ruby").mark("#{probe_name}") { #{set_argument(arg1, 1)} @@ -56,7 +57,7 @@ text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+)\);/) { arg1 = $2 arg2 = $3 - probe = <<-End + <<-End probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") { #{set_argument(arg1, 1)} @@ -72,7 +73,7 @@ text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+),([^,)]+)\);/) { arg2 = $3 arg3 = $4 - probe = <<-End + <<-End probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") { #{set_argument(arg1, 1)} @@ -90,7 +91,7 @@ text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+),([^,)]+),([^,)]+)\);/) { arg3 = $4 arg4 = $5 - probe = <<-End + <<-End probe #{probe_name} = process("#{ruby_path}").provider("ruby").mark("#{probe_name}") { #{set_argument(arg1, 1)} @@ -102,4 +103,3 @@ text.gsub!(/ *probe (.+)\(([^,)]+),([^,)]+),([^,)]+),([^,)]+)\);/) { } print text - diff --git a/tool/generic_erb.rb b/tool/generic_erb.rb index ac1ef7db40..b6cb51474e 100644 --- a/tool/generic_erb.rb +++ b/tool/generic_erb.rb @@ -1,55 +1,35 @@ # -*- coding: us-ascii -*- + +# Used to expand Ruby template files by common.mk, uncommon.mk and +# some Ruby extension libraries. + require 'erb' require 'optparse' -require 'fileutils' -$:.unshift(File.dirname(__FILE__)) -require 'vpath' +require_relative 'lib/output' -vpath = VPath.new -timestamp = nil -output = nil -ifchange = nil +out = Output.new source = false -color = nil +templates = [] -opt = OptionParser.new do |o| - o.on('-t', '--timestamp[=PATH]') {|v| timestamp = v || true} - o.on('-o', '--output=PATH') {|v| output = v} - o.on('-c', '--[no-]if-change') {|v| ifchange = v} +ARGV.options do |o| + o.on('-i', '--input=PATH') {|v| templates << v} o.on('-x', '--source') {source = true} - o.on('--color') {color = true} - vpath.def_options(o) + out.def_options(o) o.order!(ARGV) + templates << (ARGV.shift or abort o.to_s) if templates.empty? end -unchanged = "unchanged" -updated = "updated" -if color or (color == nil && STDOUT.tty?) - if (/\A(?:\e\[.*m|)\z/ =~ IO.popen("tput smso", "r", err: IO::NULL, &:read) rescue nil) - beg = "\e[" - colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} - reset = "#{beg}m" - unchanged = "#{beg}#{colors["pass"] || "32;1"}m#{unchanged}#{reset}" - updated = "#{beg}#{colors["fail"] || "31;1"}m#{updated}#{reset}" - end -end -template = ARGV.shift or abort opt.to_s -erb = ERB.new(File.read(template), nil, '%-') -erb.filename = template -result = source ? erb.src : erb.result -if output - if ifchange and (vpath.open(output, "rb") {|f| f.read} rescue nil) == result - puts "#{output} #{unchanged}" - else - open(output, "wb") {|f| f.print result} - puts "#{output} #{updated}" - end - if timestamp - if timestamp == true - dir, base = File.split(output) - timestamp = File.join(dir, ".time." + base) - end - FileUtils.touch(timestamp) - end -else - print result + +# Used in prelude.c.tmpl and unicode_norm_gen.tmpl +output = out.path +vpath = out.vpath + +# A hack to prevent "unused variable" warnings +output, vpath = output, vpath + +result = templates.map do |template| + erb = ERB.new(File.read(template), trim_mode: '%-') + erb.filename = template + source ? erb.src : proc{erb.result(binding)}.call end +result = result.size == 1 ? result[0] : result.join("") +out.write(result) diff --git a/tool/git-refresh b/tool/git-refresh new file mode 100755 index 0000000000..9ed7d7c76e --- /dev/null +++ b/tool/git-refresh @@ -0,0 +1,46 @@ +#!/bin/sh +set -e + +if (cd -P .) 2>/dev/null; then + CHDIR='cd -P' +else + CHDIR='cd' +fi + +quiet= +branch= + +until [ $# = 0 ]; do + case "$1" in + --) shift; break;; + -C|--directory) shift; $CHDIR "$1";; + -C*) $CHDIR `expr "$1" : '-C\(.*\)'`;; + --directory=*) $CHDIR `expr "$1" : '[^=]*=\(.*\)'`;; + -q) quiet=1;; + -b|--branch) shift; branch="$1";; + -b*) branch=`expr "$1" : '-b\(.*\)'`;; + --branch=*) branch=`expr "$1" : '[^=]*=\(.*\)'`;; + -*) echo "unknown option: $1" 1>&2; exit 1;; + *) break;; + esac + shift +done + +url="$1" +dir="$2" +shift 2 +[ x"$branch" = x ] && unset branch || : +if [ -d "$dir" ]; then + if [ x"$(git -C "$dir" describe --tags)" = x"$branch" ]; then + exit 0 # already up-to-date + fi + echo updating `expr "/$dir/" : '.*/\([^/][^/]*\)/'` ... + [ $quiet ] || set -x + $CHDIR "$dir" + ${branch+git} ${branch+fetch} ${branch+"$@"} + exec git ${branch+checkout} "${branch-pull}" "$@" +else + echo retrieving `expr "/$dir/" : '.*/\([^/][^/]*\)/'` ... + [ $quiet ] || set -x + exec git clone ${branch+--branch} ${branch+"$branch"} "$url" "$dir" "$@" +fi diff --git a/tool/gperf.sed b/tool/gperf.sed new file mode 100644 index 0000000000..9550a522b9 --- /dev/null +++ b/tool/gperf.sed @@ -0,0 +1,4 @@ +/^[a-zA-Z_0-9]*hash/,/^}/{ + s/ hval = / hval = (unsigned int)/ + s/ return / return (unsigned int)/ +} diff --git a/tool/id2token.rb b/tool/id2token.rb index 191b872c5f..cf73095842 100755 --- a/tool/id2token.rb +++ b/tool/id2token.rb @@ -1,20 +1,19 @@ #! /usr/bin/ruby -p # -*- coding: us-ascii -*- + +# Used to build the Ruby parsing code in common.mk and Ripper. + BEGIN { require 'optparse' - $:.unshift(File.dirname(__FILE__)) - require 'vpath' - vpath = VPath.new - header = nil opt = OptionParser.new do |o| - vpath.def_options(o) - header = o.order!(ARGV).shift + o.order!(ARGV) end or abort opt.opt_s TOKENS = {} - h = vpath.read(header) rescue abort("#{header} not found in #{vpath.inspect}") - h.scan(/^#define\s+RUBY_TOKEN_(\w+)\s+(\d+)/) do |token, id| + defs = File.join(File.dirname(File.dirname(__FILE__)), "defs/id.def") + ids = eval(File.read(defs), nil, defs) + ids[:token_op].each do |_id, _op, token, id| TOKENS[token] = id end diff --git a/tool/ifchange b/tool/ifchange index 4164572b76..9e4a89533c 100755 --- a/tool/ifchange +++ b/tool/ifchange @@ -1,13 +1,31 @@ #!/bin/sh # usage: ifchange target temporary +# Used in generating revision.h via Makefiles. + +help() { + cat <<HELP +usage: $0 [options] target new-file +options: + --timestamp[=file] touch timestamp file. (default: prefixed with ".time". + under the directory of the target) + --keep[=suffix] keep old file with suffix. (default: '.old') + --empty assume unchanged if the new file is empty. + --color[=always|auto|never] colorize output. +HELP +} + set -e timestamp= keepsuffix= -empty= +srcavail=f color=auto until [ $# -eq 0 ]; do case "$1" in + --) + shift + break; + ;; --timestamp) timestamp=. ;; @@ -21,7 +39,7 @@ until [ $# -eq 0 ]; do keepsuffix=`expr \( "$1" : '[^=]*=\(.*\)' \)` ;; --empty) - empty=yes + srcavail=s ;; --color) color=always @@ -29,6 +47,17 @@ until [ $# -eq 0 ]; do --color=*) color=`expr \( "$1" : '[^=]*=\(.*\)' \)` ;; + --debug) + set -x + ;; + --help) + help + exit + ;; + --*) + echo "$0: unknown option: $1" 1>&2 + exit 1 + ;; *) break ;; @@ -36,11 +65,16 @@ until [ $# -eq 0 ]; do shift done +if [ "$#" != 2 ]; then + help + exit 1 +fi + target="$1" temp="$2" if [ "$temp" = - ]; then temp="tmpdata$$.tmp~" - cat > "$temp" || exit $? + cat > "$temp" trap 'rm -f "$temp"' 0 fi @@ -48,12 +82,12 @@ msg_begin= msg_unchanged= msg_updated= msg_reset= if [ "$color" = always -o \( "$color" = auto -a -t 1 \) ]; then msg_begin="[" case "`tput smso 2>/dev/null`" in - "$msg_begin"*m|"") + "$msg_begin"*m) if [ ${TEST_COLORS:+set} ]; then - msg_unchanged=`expr ":$TEST_COLORS:" : ".*:pass=\([^:]*\):"` - msg_updated=`expr ":$TEST_COLORS:" : ".*:fail=\([^:]*\):"` + msg_unchanged=`expr ":$TEST_COLORS:" : ".*:pass=\([^:]*\):"` || : + msg_updated=`expr ":$TEST_COLORS:" : ".*:fail=\([^:]*\):"` || : fi - msg_unchanged="${msg_begin}${msg_unchanged:-32;1}m" + msg_unchanged="${msg_begin}${msg_unchanged:-32}m" msg_updated="${msg_begin}${msg_updated:-31;1}m" msg_reset="${msg_begin}m" ;; @@ -61,25 +95,25 @@ if [ "$color" = always -o \( "$color" = auto -a -t 1 \) ]; then unset msg_begin fi -if [ -f "$target" -a ! -${empty:+f}${empty:-s} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then +targetdir= +case "$target" in */*) targetdir=`dirname "$target"`;; esac +if [ -f "$target" -a ! -${srcavail} "$temp" ] || cmp "$target" "$temp" >/dev/null 2>&1; then echo "$target ${msg_unchanged}unchanged${msg_reset}" rm -f "$temp" else echo "$target ${msg_updated}updated${msg_reset}" - [ x"${keepsuffix}" = x ] || mv -f "$target" "${target}${keepsuffix}" + [ x"${targetdir}" = x -o -d "${targetdir}" ] || mkdir -p "${targetdir}" + [ x"${keepsuffix}" != x -a -f "$target" ] && mv -f "$target" "${target}${keepsuffix}" mv -f "$temp" "$target" fi if [ -n "${timestamp}" ]; then if [ x"${timestamp}" = x. ]; then - case "$target" in - */*) - timestamp=`dirname "$target"`/.time.`basename "$target"` - ;; - *) - timestamp=.time."$target" - ;; - esac + if [ x"$targetdir" = x ]; then + timestamp=.time."$target" + else + timestamp="$targetdir"/.time.`basename "$target"` + fi fi : > "$timestamp" fi diff --git a/tool/insns2vm.rb b/tool/insns2vm.rb index f518707f11..027dc4e380 100755 --- a/tool/insns2vm.rb +++ b/tool/insns2vm.rb @@ -1,15 +1,15 @@ #!ruby -require 'optparse' +# This is used by Makefile.in to generate .inc files. +# See Makefile.in for details. -Version = %w$Revision: 11626 $[1..-1] - -require "#{File.join(File.dirname(__FILE__), 'instruction')}" +require_relative 'ruby_vm/scripts/insns2vm' if $0 == __FILE__ - opts = ARGV.options - maker = RubyVM::SourceCodeGenerator.def_options(opts) - files = opts.parse! - generator = maker.call - generator.generate(files) + RubyVM::Insns2VM.router(ARGV).each do |(path, generator)| + str = generator.generate path + path.open 'wb:utf-8' do |fp| + fp.write str + end + end end diff --git a/tool/install-sh b/tool/install-sh index af97fa6af1..11e502f56d 100644 --- a/tool/install-sh +++ b/tool/install-sh @@ -1,13 +1,13 @@ #!/bin/sh -# Just only for using AC_PROG_INSTALL in configure.in. +# Just only for using AC_PROG_INSTALL in configure.ac. # See autoconf.info for more detail. cat <<EOF >&2 -Ruby uses a BSD-compatible install(1) if possible. If not, Ruby +Ruby uses a BSD-compatible install(1) if possible. If not, Ruby provides its own install(1) alternative. -This script a place holder for AC_PROG_INSTALL in configure.in. +This script is a place holder for AC_PROG_INSTALL in configure.ac. Please report a bug in Ruby to http://bugs.ruby-lang.org if you see this message. diff --git a/tool/instruction.rb b/tool/instruction.rb deleted file mode 100755 index 4f7d08c369..0000000000 --- a/tool/instruction.rb +++ /dev/null @@ -1,1346 +0,0 @@ -#!./miniruby -# -*- coding: us-ascii -*- -# -# - -require 'erb' -$:.unshift(File.dirname(__FILE__)) -require 'vpath' - -class RubyVM - class Instruction - def initialize name, opes, pops, rets, comm, body, tvars, sp_inc, - orig = self, defopes = [], type = nil, - nsc = [], psc = [[], []] - - @name = name - @opes = opes # [[type, name], ...] - @pops = pops # [[type, name], ...] - @rets = rets # [[type, name], ...] - @comm = comm # {:c => category, :e => en desc, :j => ja desc} - @body = body # '...' - - @orig = orig - @defopes = defopes - @type = type - @tvars = tvars - - @nextsc = nsc - @pushsc = psc - @sc = [] - @unifs = [] - @optimized = [] - @is_sc = false - @sp_inc = sp_inc - end - - def add_sc sci - @sc << sci - sci.set_sc - end - - attr_reader :name, :opes, :pops, :rets - attr_reader :body, :comm - attr_reader :nextsc, :pushsc - attr_reader :orig, :defopes, :type - attr_reader :sc - attr_reader :unifs, :optimized - attr_reader :is_sc - attr_reader :tvars - attr_reader :sp_inc - - def set_sc - @is_sc = true - end - - def add_unif insns - @unifs << insns - end - - def add_optimized insn - @optimized << insn - end - - def sp_increase_c_expr - if(pops.any?{|t, v| v == '...'} || - rets.any?{|t, v| v == '...'}) - # user definition - raise "no sp increase definition" if @sp_inc.nil? - ret = "int inc = 0;\n" - - @opes.each_with_index{|(t, v), i| - if (t == 'rb_num_t' && ((re = /\b#{v}\b/n) =~ @sp_inc)) || - (@defopes.any?{|t, val| re =~ val}) - ret << " int #{v} = FIX2INT(opes[#{i}]);\n" - elsif (t == 'CALL_INFO' && ((re = /\b#{v}\b/n) =~ @sp_inc)) - ret << " CALL_INFO #{v} = (CALL_INFO)(opes[#{i}]);\n" - end - } - - @defopes.each_with_index{|((t, var), val), i| - if t == 'rb_num_t' && val != '*' && /\b#{var}\b/ =~ @sp_inc - ret << " #{t} #{var} = #{val};\n" - end - } - - ret << " #{@sp_inc};\n" - ret << " return depth + inc;" - ret - else - "return depth + #{rets.size - pops.size};" - end - end - - def inspect - "#<Instruction:#{@name}>" - end - end - - class InstructionsLoader - def initialize opts = {} - @insns = [] - @insn_map = {} - - @vpath = opts[:VPATH] || File - @use_const = opts[:use_const] - @verbose = opts[:verbose] - @destdir = opts[:destdir] - - (@vm_opts = load_vm_opts).each {|k, v| - @vm_opts[k] = opts[k] if opts.key?(k) - } - - load_insns_def opts[:"insns.def"] || 'insns.def' - - load_opt_operand_def opts[:"opope.def"] || 'defs/opt_operand.def' - load_insn_unification_def opts[:"unif.def"] || 'defs/opt_insn_unif.def' - make_stackcaching_insns if vm_opt?('STACK_CACHING') - end - - attr_reader :vpath - attr_reader :destdir - - %w[use_const verbose].each do |attr| - attr_reader attr - alias_method "#{attr}?", attr - remove_method attr - end - - def [](s) - @insn_map[s.to_s] - end - - def each - @insns.each{|insn| - yield insn - } - end - - def size - @insns.size - end - - ### - private - - def vm_opt? name - @vm_opts[name] - end - - def load_vm_opts file = nil - file ||= 'vm_opts.h' - opts = {} - vpath.open(file) do |f| - f.grep(/^\#define\s+OPT_([A-Z_]+)\s+(\d+)/) do - opts[$1] = !$2.to_i.zero? - end - end - opts - end - - SKIP_COMMENT_PATTERN = Regexp.compile(Regexp.escape('/** ##skip')) - - include Enumerable - - def add_insn insn - @insns << insn - @insn_map[insn.name] = insn - end - - def make_insn name, opes, pops, rets, comm, body, sp_inc - add_insn Instruction.new(name, opes, pops, rets, comm, body, [], sp_inc) - end - - # str -> [[type, var], ...] - def parse_vars line - raise unless /\((.*?)\)/ =~ line - vars = $1.split(',') - vars.map!{|v| - if /\s*(\S+)\s+(\S+)\s*/ =~ v - type = $1 - var = $2 - elsif /\s*\.\.\.\s*/ =~ v - type = var = '...' - else - raise - end - [type, var] - } - vars - end - - def parse_comment comm - c = 'others' - j = '' - e = '' - comm.each_line{|line| - case line - when /@c (.+)/ - c = $1 - when /@e (.+)/ - e = $1 - when /@e\s*$/ - e = '' - when /@j (.+)$/ - j = $1 - when /@j\s*$/ - j = '' - end - } - { :c => c, - :e => e, - :j => j, - } - end - - def load_insns_def file - body = insn = opes = pops = rets = nil - comment = '' - - vpath.open(file) {|f| - f.instance_variable_set(:@line_no, 0) - class << f - def line_no - @line_no - end - def gets - @line_no += 1 - super - end - end - - while line = f.gets - line.chomp! - case line - - when SKIP_COMMENT_PATTERN - while line = f.gets.chomp - if /\s+\*\/$/ =~ line - break - end - end - - # collect instruction comment - when /^\/\*\*$/ - while line = f.gets - if /\s+\*\/\s*$/ =~ line - break - else - comment << line - end - end - - # start instruction body - when /^DEFINE_INSN$/ - insn = f.gets.chomp - opes = parse_vars(f.gets.chomp) - pops = parse_vars(f.gets.chomp).reverse - rets_str = f.gets.chomp - rets = parse_vars(rets_str).reverse - comment = parse_comment(comment) - insn_in = true - body = '' - - sp_inc = rets_str[%r"//\s*(.+)", 1] - - raise unless /^\{$/ =~ f.gets.chomp - line_no = f.line_no - - # end instruction body - when /^\}/ - if insn_in - body.instance_variable_set(:@line_no, line_no) - body.instance_variable_set(:@file, f.path) - insn = make_insn(insn, opes, pops, rets, comment, body, sp_inc) - insn_in = false - comment = '' - end - - else - if insn_in - body << line + "\n" - end - end - end - } - end - - ## opt op - def load_opt_operand_def file - vpath.foreach(file) {|line| - line = line.gsub(/\#.*/, '').strip - next if line.length == 0 - break if /__END__/ =~ line - /(\S+)\s+(.+)/ =~ line - insn = $1 - opts = $2 - add_opt_operand insn, opts.split(/,/).map{|e| e.strip} - } if file - end - - def label_escape label - label.gsub(/\(/, '_O_'). - gsub(/\)/, '_C_'). - gsub(/\*/, '_WC_') - end - - def add_opt_operand insn_name, opts - insn = @insn_map[insn_name] - opes = insn.opes - - if opes.size != opts.size - raise "operand size mismatch for #{insn.name} (opes: #{opes.size}, opts: #{opts.size})" - end - - ninsn = insn.name + '_OP_' + opts.map{|e| label_escape(e)}.join('_') - nopes = [] - defv = [] - - opts.each_with_index{|e, i| - if e == '*' - nopes << opes[i] - end - defv << [opes[i], e] - } - - make_insn_operand_optimized(insn, ninsn, nopes, defv) - end - - def make_insn_operand_optimized orig_insn, name, opes, defopes - comm = orig_insn.comm.dup - comm[:c] = 'optimize' - add_insn insn = Instruction.new( - name, opes, orig_insn.pops, orig_insn.rets, comm, - orig_insn.body, orig_insn.tvars, orig_insn.sp_inc, - orig_insn, defopes) - orig_insn.add_optimized insn - end - - ## insn unif - def load_insn_unification_def file - vpath.foreach(file) {|line| - line = line.gsub(/\#.*/, '').strip - next if line.length == 0 - break if /__END__/ =~ line - make_unified_insns line.split.map{|e| - raise "unknown insn: #{e}" unless @insn_map[e] - @insn_map[e] - } - } if file - end - - def all_combination sets - ret = sets.shift.map{|e| [e]} - - sets.each{|set| - prev = ret - ret = [] - prev.each{|ary| - set.each{|e| - eary = ary.dup - eary << e - ret << eary - } - } - } - ret - end - - def make_unified_insns insns - if vm_opt?('UNIFY_ALL_COMBINATION') - insn_sets = insns.map{|insn| - [insn] + insn.optimized - } - - all_combination(insn_sets).each{|insns_set| - make_unified_insn_each insns_set - } - else - make_unified_insn_each insns - end - end - - def mk_private_val vals, i, redef - vals.dup.map{|v| - # v[0] : type - # v[1] : var name - - v = v.dup - if v[0] != '...' - redef[v[1]] = v[0] - v[1] = "#{v[1]}_#{i}" - end - v - } - end - - def mk_private_val2 vals, i, redef - vals.dup.map{|v| - # v[0][0] : type - # v[0][1] : var name - # v[1] : default val - - pv = v.dup - v = pv[0] = pv[0].dup - if v[0] != '...' - redef[v[1]] = v[0] - v[1] = "#{v[1]}_#{i}" - end - pv - } - end - - def make_unified_insn_each insns - names = [] - opes = [] - pops = [] - rets = [] - comm = { - :c => 'optimize', - :e => 'unified insn', - :j => 'unified insn', - } - body = '' - passed = [] - tvars = [] - defopes = [] - sp_inc = '' - - insns.each_with_index{|insn, i| - names << insn.name - redef_vars = {} - - e_opes = mk_private_val(insn.opes, i, redef_vars) - e_pops = mk_private_val(insn.pops, i, redef_vars) - e_rets = mk_private_val(insn.rets, i, redef_vars) - # ToDo: fix it - e_defs = mk_private_val2(insn.defopes, i, redef_vars) - - passed_vars = [] - while pvar = e_pops.pop - rvar = rets.pop - if rvar - raise "unsupported unif insn: #{insns.inspect}" if rvar[0] == '...' - passed_vars << [pvar, rvar] - tvars << rvar - else - e_pops.push pvar - break - end - end - - opes.concat e_opes - pops.concat e_pops - rets.concat e_rets - defopes.concat e_defs - sp_inc += "#{insn.sp_inc}" - - body += "{ /* unif: #{i} */\n" + - passed_vars.map{|rpvars| - pv = rpvars[0] - rv = rpvars[1] - "#define #{pv[1]} #{rv[1]}" - }.join("\n") + - "\n" + - redef_vars.map{|v, type| - "#define #{v} #{v}_#{i}" - }.join("\n") + "\n" + - insn.body + - passed_vars.map{|rpvars| - "#undef #{rpvars[0][1]}" - }.join("\n") + - "\n" + - redef_vars.keys.map{|v| - "#undef #{v}" - }.join("\n") + - "\n}\n" - } - - tvars_ary = [] - tvars.each{|tvar| - unless opes.any?{|var| - var[1] == tvar[1] - } || defopes.any?{|pvar| - pvar[0][1] == tvar[1] - } - tvars_ary << tvar - end - } - add_insn insn = Instruction.new("UNIFIED_" + names.join('_'), - opes, pops, rets.reverse, comm, body, - tvars_ary, sp_inc) - insn.defopes.replace defopes - insns[0].add_unif [insn, insns] - end - - ## sc - SPECIAL_INSN_FOR_SC_AFTER = { - /\Asend/ => [:a], - /\Aend/ => [:a], - /\Ayield/ => [:a], - /\Aclassdef/ => [:a], - /\Amoduledef/ => [:a], - } - FROM_SC = [[], [:a], [:b], [:a, :b], [:b, :a]] - - def make_stackcaching_insns - pops = rets = nil - - @insns.dup.each{|insn| - opops = insn.pops - orets = insn.rets - oopes = insn.opes - ocomm = insn.comm - oname = insn.name - - after = SPECIAL_INSN_FOR_SC_AFTER.find {|k, v| k =~ oname} - - insns = [] - FROM_SC.each{|from| - name, pops, rets, pushs1, pushs2, nextsc = - *calc_stack(insn, from, after, opops, orets) - - make_insn_sc(insn, name, oopes, pops, rets, [pushs1, pushs2], nextsc) - } - } - end - - def make_insn_sc orig_insn, name, opes, pops, rets, pushs, nextsc - comm = orig_insn.comm.dup - comm[:c] = 'optimize(sc)' - - scinsn = Instruction.new( - name, opes, pops, rets, comm, - orig_insn.body, orig_insn.tvars, orig_insn.sp_inc, - orig_insn, orig_insn.defopes, :sc, nextsc, pushs) - - add_insn scinsn - orig_insn.add_sc scinsn - end - - def self.complement_name st - "#{st[0] ? st[0] : 'x'}#{st[1] ? st[1] : 'x'}" - end - - def add_stack_value st - len = st.length - if len == 0 - st[0] = :a - [nil, :a] - elsif len == 1 - if st[0] == :a - st[1] = :b - else - st[1] = :a - end - [nil, st[1]] - else - st[0], st[1] = st[1], st[0] - [st[1], st[1]] - end - end - - def calc_stack insn, ofrom, oafter, opops, orets - from = ofrom.dup - pops = opops.dup - rets = orets.dup - rest_scr = ofrom.dup - - pushs_before = [] - pushs= [] - - pops.each_with_index{|e, i| - if e[0] == '...' - pushs_before = from - from = [] - end - r = from.pop - break unless r - pops[i] = pops[i].dup << r - } - - if oafter - from = oafter - from.each_with_index{|r, i| - rets[i] = rets[i].dup << r if rets[i] - } - else - rets = rets.reverse - rets.each_with_index{|e, i| - break if e[0] == '...' - pushed, r = add_stack_value from - rets[i] = rets[i].dup << r - if pushed - if rest_scr.pop - pushs << pushed - end - - if i - 2 >= 0 - rets[i-2].pop - end - end - } - end - - if false #|| insn.name =~ /test3/ - p ofrom - p pops - p rets - p pushs_before - p pushs - p from - exit - end - - ret = ["#{insn.name}_SC_#{InstructionsLoader.complement_name(ofrom)}_#{complement_name(from)}", - pops, rets, pushs_before, pushs, from] - end - end - - class SourceCodeGenerator - def initialize insns - @insns = insns - end - - attr_reader :insns - - def generate - raise "should not reach here" - end - - def vpath - @insns.vpath - end - - def verbose? - @insns.verbose? - end - - def use_const? - @insns.use_const? - end - - def build_string - @lines = [] - yield - @lines.join("\n") - end - - EMPTY_STRING = ''.freeze - - def commit str = EMPTY_STRING - @lines << str - end - - def comment str - @lines << str if verbose? - end - - def output_path(fn) - d = @insns.destdir - fn = File.join(d, fn) if d - fn - end - end - - ################################################################### - # vm.inc - class VmBodyGenerator < SourceCodeGenerator - # vm.inc - def generate - vm_body = '' - @insns.each{|insn| - vm_body << "\n" - vm_body << make_insn_def(insn) - } - src = vpath.read('template/vm.inc.tmpl') - ERB.new(src).result(binding) - end - - def generate_from_insnname insnname - make_insn_def @insns[insnname.to_s] - end - - ####### - private - - def make_header_prepare_stack insn - comment " /* prepare stack status */" - - push_ba = insn.pushsc - raise "unsupport" if push_ba[0].size > 0 && push_ba[1].size > 0 - - n = 0 - push_ba.each {|pushs| n += pushs.length} - commit " CHECK_VM_STACK_OVERFLOW_FOR_INSN(REG_CFP, #{n});" if n > 0 - push_ba.each{|pushs| - pushs.each{|r| - commit " PUSH(SCREG(#{r}));" - } - } - end - - def make_header_operands insn - comment " /* declare and get from iseq */" - - vars = insn.opes - n = 0 - ops = [] - - vars.each_with_index{|(type, var), i| - if type == '...' - break - end - - # skip make operands when body has no reference to this operand - # TODO: really needed? - re = /\b#{var}\b/n - if re =~ insn.body or re =~ insn.sp_inc or insn.rets.any?{|t, v| re =~ v} or re =~ 'ic' or re =~ 'ci' or re =~ 'cc' - ops << " #{type} #{var} = (#{type})GET_OPERAND(#{i+1});" - end - - n += 1 - } - @opn = n - - # reverse or not? - # ops.join - commit ops.reverse - end - - def make_header_default_operands insn - vars = insn.defopes - - vars.each{|e| - next if e[1] == '*' - if use_const? - commit " const #{e[0][0]} #{e[0][1]} = #{e[1]};" - else - commit " #define #{e[0][1]} #{e[1]}" - end - } - end - - def make_footer_default_operands insn - comment " /* declare and initialize default opes */" - if use_const? - commit - else - vars = insn.defopes - - vars.each{|e| - next if e[1] == '*' - commit "#undef #{e[0][1]}" - } - end - end - - def make_header_stack_pops insn - comment " /* declare and pop from stack */" - - n = 0 - pops = [] - vars = insn.pops - vars.each_with_index{|iter, i| - type, var, r = *iter - if type == '...' - break - end - if r - pops << " #{type} #{var} = SCREG(#{r});" - else - pops << " #{type} #{var} = TOPN(#{n});" - n += 1 - end - } - @popn = n - - # reverse or not? - commit pops.reverse - end - - def make_header_temporary_vars insn - comment " /* declare temporary vars */" - - insn.tvars.each{|var| - commit " #{var[0]} #{var[1]};" - } - end - - def make_header_stack_val insn - comment "/* declare stack push val */" - - vars = insn.opes + insn.pops + insn.defopes.map{|e| e[0]} - - insn.rets.each{|var| - if vars.all?{|e| e[1] != var[1]} && var[1] != '...' - commit " #{var[0]} #{var[1]};" - end - } - end - - def make_header_analysis insn - commit " COLLECT_USAGE_INSN(BIN(#{insn.name}));" - insn.opes.each_with_index{|op, i| - commit " COLLECT_USAGE_OPERAND(BIN(#{insn.name}), #{i}, #{op[1]});" - } - end - - def make_header_pc insn - commit " ADD_PC(1+#{@opn});" - commit " PREFETCH(GET_PC());" - end - - def make_header_popn insn - comment " /* management */" - commit " POPN(#{@popn});" if @popn > 0 - end - - def make_header_debug insn - comment " /* for debug */" - commit " DEBUG_ENTER_INSN(\"#{insn.name}\");" - end - - def make_header_defines insn - commit " #define CURRENT_INSN_#{insn.name} 1" - commit " #define INSN_IS_SC() #{insn.sc ? 0 : 1}" - commit " #define INSN_LABEL(lab) LABEL_#{insn.name}_##lab" - commit " #define LABEL_IS_SC(lab) LABEL_##lab##_###{insn.sc.size == 0 ? 't' : 'f'}" - end - - def each_footer_stack_val insn - insn.rets.reverse_each{|v| - break if v[1] == '...' - yield v - } - end - - def make_footer_stack_val insn - comment " /* push stack val */" - - n = 0 - each_footer_stack_val(insn){|v| - n += 1 unless v[2] - } - commit " CHECK_VM_STACK_OVERFLOW_FOR_INSN(REG_CFP, #{n});" if n > 0 - each_footer_stack_val(insn){|v| - if v[2] - commit " SCREG(#{v[2]}) = #{v[1]};" - else - commit " PUSH(#{v[1]});" - end - } - end - - def make_footer_undefs insn - commit "#undef CURRENT_INSN_#{insn.name}" - commit "#undef INSN_IS_SC" - commit "#undef INSN_LABEL" - commit "#undef LABEL_IS_SC" - end - - def make_header insn - commit "INSN_ENTRY(#{insn.name}){" - make_header_prepare_stack insn - commit "{" - make_header_stack_val insn - make_header_default_operands insn - make_header_operands insn - make_header_stack_pops insn - make_header_temporary_vars insn - # - make_header_debug insn - make_header_pc insn - make_header_popn insn - make_header_defines insn - make_header_analysis insn - commit "{" - end - - def make_footer insn - make_footer_stack_val insn - make_footer_default_operands insn - make_footer_undefs insn - commit " END_INSN(#{insn.name});}}}" - end - - def make_insn_def insn - build_string do - make_header insn - if line = insn.body.instance_variable_get(:@line_no) - file = insn.body.instance_variable_get(:@file) - commit "#line #{line+1} \"#{file}\"" - commit insn.body - commit '#line __CURRENT_LINE__ "__CURRENT_FILE__"' - else - insn.body - end - make_footer(insn) - end - end - end - - ################################################################### - # vmtc.inc - class VmTCIncGenerator < SourceCodeGenerator - def generate - - insns_table = build_string do - @insns.each{|insn| - commit " LABEL_PTR(#{insn.name})," - } - end - - insn_end_table = build_string do - @insns.each{|insn| - commit " ELABEL_PTR(#{insn.name}),\n" - } - end - - ERB.new(vpath.read('template/vmtc.inc.tmpl')).result(binding) - end - end - - ################################################################### - # insns_info.inc - class InsnsInfoIncGenerator < SourceCodeGenerator - def generate - insns_info_inc - end - - ### - private - - def op2typesig op - case op - when /^OFFSET/ - "TS_OFFSET" - when /^rb_num_t/ - "TS_NUM" - when /^lindex_t/ - "TS_LINDEX" - when /^VALUE/ - "TS_VALUE" - when /^ID/ - "TS_ID" - when /GENTRY/ - "TS_GENTRY" - when /^IC/ - "TS_IC" - when /^CALL_INFO/ - "TS_CALLINFO" - when /^CALL_CACHE/ - "TS_CALLCACHE" - when /^\.\.\./ - "TS_VARIABLE" - when /^CDHASH/ - "TS_CDHASH" - when /^ISEQ/ - "TS_ISEQ" - when /rb_insn_func_t/ - "TS_FUNCPTR" - else - raise "unknown op type: #{op}" - end - end - - TYPE_CHARS = { - 'TS_OFFSET' => 'O', - 'TS_NUM' => 'N', - 'TS_LINDEX' => 'L', - 'TS_VALUE' => 'V', - 'TS_ID' => 'I', - 'TS_GENTRY' => 'G', - 'TS_IC' => 'K', - 'TS_CALLINFO' => 'C', - 'TS_CALLCACHE' => 'E', - 'TS_CDHASH' => 'H', - 'TS_ISEQ' => 'S', - 'TS_VARIABLE' => '.', - 'TS_FUNCPTR' => 'F', - } - - # insns_info.inc - def insns_info_inc - # insn_type_chars - insn_type_chars = TYPE_CHARS.map{|t, c| - "#define #{t} '#{c}'" - }.join("\n") - - # insn_names - insn_names = '' - @insns.each{|insn| - insn_names << " \"#{insn.name}\",\n" - } - - # operands info - operands_info = '' - operands_num_info = '' - - @insns.each{|insn| - opes = insn.opes - operands_info << ' ' - ot = opes.map{|type, var| - TYPE_CHARS.fetch(op2typesig(type)) - } - operands_info << "\"#{ot.join}\"" << ",\n" - - num = opes.size + 1 - operands_num_info << " #{num},\n" - } - - # stack num - stack_num_info = '' - @insns.each{|insn| - num = insn.rets.size - stack_num_info << " #{num},\n" - } - - # stack increase - stack_increase = '' - @insns.each{|insn| - stack_increase << <<-EOS - case BIN(#{insn.name}):{ - #{insn.sp_increase_c_expr} - } - EOS - } - ERB.new(vpath.read('template/insns_info.inc.tmpl')).result(binding) - end - end - - ################################################################### - # insns.inc - class InsnsIncGenerator < SourceCodeGenerator - def generate - i=0 - insns = build_string do - @insns.each{|insn| - commit " %-30s = %d," % ["BIN(#{insn.name})", i] - i+=1 - } - end - - ERB.new(vpath.read('template/insns.inc.tmpl')).result(binding) - end - end - - ################################################################### - # minsns.inc - class MInsnsIncGenerator < SourceCodeGenerator - def generate - i=0 - defs = build_string do - @insns.each{|insn| - commit " rb_define_const(mYarvInsns, %-30s, INT2FIX(%d));" % - ["\"I#{insn.name}\"", i] - i+=1 - } - end - ERB.new(vpath.read('template/minsns.inc.tmpl')).result(binding) - end - end - - ################################################################### - # optinsn.inc - class OptInsnIncGenerator < SourceCodeGenerator - def generate - optinsn_inc - end - - ### - private - - def val_as_type op - type = op[0][0] - val = op[1] - - case type - when /^long/, /^rb_num_t/, /^lindex_t/ - "INT2FIX(#{val})" - when /^VALUE/ - val - when /^ID/ - "INT2FIX(#{val})" - when /^ISEQ/, /^rb_insn_func_t/ - val - when /GENTRY/ - raise - when /^\.\.\./ - raise - else - raise "type: #{type}" - end - end - - # optinsn.inc - def optinsn_inc - rule = '' - opt_insns_map = Hash.new{|h, k| h[k] = []} - - @insns.each{|insn| - next if insn.defopes.size == 0 - next if insn.type == :sc - next if /^UNIFIED/ =~ insn.name.to_s - - originsn = insn.orig - opt_insns_map[originsn] << insn - } - - rule = build_string do - opt_insns_map.each{|originsn, optinsns| - commit "case BIN(#{originsn.name}):" - - optinsns.sort_by{|opti| - opti.defopes.find_all{|e| e[1] == '*'}.size - }.each{|opti| - commit " if(" - i = 0 - commit " " + opti.defopes.map{|opinfo| - i += 1 - next if opinfo[1] == '*' - "insnobj->operands[#{i-1}] == #{val_as_type(opinfo)}" - }.compact.join('&& ') - commit " ){" - idx = 0 - n = 0 - opti.defopes.each{|opinfo| - if opinfo[1] == '*' - if idx != n - commit " insnobj->operands[#{idx}] = insnobj->operands[#{n}];" - end - idx += 1 - else - # skip - end - n += 1 - } - commit " insnobj->insn_id = BIN(#{opti.name});" - commit " insnobj->operand_size = #{idx};" - commit " break;\n }\n" - } - commit " break;"; - } - end - - ERB.new(vpath.read('template/optinsn.inc.tmpl')).result(binding) - end - end - - ################################################################### - # optunifs.inc - class OptUnifsIncGenerator < SourceCodeGenerator - def generate - unif_insns_each = '' - unif_insns = '' - unif_insns_data = [] - - insns = @insns.find_all{|insn| !insn.is_sc} - insns.each{|insn| - size = insn.unifs.size - if size > 0 - insn.unifs.sort_by{|unif| -unif[1].size}.each_with_index{|unif, i| - - uni_insn, uni_insns = *unif - uni_insns = uni_insns[1..-1] - unif_insns_each << "static const int UNIFIED_#{insn.name}_#{i}[] = {" + - " BIN(#{uni_insn.name}), #{uni_insns.size + 2},\n " + - uni_insns.map{|e| "BIN(#{e.name})"}.join(", ") + "};\n" - } - else - - end - if size > 0 - unif_insns << "static const int *const UNIFIED_#{insn.name}[] = {(int *)#{size+1},\n" - unif_insns << (0...size).map{|e| " UNIFIED_#{insn.name}_#{e}"}.join(",\n") + "};\n" - unif_insns_data << " UNIFIED_#{insn.name}" - else - unif_insns_data << " 0" - end - } - unif_insns_data = "static const int *const *const unified_insns_data[] = {\n" + - unif_insns_data.join(",\n") + "};\n" - ERB.new(vpath.read('template/optunifs.inc.tmpl')).result(binding) - end - end - - ################################################################### - # opt_sc.inc - class OptSCIncGenerator < SourceCodeGenerator - def generate - sc_insn_info = [] - @insns.each{|insn| - insns = insn.sc - if insns.size > 0 - insns = ['SC_ERROR'] + insns.map{|e| " BIN(#{e.name})"} - else - insns = Array.new(6){'SC_ERROR'} - end - sc_insn_info << " {\n#{insns.join(",\n")}}" - } - sc_insn_info = sc_insn_info.join(",\n") - - sc_insn_next = @insns.map{|insn| - " SCS_#{InstructionsLoader.complement_name(insn.nextsc).upcase}" + - (verbose? ? " /* #{insn.name} */" : '') - }.join(",\n") - ERB.new(vpath.read('template/opt_sc.inc.tmpl')).result(binding) - end - end - - ################################################################### - # yasmdata.rb - class YASMDataRbGenerator < SourceCodeGenerator - def generate - insn_id2no = '' - @insns.each_with_index{|insn, i| - insn_id2no << " :#{insn.name} => #{i},\n" - } - ERB.new(vpath.read('template/yasmdata.rb.tmpl')).result(binding) - end - end - - ################################################################### - # yarvarch.* - class YARVDocGenerator < SourceCodeGenerator - def generate - - end - - def desc lang - d = '' - i = 0 - cat = nil - @insns.each{|insn| - seq = insn.opes.map{|t,v| v}.join(' ') - before = insn.pops.reverse.map{|t,v| v}.join(' ') - after = insn.rets.reverse.map{|t,v| v}.join(' ') - - if cat != insn.comm[:c] - d << "** #{insn.comm[:c]}\n\n" - cat = insn.comm[:c] - end - - d << "*** #{insn.name}\n" - d << "\n" - d << insn.comm[lang] + "\n\n" - d << ":instruction sequence: 0x%02x #{seq}\n" % i - d << ":stack: #{before} => #{after}\n\n" - i+=1 - } - d - end - - def desc_ja - d = desc :j - ERB.new(vpath.read('template/yarvarch.ja')).result(binding) - end - - def desc_en - d = desc :e - ERB.new(vpath.read('template/yarvarch.en')).result(binding) - end - end - - class SourceCodeGenerator - Files = { # codes - 'vm.inc' => VmBodyGenerator, - 'vmtc.inc' => VmTCIncGenerator, - 'insns.inc' => InsnsIncGenerator, - 'insns_info.inc' => InsnsInfoIncGenerator, - # 'minsns.inc' => MInsnsIncGenerator, - 'optinsn.inc' => OptInsnIncGenerator, - 'optunifs.inc' => OptUnifsIncGenerator, - 'opt_sc.inc' => OptSCIncGenerator, - 'yasmdata.rb' => YASMDataRbGenerator, - } - - def generate args = [] - args = Files.keys if args.empty? - args.each{|fn| - s = Files[fn].new(@insns).generate - open(output_path(fn), 'w') {|f| f.puts(s)} - } - end - - def self.def_options(opt) - opts = { - :"insns.def" => 'insns.def', - :"opope.def" => 'defs/opt_operand.def', - :"unif.def" => 'defs/opt_insn_unif.def', - } - - opt.on("-Dname", /\AOPT_(\w+)\z/, "enable VM option") {|s, v| - opts[v] = true - } - opt.on("--enable=name[,name...]", Array, - "enable VM options (without OPT_ prefix)") {|*a| - a.each {|v| opts[v] = true} - } - opt.on("-Uname", /\AOPT_(\w+)\z/, "disable VM option") {|s, v| - opts[v] = false - } - opt.on("--disable=name[,name...]", Array, - "disable VM options (without OPT_ prefix)") {|*a| - a.each {|v| opts[v] = false} - } - opt.on("-i", "--insnsdef=FILE", "--instructions-def", - "instructions definition file") {|n| - opts[:insns_def] = n - } - opt.on("-o", "--opt-operanddef=FILE", "--opt-operand-def", - "vm option: operand definition file") {|n| - opts[:opope_def] = n - } - opt.on("-u", "--opt-insnunifdef=FILE", "--opt-insn-unif-def", - "vm option: instruction unification file") {|n| - opts[:unif_def] = n - } - opt.on("-C", "--[no-]use-const", - "use consts for default operands instead of macros") {|v| - opts[:use_const] = v - } - opt.on("-d", "--destdir", "--output-directory=DIR", - "make output file underneath DIR") {|v| - opts[:destdir] = v - } - opt.on("-V", "--[no-]verbose") {|v| - opts[:verbose] = v - } - - vpath = VPath.new - vpath.def_options(opt) - - proc { - opts[:VPATH] = vpath - build opts - } - end - - def self.build opts, vpath = ['./'] - opts[:VPATH] ||= VPath.new(*vpath) - self.new InstructionsLoader.new(opts) - end - end -end - diff --git a/tool/intern_ids.rb b/tool/intern_ids.rb new file mode 100755 index 0000000000..20483195e0 --- /dev/null +++ b/tool/intern_ids.rb @@ -0,0 +1,35 @@ +#!/usr/bin/ruby -sp +# $ ruby -i tool/intern_ids.rb -prefix=_ foo.c + +BEGIN { + $prefix ||= nil + + defs = File.join(File.dirname(__dir__), "defs/id.def") + ids = eval(File.read(defs), binding, defs) + table = {} + ids[:predefined].each {|v, t| table[t] = "id#{v}"} + ids[:token_op].each {|v, t, *| table[t] = "id#{v}"} + predefined = table.keys +} + +$_.gsub!(/rb_intern\("([^\"]+)"\)/) do + token = $1 + table[token] ||= "id" + id2varname(token, $prefix) +end + +END { + predefined.each {|t| table.delete(t)} + unless table.empty? + table = table.sort_by {|t, v| v} + + # Append at the last, then edit and move appropriately. + puts + puts "==== defs" + table.each {|t, v| puts "static ID #{v};"} + puts ">>>>" + puts + puts "==== init" + table.each {|t, v|puts "#{v} = rb_intern_const(\"#{t}\");"} + puts ">>>>" + end +} diff --git a/tool/leaked-globals b/tool/leaked-globals new file mode 100755 index 0000000000..6118cd56e8 --- /dev/null +++ b/tool/leaked-globals @@ -0,0 +1,116 @@ +#!/usr/bin/ruby +require_relative 'lib/colorize' +require 'shellwords' + +until ARGV.empty? + case ARGV[0] + when /\A SYMBOL_PREFIX=(.*)/x + SYMBOL_PREFIX = $1 + when /\A NM=(.*)/x # may be multiple words + NM = $1.shellsplit + when /\A PLATFORM=(.+)?/x + platform = $1 + when /\A SOEXT=(.+)?/x + soext = $1 + when /\A SYMBOLS_IN_EMPTYLIB=(.*)/x + SYMBOLS_IN_EMPTYLIB = $1.split(" ") + when /\A EXTSTATIC=(.+)?/x + EXTSTATIC = true + else + break + end + ARGV.shift +end +SYMBOLS_IN_EMPTYLIB ||= nil +EXTSTATIC ||= false + +config = ARGV.shift +count = 0 +col = Colorize.new + +config_code = File.read(config) +REPLACE = config_code.scan(/\bAC_(?:REPLACE|CHECK)_FUNCS?\((\w+)/).flatten +# REPLACE << 'memcmp' if /\bAC_FUNC_MEMCMP\b/ =~ config_code +REPLACE.push('main', 'DllMain') +if platform and !platform.empty? + begin + h = File.read(platform) + rescue Errno::ENOENT + else + REPLACE.concat( + h .gsub(%r[/\*.*?\*/]m, " ") # delete block comments + .gsub(%r[//.*], " ") # delete oneline comments + .gsub(/^\s*#.*(?:\\\n.*)*/, "") # delete preprocessor directives + .gsub(/(?:\A|;)\K\s*typedef\s.*?;/m, "") + .scan(/\b((?!rb_|DEPRECATED|_)\w+)\s*\(.*\);/) + .flatten) + end +end +missing = File.dirname(config) + "/missing/" +ARGV.reject! do |n| + base = File.basename(n, ".*") + next true if REPLACE.include?(base) + unless (src = Dir.glob(missing + base + ".[cS]")).empty? + puts "Ignore #{col.skip(n)} because of #{src.map {|s| File.basename(s)}.join(', ')} under missing" + true + end +end + +# darwin's ld64 seems to require exception handling personality functions to be +# extern, so we allow the Rust one. +REPLACE.push("rust_eh_personality") if RUBY_PLATFORM.include?("darwin") + +print "Checking leaked global symbols..." +STDOUT.flush +if soext + soext = /\.#{soext}(?:$|\.)/ + if EXTSTATIC + ARGV.delete_if {|n| soext =~ n} + elsif ARGV.size == 1 + so = soext =~ ARGV.first + end +end + +Pipe = Struct.new(:command) do + def open(&block) IO.popen(command, &block) end + def each(&block) open {|f| f.each(&block)} end +end + +Pipe.new(NM + ARGV).each do |line| + line.chomp! + next so = nil if line.empty? + if so.nil? and line.chomp!(":") + so = soext =~ line || false + next + end + n, t, = line.split + next unless /[A-TV-Z]/ =~ t + next unless n.sub!(/^#{SYMBOL_PREFIX}/o, "") + next if n.include?(".") + next if !so and n.start_with?("___asan_") + next if !so and n.start_with?("__odr_asan_") + next if !so and n.start_with?("__retguard_") + next if !so and n.start_with?("__dtrace") + case n + when /\A(?:Init_|InitVM_|pm_|[Oo]nig|dln_|coroutine_)/ + next + when /\Aruby_static_id_/ + next unless so + when /\A(?:RUBY_|ruby_|rb_)/ + next unless so and /_(threadptr|ec)_/ =~ n + when *SYMBOLS_IN_EMPTYLIB + next + end + next if REPLACE.include?(n) + puts col.fail("leaked") if count.zero? + count += 1 + puts " #{n}" +end +case count +when 0 + puts col.pass("none") +when 1 + abort col.fail("1 un-prefixed symbol leaked") +else + abort col.fail("#{count} un-prefixed symbols leaked") +end diff --git a/tool/lib/-test-/integer.rb b/tool/lib/-test-/integer.rb new file mode 100644 index 0000000000..e60abf03a0 --- /dev/null +++ b/tool/lib/-test-/integer.rb @@ -0,0 +1,14 @@ +require 'test/unit' +require '-test-/integer.so' + +module Test::Unit::Assertions + def assert_fixnum(v, msg=nil) + assert_instance_of(Integer, v, msg) + assert_send([Bug::Integer, :fixnum?, v], msg) + end + + def assert_bignum(v, msg=nil) + assert_instance_of(Integer, v, msg) + assert_send([Bug::Integer, :bignum?, v], msg) + end +end diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb new file mode 100644 index 0000000000..daa1a1f235 --- /dev/null +++ b/tool/lib/_tmpdir.rb @@ -0,0 +1,100 @@ +template = "rubytest." + +# This path is only for tests. +# Assume the directory by these environment variables are safe. +base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| + next unless tmp and tmp.size <= 50 and File.directory?(tmp) + # On macOS, the default TMPDIR is very long, in spite of UNIX socket + # path length being limited. + # + # Also Rubygems creates its own temporary directory per tests, and + # some tests copy the full path of gemhome there. In that case, the + # path contains both temporary names twice, and can exceed path name + # limit very easily. + tmp +end +begin + tmpdir = File.join(base, template + Random.new_seed.to_s(36)[-6..-1]) + Dir.mkdir(tmpdir, 0o700) +rescue Errno::EEXIST + retry +end +# warn "tmpdir(#{tmpdir.size}) = #{tmpdir}" + +pid = $$ +END { + if pid == $$ + begin + Dir.rmdir(tmpdir) + rescue Errno::ENOENT + rescue Errno::ENOTEMPTY + require_relative "colorize" + colorize = Colorize.new + ls = Struct.new(:colorize) do + def mode_inspect(m, s) + [ + (m & 0o4 == 0 ? ?- : ?r), + (m & 0o2 == 0 ? ?- : ?w), + (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)), + ] + end + def decorate_path(path, st) + case + when st.directory? + color = "bold;blue" + type = "/" + when st.symlink? + color = "bold;cyan" + # type = "@" + when st.executable? + color = "bold;green" + type = "*" + when path.end_with?(".gem") + color = "green" + end + colorize.decorate(path, color) + (type || "") + end + def list_tree(parent, indent = "", &block) + children = Dir.children(parent).map do |child| + [child, path = File.join(parent, child), File.lstat(path)] + end + nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size + size_width = children.map {|child, path, st| st.size}.max.to_s.size + + children.each do |child, path, st| + m = st.mode + m = [ + (st.file? ? ?- : st.ftype[0]), + mode_inspect(m >> 6, (?s unless m & 04000 == 0)), + mode_inspect(m >> 3, (?s unless m & 02000 == 0)), + mode_inspect(m, (?t unless m & 01000 == 0)), + ].join("") + warn sprintf("%s* %s %*d %*d %s % s%s", + indent, m, nlink_width, st.nlink, size_width, st.size, + st.mtime.to_s, decorate_path(child, st), + (" -> " + decorate_path(File.readlink(path), File.stat(path)) if + st.symlink?)) + if st.directory? + list_tree(File.join(parent, child), indent + " ", &block) + end + yield path, st if block + end + end + end.new(colorize) + warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":" + Dir.chdir(tmpdir) do + ls.list_tree(".") do |path, st| + if st.directory? + Dir.rmdir(path) + else + File.unlink(path) + end + end + end + require "fileutils" + FileUtils.rm_rf(tmpdir) + end + end +} + +ENV["TMPDIR"] = ENV["SPEC_TEMP_DIR"] = ENV["GEM_TEST_TMPDIR"] = tmpdir diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb new file mode 100644 index 0000000000..9ad5ea220b --- /dev/null +++ b/tool/lib/bundle_env.rb @@ -0,0 +1,4 @@ +ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_PATH__SYSTEM"] = "true" +ENV["BUNDLE_WITHOUT"] = "lint doc" diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb new file mode 100644 index 0000000000..d2ed61a508 --- /dev/null +++ b/tool/lib/bundled_gem.rb @@ -0,0 +1,126 @@ +require 'fileutils' +require 'rubygems' +require 'rubygems/package' + +# This library is used by "make extract-gems" to +# unpack bundled gem files. + +module BundledGem + DEFAULT_GEMS_DEPENDENCIES = [ + "net-protocol", # net-ftp + "time", # net-ftp + "singleton", # prime + "ipaddr", # rinda + "forwardable", # prime, rinda + "strscan", # rexml + "psych" # rdoc + ] + + module_function + + def unpack(file, *rest) + pkg = Gem::Package.new(file) + prepare_test(pkg.spec, *rest) do |dir| + pkg.extract_files(dir) + FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)}) + end + puts "Unpacked #{file}" + rescue Gem::Package::FormatError, Errno::ENOENT + puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." + if file =~ /^gems\/(\w+)-/ + file = Dir.glob("gems/#{$1}-*.gem").first + end + retry + end + + def build(gemspec, version, outdir = ".", validation: true) + outdir = File.expand_path(outdir) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.load(gemfile) + abort "Failed to load #{gemspec}" unless spec + output = File.join(outdir, spec.file_name) + FileUtils.rm_rf(output) + package = Gem::Package.new(output) + package.spec = spec + package.build(validation == false) + end + end + + def copy(path, *rest) + path, n = File.split(path) + spec = Dir.chdir(path) {Gem::Specification.load(n)} or raise "Cannot load #{path}" + prepare_test(spec, *rest) do |dir| + FileUtils.rm_rf(dir) + files = spec.files.reject {|f| f.start_with?(".git")} + dirs = files.map {|f| File.dirname(f) if f.include?("/")}.uniq + FileUtils.mkdir_p(dirs.map {|d| d ? "#{dir}/#{d}" : dir}.sort_by {|d| d.count("/")}) + files.each do |f| + File.copy_stream(File.join(path, f), File.join(dir, f)) + end + end + puts "Copied #{path}" + end + + def prepare_test(spec, dir = ".") + target = spec.full_name + Gem.ensure_gem_subdirectories(dir) + gem_dir = File.join(dir, "gems", target) + yield gem_dir + spec_dir = spec.extensions.empty? ? "specifications" : File.join("gems", target) + if spec.extensions.empty? + spec.dependencies.reject! {|dep| DEFAULT_GEMS_DEPENDENCIES.include?(dep.name)} + end + File.binwrite(File.join(dir, spec_dir, "#{target}.gemspec"), spec.to_ruby) + unless spec.extensions.empty? + spec.dependencies.clear + File.binwrite(File.join(dir, spec_dir, ".bundled.#{target}.gemspec"), spec.to_ruby) + end + if spec.bindir and spec.executables + bindir = File.join(dir, "bin") + Dir.mkdir(bindir) rescue nil + spec.executables.each do |exe| + File.open(File.join(bindir, exe), "wb", 0o777) {|f| + f.print "#!ruby\n", + %[load File.realpath("../gems/#{target}/#{spec.bindir}/#{exe}", __dir__)\n] + } + end + end + FileUtils.rm_rf(Dir.glob("#{gem_dir}/.git*")) + end + + def dummy_gemspec(gemspec) + return if File.exist?(gemspec) + gemdir, gemfile = File.split(gemspec) + Dir.chdir(gemdir) do + spec = Gem::Specification.new do |s| + s.name = gemfile.chomp(".gemspec") + s.version = + File.read("lib/#{s.name}.rb")[/VERSION = "(.+?)"/, 1] || + begin File.read("lib/#{s.name}/version.rb")[/VERSION = "(.+?)"/, 1]; rescue; nil; end || + raise("cannot find the version of #{ s.name } gem") + s.authors = ["DUMMY"] + s.email = ["dummy@ruby-lang.org"] + s.files = Dir.glob("{lib,ext}/**/*").select {|f| File.file?(f)} + s.licenses = ["Ruby"] + s.description = "DO NOT USE; dummy gemspec only for test" + s.summary = "(dummy gemspec)" + end + File.write(gemfile, spec.to_ruby) + end + end + + def checkout(gemdir, repo, rev, git: $git) + return unless rev or !git or git.empty? + unless File.exist?("#{gemdir}/.git") + puts "Cloning #{repo}" + command = "#{git} clone #{repo} #{gemdir}" + system(command) or raise "failed: #{command}" + end + puts "Update #{File.basename(gemdir)} to #{rev}" + command = "#{git} fetch origin #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + command = "#{git} checkout --detach #{rev}" + system(command, chdir: gemdir) or raise "failed: #{command}" + end +end diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb new file mode 100644 index 0000000000..0904312119 --- /dev/null +++ b/tool/lib/colorize.rb @@ -0,0 +1,82 @@ +# frozen-string-literal: true + +class Colorize + # call-seq: + # Colorize.new(colorize = nil) + # Colorize.new(color: color, colors_file: colors_file) + def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color)) + @colors = @reset = nil + @color = opts && opts[:color] || color + if color or (color == nil && coloring?) + if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil) + @beg = "\e[" + colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {} + if opts and colors_file = opts[:colors_file] + begin + File.read(colors_file).scan(/(\w+)=([^:\n]*)/) do |n, c| + colors[n] ||= c + end + rescue Errno::ENOENT + end + end + @colors = colors + @reset = "#{@beg}m" + end + end + self + end + + DEFAULTS = { + # color names + "black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33", + "blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37", + "bold"=>"1", "underline"=>"4", "reverse"=>"7", + "bright_black"=>"90", "bright_red"=>"91", "bright_green"=>"92", "bright_yellow"=>"93", + "bright_blue"=>"94", "bright_magenta"=>"95", "bright_cyan"=>"96", "bright_white"=>"97", + + # abstract decorations + "pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold", + "note"=>"bright_yellow", "notice"=>"bright_yellow", "info"=>"bright_magenta", + } + + def coloring? + STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?) + end + + # colorize.decorate(str, name = color_name) + def decorate(str, name = @color) + if coloring? and color = resolve_color(name) + "#{@beg}#{color}m#{str}#{@reset}" + else + str + end + end + + def resolve_color(color = @color, seen = {}, colors = nil) + return unless @colors + color.to_s.gsub(/\b[a-z][\w ]+/) do |n| + n.gsub!(/\W+/, "_") + n.downcase! + c = seen[n] and next c + if colors + c = colors[n] + elsif (c = (tbl = @colors)[n] || (tbl = DEFAULTS)[n]) + colors = tbl + else + next n + end + seen[n] = resolve_color(c, seen, colors) + end + end + + DEFAULTS.each_key do |name| + define_method(name) {|str| + decorate(str, name) + } + end +end + +if $0 == __FILE__ + colorize = Colorize.new(ARGV.shift) + ARGV.each {|str| puts colorize.decorate(str)} +end diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb new file mode 100644 index 0000000000..e29a0e3c25 --- /dev/null +++ b/tool/lib/core_assertions.rb @@ -0,0 +1,1033 @@ +# frozen_string_literal: true + +module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + + module Unit + module Assertions + def assert_raises(*exp, &b) + raise NoMethodError, "use assert_raise", caller + end + + def _assertions= n # :nodoc: + @_assertions = n + end + + def _assertions # :nodoc: + @_assertions ||= 0 + end + + ## + # Returns a proc that will output +msg+ along with the default message. + + def message msg = nil, ending = nil, &default + proc { + ending ||= (ending_pattern = /(?<!\.)\z/; ".") + ending_pattern ||= /(?<!#{Regexp.quote(ending)})\z/ + msg = msg.call if Proc === msg + ary = [msg, (default.call if default)].compact.reject(&:empty?) + ary.map! {|str| str.to_s.sub(ending_pattern, ending) } + begin + ary.join("\n") + rescue Encoding::CompatibilityError + ary.map(&:b).join("\n") + end + } + end + end + + module CoreAssertions + require_relative 'envutil' + require 'pp' + begin + require '-test-/sanitizers' + rescue LoadError + # in test-unit-ruby-core gem + def sanitizers + nil + end + else + def sanitizers + Test::Sanitizers + end + end + module_function :sanitizers + + nil.pretty_inspect + + def mu_pp(obj) #:nodoc: + obj.pretty_inspect.chomp + end + + def assert_file + AssertFile + end + + FailDesc = proc do |status, message = "", out = ""| + now = Time.now + proc do + EnvUtil.failure_description(status, now, message, out) + end + end + + def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, + success: nil, failed: nil, gems: false, **opt) + args = Array(args).dup + unless gems.nil? + args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems") + end + stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) + desc = failed[status, message, stderr] if failed + desc ||= FailDesc[status, message, stderr] + if block_given? + raise "test_stdout ignored, use block only or without block" if test_stdout != [] + raise "test_stderr ignored, use block only or without block" if test_stderr != [] + yield(stdout.lines.map {|l| l.chomp }, stderr.lines.map {|l| l.chomp }, status) + else + all_assertions(desc) do |a| + [["stdout", test_stdout, stdout], ["stderr", test_stderr, stderr]].each do |key, exp, act| + a.for(key) do + if exp.is_a?(Regexp) + assert_match(exp, act) + elsif exp.all? {|e| String === e} + assert_equal(exp, act.lines.map {|l| l.chomp }) + else + assert_pattern_list(exp, act) + end + end + end + unless success.nil? + a.for("success?") do + if success + assert_predicate(status, :success?) + else + assert_not_predicate(status, :success?) + end + end + end + end + status + end + end + + if defined?(RubyVM::InstructionSequence) + def syntax_check(code, fname, line) + code = code.dup.force_encoding(Encoding::UTF_8) + RubyVM::InstructionSequence.compile(code, fname, fname, line) + :ok + ensure + raise if SyntaxError === $! + end + else + def syntax_check(code, fname, line) + code = code.b + code.sub!(/\A(?:\xef\xbb\xbf)?(\s*\#.*$)*(\n)?/n) { + "#$&#{"\n" if $1 && !$2}BEGIN{throw tag, :ok}\n" + } + code = code.force_encoding(Encoding::UTF_8) + catch {|tag| eval(code, binding, fname, line - 1)} + end + end + + def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) + # TODO: consider choosing some appropriate limit for RJIT and stop skipping this once it does not randomly fail + pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + # For previous versions which implemented MJIT + pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? + # ASAN has the same problem - its shadow memory greatly increases memory usage + # (plus asan has better ways to detect memory leaks than this assertion) + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled? + + require_relative 'memory_status' + raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) + + token_dump, token_re = new_test_token + envs = args.shift if Array === args and Hash === args.first + args = [ + "--disable=gems", + "-r", File.expand_path("../memory_status", __FILE__), + *args, + "-v", "-", + ] + if defined? Memory::NO_MEMORY_LEAK_ENVS then + envs ||= {} + newenvs = envs.merge(Memory::NO_MEMORY_LEAK_ENVS) { |_, _, _| break } + envs = newenvs if newenvs + end + args.unshift(envs) if envs + cmd = [ + 'END {STDERR.puts '"#{token_dump}"'"FINAL=#{Memory::Status.new}"}', + prepare, + 'STDERR.puts('"#{token_dump}"'"START=#{$initial_status = Memory::Status.new}")', + '$initial_size = $initial_status.size', + code, + 'GC.start', + ].join("\n") + _, err, status = EnvUtil.invoke_ruby(args, cmd, true, true, **opt) + before = err.sub!(/^#{token_re}START=(\{.*\})\n/, '') && Memory::Status.parse($1) + after = err.sub!(/^#{token_re}FINAL=(\{.*\})\n/, '') && Memory::Status.parse($1) + assert(status.success?, FailDesc[status, message, err]) + ([:size, (rss && :rss)] & after.members).each do |n| + b = before[n] + a = after[n] + next unless a > 0 and b > 0 + assert_operator(a.fdiv(b), :<, limit, message(message) {"#{n}: #{b} => #{a}"}) + end + rescue LoadError + pend + end + + # :call-seq: + # assert_nothing_raised( *args, &block ) + # + #If any exceptions are given as arguments, the assertion will + #fail if one of those exceptions are raised. Otherwise, the test fails + #if any exceptions are raised. + # + #The final argument may be a failure message. + # + # assert_nothing_raised RuntimeError do + # raise Exception #Assertion passes, Exception is not a RuntimeError + # end + # + # assert_nothing_raised do + # raise Exception #Assertion fails + # end + def assert_nothing_raised(*args) + self._assertions += 1 + if Module === args.last + msg = nil + else + msg = args.pop + end + begin + yield + rescue Test::Unit::PendedError, *(Test::Unit::AssertionFailedError if args.empty?) + raise + rescue *(args.empty? ? Exception : args) => e + msg = message(msg) { + "Exception raised:\n<#{mu_pp(e)}>\n""Backtrace:\n" << + Test.filter_backtrace(e.backtrace).map{|frame| " #{frame}"}.join("\n") + } + raise Test::Unit::AssertionFailedError, msg.call, e.backtrace + end + end + + def prepare_syntax_check(code, fname = nil, mesg = nil, verbose: nil) + fname ||= caller_locations(2, 1)[0] + mesg ||= fname.to_s + verbose, $VERBOSE = $VERBOSE, verbose + case + when Array === fname + fname, line = *fname + when defined?(fname.path) && defined?(fname.lineno) + fname, line = fname.path, fname.lineno + else + line = 1 + end + yield(code, fname, line, message(mesg) { + if code.end_with?("\n") + "```\n#{code}```\n" + else + "```\n#{code}\n```\n""no-newline" + end + }) + ensure + $VERBOSE = verbose + end + + def assert_valid_syntax(code, *args, **opt) + prepare_syntax_check(code, *args, **opt) do |src, fname, line, mesg| + yield if defined?(yield) + assert_nothing_raised(SyntaxError, mesg) do + assert_equal(:ok, syntax_check(src, fname, line), mesg) + end + end + end + + def assert_normal_exit(testsrc, message = '', child_env: nil, **opt) + assert_valid_syntax(testsrc, caller_locations(1, 1)[0]) + if child_env + child_env = [child_env] + else + child_env = [] + end + out, _, status = EnvUtil.invoke_ruby(child_env + %W'-W0', testsrc, true, :merge_to_stdout, **opt) + assert !status.signaled?, FailDesc[status, message, out] + end + + def assert_ruby_status(args, test_stdin="", message=nil, **opt) + out, _, status = EnvUtil.invoke_ruby(args, test_stdin, true, :merge_to_stdout, **opt) + desc = FailDesc[status, message, out] + assert(!status.signaled?, desc) + message ||= "ruby exit status is not success:" + assert(status.success?, desc) + end + + ABORT_SIGNALS = Signal.list.values_at(*%w"ILL ABRT BUS SEGV TERM") + + def separated_runner(token, out = nil) + include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + + out = out ? IO.new(out, 'w') : STDOUT + + # avoid method redefinitions + out_write = out.method(:write) + integer_to_s = Integer.instance_method(:to_s) + array_pack = Array.instance_method(:pack) + marshal_dump = Marshal.method(:dump) + assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set) + assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get) + Test::Unit::Assertions.module_eval do + @_assertions = 0 + + undef _assertions= + define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)}) + + undef _assertions + define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)}) + end + # assume Method#call and UnboundMethod#bind_call need to work as the original + + at_exit { + assertions = assertions_ivar_get.call(:@_assertions) + out_write.call <<~OUT + <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}> + #{array_pack.bind_call([marshal_dump.call($!)], 'm0')} + </error id="#{token}"> + OUT + } + if defined?(Test::Unit::Runner) + Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) + elsif defined?(Test::Unit::AutoRunner) + Test::Unit::AutoRunner.need_auto_run = false + end + end + + def assert_separately(args, file = nil, line = nil, src, ignore_stderr: nil, **opt) + unless file and line + loc, = caller_locations(1,1) + file ||= loc.path + line ||= loc.lineno + end + capture_stdout = true + unless /mswin|mingw/ =~ RbConfig::CONFIG['host_os'] + capture_stdout = false + opt[:out] = Test::Unit::Runner.output if defined?(Test::Unit::Runner) + res_p, res_c = IO.pipe + opt[:ios] = [res_c] + end + token_dump, token_re = new_test_token + src = <<eom +# -*- coding: #{line += __LINE__; src.encoding}; -*- +BEGIN { + require "test/unit";include Test::Unit::Assertions;require #{__FILE__.dump};include Test::Unit::CoreAssertions + separated_runner #{token_dump}, #{res_c&.fileno || 'nil'} +} +#{line -= __LINE__; src} +eom + args = args.dup + args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) + args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + # power_assert 3 requires ruby 3.1 or later + args << "-W:no-experimental" if RUBY_VERSION < "3.1." + stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + + if sanitizers&.lsan_enabled? + # LSAN may output messages like the following line into stderr. We should ignore it. + # ==276855==Running thread 276851 was not suspended. False leaks are possible. + # See https://github.com/google/sanitizers/issues/1479 + stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "") + end + ensure + if res_c + res_c.close + res = res_p.read + res_p.close + else + res = stdout + end + raise if $! + abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + marshal_error = nil + assert(!abort, FailDesc[status, nil, stderr]) + res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do + self._assertions += $1.to_i + res = Marshal.load($2.unpack1("m")) or next + rescue => marshal_error + ignore_stderr = nil + res = nil + else + next if SystemExit === res + if bt = res.backtrace + bt.each do |l| + l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} + end + bt.concat(caller) + else + res.set_backtrace(caller) + end + raise res + end + + # really did it succeed? + unless ignore_stderr + # the body of assert_separately must not output anything to detect error + assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) + end + assert(status.success?, FailDesc[status, "assert_separately failed", stderr]) + raise marshal_error if marshal_error + end + + # Run Ractor-related test without influencing the main test suite + def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) + omit unless defined?(Ractor) + + # https://bugs.ruby-lang.org/issues/21262 + shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value) + shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join) + + if require + require = [require] unless require.is_a?(Array) + require = require.map {|r| "require #{r.inspect}"}.join("\n") + end + + if require_relative + dir = File.dirname(caller_locations[0,1][0].absolute_path) + full_path = File.expand_path(require_relative, dir) + require = "#{require}; require #{full_path.inspect}" + end + + assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{shim_value} + #{shim_join} + #{require} + previous_verbose = $VERBOSE + $VERBOSE = nil + Ractor.new {} # trigger initial warning + $VERBOSE = previous_verbose + #{src} + RUBY + end + + # :call-seq: + # assert_throw( tag, failure_message = nil, &block ) + # + #Fails unless the given block throws +tag+, returns the caught + #value otherwise. + # + #An optional failure message may be provided as the final argument. + # + # tag = Object.new + # assert_throw(tag, "#{tag} was not thrown!") do + # throw tag + # end + def assert_throw(tag, msg = nil) + ret = catch(tag) do + begin + yield(tag) + rescue UncaughtThrowError => e + thrown = e.tag + end + msg = message(msg) { + "Expected #{mu_pp(tag)} to have been thrown"\ + "#{%Q[, not #{thrown}] if thrown}" + } + assert(false, msg) + end + assert(true) + ret + end + + # :call-seq: + # assert_raise( *args, &block ) + # + #Tests if the given block raises an exception. Acceptable exception + #types may be given as optional arguments. If the last argument is a + #String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise NameError do + # puts x #Raises NameError, so assertion succeeds + # end + def assert_raise(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + return e if exp.include? Test::Unit::PendedError + raise e + rescue Exception => e + expected = exp.any? { |ex| + if ex.instance_of? Module then + e.kind_of? ex + else + e.instance_of? ex + end + } + + assert expected, proc { + flunk(message(msg) {"#{mu_pp(exp)} exception expected, not #{mu_pp(e)}"}) + } + + return e + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} expected but nothing was raised"}) + end + end + end + + # :call-seq: + # assert_raise_with_message(exception, expected, msg = nil, &block) + # + #Tests if the given block raises an exception with the expected + #message. + # + # assert_raise_with_message(RuntimeError, "foo") do + # nil #Fails, no Exceptions are raised + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise ArgumentError, "foo" #Fails, different Exception is raised + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise "bar" #Fails, RuntimeError is raised but the message differs + # end + # + # assert_raise_with_message(RuntimeError, "foo") do + # raise "foo" #Raises RuntimeError with the message, so assertion succeeds + # end + def assert_raise_with_message(exception, expected, msg = nil, &block) + case expected + when String + assert = :assert_equal + else + assert_respond_to(expected, :===) + assert = :assert_match + end + + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield + end + m = ex.message + msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} + + if assert == :assert_equal + assert_equal(expected, m, msg) + else + msg = message(msg) { "Expected #{mu_pp expected} to match #{mu_pp m}" } + assert expected =~ m, msg + block.binding.eval("proc{|_|$~=_}").call($~) + end + ex + end + + # :call-seq: + # assert_raise_kind_of(*args, &block) + # + #Tests if the given block raises one of the given exceptions or + #sub exceptions of the given exceptions. If the last argument + #is a String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise SystemCallErr do + # Dir.chdir(__FILE__) #Raises Errno::ENOTDIR, so assertion succeeds + # end + def assert_raise_kind_of(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + raise e unless exp.include? Test::Unit::PendedError + rescue *exp => e + pass + rescue Exception => e + flunk(message(msg) {"#{mu_pp(exp)} family exception expected, not #{mu_pp(e)}"}) + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} family expected but nothing was raised"}) + end + end + e + end + + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: + + # :call-seq: + # assert(test, [failure_message]) + # + #Tests if +test+ is true. + # + #+msg+ may be a String or a Proc. If +msg+ is a String, it will be used + #as the failure message. Otherwise, the result of calling +msg+ will be + #used as the message if the assertion fails. + # + #If no +msg+ is given, a default message will be used. + # + # assert(false, "This was expected to be true") + def assert(test, *msgs) + case msg = msgs.first + when String, Proc + when nil + msgs.shift + else + bt = caller.reject { |s| s.start_with?(TEST_DIR) } + raise ArgumentError, "assertion message must be String or Proc, but #{msg.class} was given.", bt + end unless msgs.empty? + super + end + + # :call-seq: + # assert_respond_to( object, method, failure_message = nil ) + # + #Tests if the given Object responds to +method+. + # + #An optional failure message may be provided as the final argument. + # + # assert_respond_to("hello", :reverse) #Succeeds + # assert_respond_to("hello", :does_not_exist) #Fails + def assert_respond_to(obj, (meth, *priv), msg = nil) + unless priv.empty? + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}#{" privately" if priv[0]}" + } + return assert obj.respond_to?(meth, *priv), msg + end + #get rid of overcounting + if caller_locations(1, 1)[0].path.start_with?(TEST_DIR) + return if obj.respond_to?(meth) + end + super(obj, meth, msg) + end + + # :call-seq: + # assert_not_respond_to( object, method, failure_message = nil ) + # + #Tests if the given Object does not respond to +method+. + # + #An optional failure message may be provided as the final argument. + # + # assert_not_respond_to("hello", :reverse) #Fails + # assert_not_respond_to("hello", :does_not_exist) #Succeeds + def assert_not_respond_to(obj, (meth, *priv), msg = nil) + unless priv.empty? + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to not respond to ##{meth}#{" privately" if priv[0]}" + } + return assert !obj.respond_to?(meth, *priv), msg + end + #get rid of overcounting + if caller_locations(1, 1)[0].path.start_with?(TEST_DIR) + return unless obj.respond_to?(meth) + end + refute_respond_to(obj, meth, msg) + end + + # pattern_list is an array which contains regexp, string and :*. + # :* means any sequence. + # + # pattern_list is anchored. + # Use [:*, regexp/string, :*] for non-anchored match. + def assert_pattern_list(pattern_list, actual, message=nil) + rest = actual + anchored = true + pattern_list.each_with_index {|pattern, i| + if pattern == :* + anchored = false + else + if anchored + match = rest.rindex(pattern, 0) + else + match = rest.index(pattern) + end + if match + post_match = $~ ? $~.post_match : rest[match+pattern.size..-1] + else + msg = message(msg) { + expect_msg = "Expected #{mu_pp pattern}\n" + if /\n[^\n]/ =~ rest + actual_mesg = +"to match\n" + rest.scan(/.*\n+/) { + actual_mesg << ' ' << $&.inspect << "+\n" + } + actual_mesg.sub!(/\+\n\z/, '') + else + actual_mesg = "to match " + mu_pp(rest) + end + actual_mesg << "\nafter #{i} patterns with #{actual.length - rest.length} characters" + expect_msg + actual_mesg + } + assert false, msg + end + rest = post_match + anchored = true + end + } + if anchored + assert_equal("", rest) + end + end + + def assert_warning(pat, msg = nil) + result = nil + stderr = EnvUtil.with_default_internal(of: pat) { + EnvUtil.verbose_warning { + result = yield + } + } + msg = message(msg) {diff pat, stderr} + assert(pat === stderr, msg) + result + end + + def assert_warn(*args) + assert_warning(*args) {$VERBOSE = false; yield} + end + + def assert_deprecated_warning(mesg = /deprecated/, &block) + assert_warning(mesg) do + EnvUtil.deprecation_warning(&block) + end + end + + def assert_deprecated_warn(mesg = /deprecated/, &block) + assert_warn(mesg) do + EnvUtil.deprecation_warning(&block) + end + end + + class << (AssertFile = Struct.new(:failure_message).new) + include Assertions + include CoreAssertions + def assert_file_predicate(predicate, *args) + if /\Anot_/ =~ predicate + predicate = $' + neg = " not" + end + result = File.__send__(predicate, *args) + result = !result if neg + mesg = "Expected file ".dup << args.shift.inspect + mesg << "#{neg} to be #{predicate}" + mesg << mu_pp(args).sub(/\A\[(.*)\]\z/m, '(\1)') unless args.empty? + mesg << " #{failure_message}" if failure_message + assert(result, mesg) + end + alias method_missing assert_file_predicate + + def for(message) + clone.tap {|a| a.failure_message = message} + end + end + + class AllFailures + attr_reader :failures + + def initialize + @count = 0 + @failures = {} + end + + def for(key) + @count += 1 + yield key + rescue Exception => e + @failures[key] = [@count, e] + end + + def foreach(*keys) + keys.each do |key| + @count += 1 + begin + yield key + rescue Exception => e + @failures[key] = [@count, e] + end + end + end + + def message + i = 0 + total = @count.to_s + fmt = "%#{total.size}d" + @failures.map {|k, (n, v)| + v = v.message + "\n#{i+=1}. [#{fmt%n}/#{total}] Assertion for #{k.inspect}\n#{v.b.gsub(/^/, ' | ').force_encoding(v.encoding)}" + }.join("\n") + end + + def pass? + @failures.empty? + end + end + + # threads should respond to shift method. + # Array can be used. + def assert_join_threads(threads, message = nil) + errs = [] + values = [] + while th = threads.shift + begin + values << th.value + rescue Exception + errs << [th, $!] + th = nil + end + end + values + ensure + if th&.alive? + th.raise(Timeout::Error.new) + th.join rescue errs << [th, $!] + end + if !errs.empty? + msg = "exceptions on #{errs.length} threads:\n" + + errs.map {|t, err| + "#{t.inspect}:\n" + + (err.respond_to?(:full_message) ? err.full_message(highlight: false, order: :top) : err.message) + }.join("\n---\n") + if message + msg = "#{message}\n#{msg}" + end + raise Test::Unit::AssertionFailedError, msg + end + end + + def assert_all?(obj, m = nil, &blk) + failed = [] + obj.each do |*a, &b| + unless blk.call(*a, &b) + failed << (a.size > 1 ? a : a[0]) + end + end + assert(failed.empty?, message(m) {failed.pretty_inspect}) + end + + def assert_all_assertions(msg = nil) + all = AllFailures.new + yield all + ensure + assert(all.pass?, message(msg) {all.message.chomp(".")}) + end + alias all_assertions assert_all_assertions + + def assert_all_assertions_foreach(msg = nil, *keys, &block) + all = AllFailures.new + all.foreach(*keys, &block) + ensure + assert(all.pass?, message(msg) {all.message.chomp(".")}) + end + alias all_assertions_foreach assert_all_assertions_foreach + + %w[ + CLOCK_THREAD_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID + CLOCK_MONOTONIC + ].find do |c| + if Process.const_defined?(c) + [c.to_sym, Process.const_get(c)].find do |clk| + begin + Process.clock_gettime(clk) + rescue + # Constants may be defined but not implemented, e.g., mingw. + else + unless Process.clock_getres(clk) < 1.0e-03 + next # needs msec precision + end + PERFORMANCE_CLOCK = clk + end + end + end + end + + # Expect +seq+ to respond to +first+ and +each+ methods, e.g., + # Array, Range, Enumerator::ArithmeticSequence and other + # Enumerable-s, and each elements should be size factors. + # + # :yield: each elements of +seq+. + def assert_linear_performance(seq, rehearsal: nil, pre: ->(n) {n}) + pend "No PERFORMANCE_CLOCK found" unless defined?(PERFORMANCE_CLOCK) + + # Timeout testing generally doesn't work when RJIT compilation happens. + rjit_enabled = defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? + measure = proc do |arg, message| + st = Process.clock_gettime(PERFORMANCE_CLOCK) + yield(*arg) + t = (Process.clock_gettime(PERFORMANCE_CLOCK) - st) + assert_operator 0, :<=, t, message unless rjit_enabled + t + end + + first = seq.first + *arg = pre.call(first) + times = (0..(rehearsal || (2 * first))).map do + measure[arg, "rehearsal"].nonzero? + end + times.compact! + tmin, tmax = times.minmax + + # safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small) + tbase = 10 * tmax * [(tmax / tmin) ** 2 / 4, 1].max + info = "(tmin: #{tmin}, tmax: #{tmax}, tbase: #{tbase})" + + seq.each do |i| + next if i == first + t = tbase * i.fdiv(first) + *arg = pre.call(i) + message = "[#{i}]: in #{t}s #{info}" + Timeout.timeout(t, Timeout::Error, message) do + measure[arg, message] + end + end + end + + def diff(exp, act) + require 'pp' + q = PP.new(+"") + q.guard_inspect_key do + q.group(2, "expected: ") do + q.pp exp + end + q.text q.newline + q.group(2, "actual: ") do + q.pp act + end + q.flush + end + q.output + end + + def new_test_token + token = "\e[7;1m#{$$.to_s}:#{Time.now.strftime('%s.%L')}:#{rand(0x10000).to_s(16)}:\e[m" + return token.dump, Regexp.quote(token) + end + + # Platform predicates + + def self.mswin? + defined?(@mswin) ? @mswin : @mswin = RUBY_PLATFORM.include?('mswin') + end + private def mswin? + CoreAssertions.mswin? + end + + def self.mingw? + defined?(@mingw) ? @mingw : @mingw = RUBY_PLATFORM.include?('mingw') + end + private def mingw? + CoreAssertions.mingw? + end + + module_function def windows? + mswin? or mingw? + end + + def self.version_compare(expected, actual) + expected.zip(actual).each {|e, a| z = (e <=> a); return z if z.nonzero?} + 0 + end + + def self.version_match?(expected, actual) + if !actual + false + elsif expected.empty? + true + elsif expected.size == 1 and Range === (range = expected.first) + b, e = range.begin, range.end + return false if b and (c = version_compare(Array(b), actual)) > 0 + return false if e and (c = version_compare(Array(e), actual)) < 0 + return false if e and range.exclude_end? and c == 0 + true + else + version_compare(expected, actual).zero? + end + end + + def self.linux?(*ver) + unless defined?(@linux) + @linux = RUBY_PLATFORM.include?('linux') && `uname -r`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @linux + end + private def linux?(*ver) + CoreAssertions.linux?(*ver) + end + + def self.glibc?(*ver) + unless defined?(@glibc) + libc = `/usr/bin/ldd /bin/sh`[/^\s*libc.*=> *\K\S*/] + if libc and /version (\d+)\.(\d+)\.$/ =~ IO.popen([libc], &:read)[] + @glibc = [$1.to_i, $2.to_i] + else + @glibc = false + end + end + version_match? ver, @glibc + end + private def glibc?(*ver) + CoreAssertions.glibc?(*ver) + end + + def self.macos?(*ver) + unless defined?(@macos) + @macos = RUBY_PLATFORM.include?('darwin') && `sw_vers -productVersion`.scan(/\d+/).map(&:to_i) + end + version_match? ver, @macos + end + private def macos?(*ver) + CoreAssertions.macos?(*ver) + end + end + end +end diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb new file mode 100644 index 0000000000..56b420a546 --- /dev/null +++ b/tool/lib/dump.gdb @@ -0,0 +1,17 @@ +set height 0 +set width 0 +set confirm off + +echo \n>>> Threads\n\n +info threads + +echo \n>>> Machine level backtrace\n\n +thread apply all info stack full + +echo \n>>> Dump Ruby level backtrace (if possible)\n\n +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +echo ">>> Finish\n" +detach +quit diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb new file mode 100644 index 0000000000..ed9cb89010 --- /dev/null +++ b/tool/lib/dump.lldb @@ -0,0 +1,13 @@ +script print("\n>>> Threads\n\n") +thread list + +script print("\n>>> Machine level backtrace\n\n") +thread backtrace all + +script print("\n>>> Dump Ruby level backtrace (if possible)\n\n") +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +script print(">>> Finish\n") +detach +quit diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb new file mode 100644 index 0000000000..6089605056 --- /dev/null +++ b/tool/lib/envutil.rb @@ -0,0 +1,497 @@ +# -*- coding: us-ascii -*- +# frozen_string_literal: true +require "open3" +require "timeout" +require_relative "find_executable" +begin + require 'rbconfig' +rescue LoadError +end +begin + require "rbconfig/sizeof" +rescue LoadError +end + +module EnvUtil + def rubybin + if ruby = ENV["RUBY"] + ruby + elsif defined?(RbConfig.ruby) + RbConfig.ruby + else + ruby = "ruby" + exeext = RbConfig::CONFIG["EXEEXT"] + rubyexe = (ruby + exeext if exeext and !exeext.empty?) + 3.times do + if File.exist? ruby and File.executable? ruby and !File.directory? ruby + return File.expand_path(ruby) + end + if rubyexe and File.exist? rubyexe and File.executable? rubyexe + return File.expand_path(rubyexe) + end + ruby = File.join("..", ruby) + end + "ruby" + end + end + module_function :rubybin + + LANG_ENVS = %w"LANG LC_ALL LC_CTYPE" + + DEFAULT_SIGNALS = Signal.list + DEFAULT_SIGNALS.delete("TERM") if /mswin|mingw/ =~ RUBY_PLATFORM + + RUBYLIB = ENV["RUBYLIB"] + + class << self + attr_accessor :timeout_scale + attr_reader :original_internal_encoding, :original_external_encoding, + :original_verbose, :original_warning + + def capture_global_values + @original_internal_encoding = Encoding.default_internal + @original_external_encoding = Encoding.default_external + @original_verbose = $VERBOSE + @original_warning = + if defined?(Warning.categories) + Warning.categories.to_h {|i| [i, Warning[i]]} + elsif defined?(Warning.[]) # 2.7+ + %i[deprecated experimental performance].to_h do |i| + [i, begin Warning[i]; rescue ArgumentError; end] + end.compact + end + end + end + + def apply_timeout_scale(t) + if scale = EnvUtil.timeout_scale + t * scale + else + t + end + end + module_function :apply_timeout_scale + + def timeout(sec, klass = nil, message = nil, &blk) + return yield(sec) if sec == nil or sec.zero? + sec = apply_timeout_scale(sec) + Timeout.timeout(sec, klass, message, &blk) + end + module_function :timeout + + class Debugger + @list = [] + + attr_accessor :name + + def self.register(name, &block) + @list << new(name, &block) + end + + def initialize(name, &block) + @name = name + instance_eval(&block) + end + + def usable?; false; end + + def start(pid, *args) end + + def dump(pid, timeout: 60, reprieve: timeout&.div(4)) + dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err) + rescue Errno::ENOENT + return + else + return unless dpid + [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig| + begin + return EnvUtil.timeout(t) {Process.wait(dpid)} + rescue Timeout::Error + Process.kill(sig, dpid) + end + end + true + end + + # sudo -n: --non-interactive + PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM) + + def spawn(*args, **opts) + super(*PRECOMMAND, *args, **opts) + end + + register("gdb") do + class << self + def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end + def start(pid, *args, **opts) + spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts) + end + def command_file(file) "--command=#{file}"; end + end + end + + register("lldb") do + class << self + def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end + def start(pid, *args, **opts) + spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts) + end + def command_file(file) ["--source", file]; end + end + end + + def self.search + @debugger ||= @list.find(&:usable?) + end + end + + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) + reprieve = apply_timeout_scale(reprieve) if reprieve + + signals = Array(signal).select do |sig| + DEFAULT_SIGNALS[sig.to_s] or + DEFAULT_SIGNALS[Signal.signame(sig)] rescue false + end + signals |= [:ABRT, :KILL] + case pgroup + when 0, true + pgroup = -pid + when nil, false + pgroup = pid + end + + dumped = false + while signal = signals.shift + + if !dumped and [:ABRT, :KILL].include?(signal) + Debugger.search&.dump(pid) + dumped = true + end + + begin + Process.kill signal, pgroup + rescue Errno::EINVAL + next + rescue Errno::ESRCH + break + end + if signals.empty? or !reprieve + Process.wait(pid) + else + begin + Timeout.timeout(reprieve) {Process.wait(pid)} + rescue Timeout::Error + else + break + end + end + end + $? + end + module_function :terminate + + def invoke_ruby(args, stdin_data = "", capture_stdout = false, capture_stderr = false, + encoding: nil, timeout: 10, reprieve: 1, timeout_error: Timeout::Error, + stdout_filter: nil, stderr_filter: nil, ios: nil, + signal: :TERM, + rubybin: EnvUtil.rubybin, precommand: nil, + **opt) + timeout = apply_timeout_scale(timeout) + + in_c, in_p = IO.pipe + out_p, out_c = IO.pipe if capture_stdout + err_p, err_c = IO.pipe if capture_stderr && capture_stderr != :merge_to_stdout + opt[:in] = in_c + opt[:out] = out_c if capture_stdout + opt[:err] = capture_stderr == :merge_to_stdout ? out_c : err_c if capture_stderr + if encoding + out_p.set_encoding(encoding) if out_p + err_p.set_encoding(encoding) if err_p + end + ios.each {|i, o = i|opt[i] = o} if ios + + c = "C" + child_env = {} + LANG_ENVS.each {|lc| child_env[lc] = c} + if Array === args and Hash === args.first + child_env.update(args.shift) + end + if RUBYLIB and lib = child_env["RUBYLIB"] + child_env["RUBYLIB"] = [lib, RUBYLIB].join(File::PATH_SEPARATOR) + end + + # remain env + %w(ASAN_OPTIONS RUBY_ON_BUG).each{|name| + child_env[name] = ENV[name] if !child_env.key?(name) and ENV.key?(name) + } + + args = [args] if args.kind_of?(String) + # use the same parser as current ruby + if (args.none? { |arg| arg.start_with?("--parser=") } and + /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read)) + args = ["--parser=#{current_parser}"] + args + end + pid = spawn(child_env, *precommand, rubybin, *args, opt) + in_c.close + out_c&.close + out_c = nil + err_c&.close + err_c = nil + if block_given? + return yield in_p, out_p, err_p, pid + else + th_stdout = Thread.new { out_p.read } if capture_stdout + th_stderr = Thread.new { err_p.read } if capture_stderr && capture_stderr != :merge_to_stdout + in_p.write stdin_data.to_str unless stdin_data.empty? + in_p.close + if (!th_stdout || th_stdout.join(timeout)) && (!th_stderr || th_stderr.join(timeout)) + timeout_error = nil + else + status = terminate(pid, signal, opt[:pgroup], reprieve) + terminated = Time.now + end + stdout = th_stdout.value if capture_stdout + stderr = th_stderr.value if capture_stderr && capture_stderr != :merge_to_stdout + out_p.close if capture_stdout + err_p.close if capture_stderr && capture_stderr != :merge_to_stdout + status ||= Process.wait2(pid)[1] + stdout = stdout_filter.call(stdout) if stdout_filter + stderr = stderr_filter.call(stderr) if stderr_filter + if timeout_error + bt = caller_locations + msg = "execution of #{bt.shift.label} expired timeout (#{timeout} sec)" + msg = failure_description(status, terminated, msg, [stdout, stderr].join("\n")) + raise timeout_error, msg, bt.map(&:to_s) + end + return stdout, stderr, status + end + ensure + [th_stdout, th_stderr].each do |th| + th.kill if th + end + [in_c, in_p, out_c, out_p, err_c, err_p].each do |io| + io&.close + end + [th_stdout, th_stderr].each do |th| + th.join if th + end + end + module_function :invoke_ruby + + def current_parser + features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}] + features&.split&.include?("+PRISM") ? "prism" : "parse.y" + end + module_function :current_parser + + def verbose_warning + class << (stderr = "".dup) + alias write concat + def flush; end + end + stderr, $stderr = $stderr, stderr + $VERBOSE = true + yield stderr + return $stderr + ensure + stderr, $stderr = $stderr, stderr + $VERBOSE = EnvUtil.original_verbose + EnvUtil.original_warning&.each {|i, v| Warning[i] = v} + end + module_function :verbose_warning + + if defined?(Warning.[]=) + def deprecation_warning + previous_deprecated = Warning[:deprecated] + Warning[:deprecated] = true + yield + ensure + Warning[:deprecated] = previous_deprecated + end + else + def deprecation_warning + yield + end + end + module_function :deprecation_warning + + def default_warning + $VERBOSE = false + yield + ensure + $VERBOSE = EnvUtil.original_verbose + end + module_function :default_warning + + def suppress_warning + $VERBOSE = nil + yield + ensure + $VERBOSE = EnvUtil.original_verbose + end + module_function :suppress_warning + + def under_gc_stress(stress = true) + stress, GC.stress = GC.stress, stress + yield + ensure + GC.stress = stress + end + module_function :under_gc_stress + + def under_gc_compact_stress(val = :empty, &block) + raise "compaction doesn't work well on s390x. Omit the test in the caller." if RUBY_PLATFORM =~ /s390x/ # https://github.com/ruby/ruby/pull/5077 + + if GC.respond_to?(:auto_compact) + auto_compact = GC.auto_compact + GC.auto_compact = val + end + + under_gc_stress(&block) + ensure + GC.auto_compact = auto_compact if GC.respond_to?(:auto_compact) + end + module_function :under_gc_compact_stress + + def without_gc + prev_disabled = GC.disable + yield + ensure + GC.enable unless prev_disabled + end + module_function :without_gc + + def with_default_external(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) + suppress_warning { Encoding.default_external = enc } + yield + ensure + suppress_warning { Encoding.default_external = EnvUtil.original_external_encoding } + end + module_function :with_default_external + + def with_default_internal(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) + suppress_warning { Encoding.default_internal = enc } + yield + ensure + suppress_warning { Encoding.default_internal = EnvUtil.original_internal_encoding } + end + module_function :with_default_internal + + def labeled_module(name, &block) + Module.new do + singleton_class.class_eval { + define_method(:to_s) {name} + alias inspect to_s + alias name to_s + } + class_eval(&block) if block + end + end + module_function :labeled_module + + def labeled_class(name, superclass = Object, &block) + Class.new(superclass) do + singleton_class.class_eval { + define_method(:to_s) {name} + alias inspect to_s + alias name to_s + } + class_eval(&block) if block + end + end + module_function :labeled_class + + if /darwin/ =~ RUBY_PLATFORM + DIAGNOSTIC_REPORTS_PATH = File.expand_path("~/Library/Logs/DiagnosticReports") + DIAGNOSTIC_REPORTS_TIMEFORMAT = '%Y-%m-%d-%H%M%S' + @ruby_install_name = RbConfig::CONFIG['RUBY_INSTALL_NAME'] + + def self.diagnostic_reports(signame, pid, now) + return unless %w[ABRT QUIT SEGV ILL TRAP].include?(signame) + cmd = File.basename(rubybin) + cmd = @ruby_install_name if "ruby-runner#{RbConfig::CONFIG["EXEEXT"]}" == cmd + path = DIAGNOSTIC_REPORTS_PATH + timeformat = DIAGNOSTIC_REPORTS_TIMEFORMAT + pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.{crash,ips}" + first = true + 30.times do + first ? (first = false) : sleep(0.1) + Dir.glob(pat) do |name| + log = File.read(name) rescue next + case name + when /\.crash\z/ + if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log + File.unlink(name) + File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil + return log + end + when /\.ips\z/ + if /^ *"pid" *: *#{pid},/ =~ log + File.unlink(name) + return log + end + end + end + end + nil + end + else + def self.diagnostic_reports(signame, pid, now) + end + end + + def self.failure_description(status, now, message = "", out = "") + pid = status.pid + if signo = status.termsig + signame = Signal.signame(signo) + sigdesc = "signal #{signo}" + end + log = diagnostic_reports(signame, pid, now) + if signame + sigdesc = "SIG#{signame} (#{sigdesc})" + end + if status.coredump? + sigdesc = "#{sigdesc} (core dumped)" + end + full_message = ''.dup + message = message.call if Proc === message + if message and !message.empty? + full_message << message << "\n" + end + full_message << "pid #{pid}" + full_message << " exit #{status.exitstatus}" if status.exited? + full_message << " killed by #{sigdesc}" if sigdesc + if out and !out.empty? + full_message << "\n" << out.b.gsub(/^/, '| ') + full_message.sub!(/(?<!\n)\z/, "\n") + end + if log + full_message << "Diagnostic reports:\n" << log.b.gsub(/^/, '| ') + end + full_message + end + + def self.gc_stress_to_class? + unless defined?(@gc_stress_to_class) + _, _, status = invoke_ruby(["-e""exit GC.respond_to?(:add_stress_to_class)"]) + @gc_stress_to_class = status.success? + end + @gc_stress_to_class + end +end + +if defined?(RbConfig) + module RbConfig + @ruby = EnvUtil.rubybin + class << self + undef ruby if method_defined?(:ruby) + attr_reader :ruby + end + dir = File.dirname(ruby) + CONFIG['bindir'] = dir + end +end + +EnvUtil.capture_global_values diff --git a/tool/lib/find_executable.rb b/tool/lib/find_executable.rb new file mode 100644 index 0000000000..89c6fb8f3b --- /dev/null +++ b/tool/lib/find_executable.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +require "rbconfig" + +module EnvUtil + def find_executable(cmd, *args) + exts = RbConfig::CONFIG["EXECUTABLE_EXTS"].split | [RbConfig::CONFIG["EXEEXT"]] + ENV["PATH"].split(File::PATH_SEPARATOR).each do |path| + next if path.empty? + path = File.join(path, cmd) + exts.each do |ext| + cmdline = [path + ext, *args] + begin + return cmdline if yield(IO.popen(cmdline, "r", err: [:child, :out], &:read)) + rescue + next + end + end + end + nil + end + module_function :find_executable +end diff --git a/tool/lib/gc_checker.rb b/tool/lib/gc_checker.rb new file mode 100644 index 0000000000..719da8cac0 --- /dev/null +++ b/tool/lib/gc_checker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module GCDisabledChecker + def before_setup + if @__gc_disabled__ = GC.enable # return true if GC is disabled + GC.disable + end + + super + end + + def after_teardown + super + + disabled = GC.enable + GC.disable if @__gc_disabled__ + + if @__gc_disabled__ != disabled + label = { + true => 'disabled', + false => 'enabled', + } + raise "GC was #{label[@__gc_disabled__]}, but is #{label[disabled]} after the test." + end + end +end + +module GCCompactChecker + def after_teardown + super + GC.compact + end +end + +Test::Unit::TestCase.include GCDisabledChecker +Test::Unit::TestCase.include GCCompactChecker if ENV['RUBY_TEST_GC_COMPACT'] diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb new file mode 100644 index 0000000000..1893e07657 --- /dev/null +++ b/tool/lib/gem_env.rb @@ -0,0 +1 @@ +ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__) diff --git a/tool/lib/iseq_loader_checker.rb b/tool/lib/iseq_loader_checker.rb new file mode 100644 index 0000000000..73784f8450 --- /dev/null +++ b/tool/lib/iseq_loader_checker.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +begin + require '-test-/iseq_load/iseq_load' +rescue LoadError +end +require 'tempfile' + +class RubyVM::InstructionSequence + def disasm_if_possible + begin + self.disasm + rescue Encoding::CompatibilityError, EncodingError, SecurityError + nil + end + end + + def self.compare_dump_and_load i1, dumper, loader + dump = dumper.call(i1) + return i1 unless dump + i2 = loader.call(dump) + + # compare disassembled result + d1 = i1.disasm_if_possible + d2 = i2.disasm_if_possible + + if d1 != d2 + STDERR.puts "expected:" + STDERR.puts d1 + STDERR.puts "actual:" + STDERR.puts d2 + + t1 = Tempfile.new("expected"); t1.puts d1; t1.close + t2 = Tempfile.new("actual"); t2.puts d2; t2.close + system("diff -u #{t1.path} #{t2.path}") # use diff if available + exit(1) + end + i2 + end + + opt = ENV['RUBY_ISEQ_DUMP_DEBUG'] + + if opt && caller.any?{|e| /test\/runner\.rb/ =~ e} + puts "RUBY_ISEQ_DUMP_DEBUG = #{opt}" if opt + end + + CHECK_TO_A = 'to_a' == opt + CHECK_TO_BINARY = 'to_binary' == opt + + def self.translate i1 + # check to_a/load_iseq + compare_dump_and_load(i1, + proc{|iseq| + ary = iseq.to_a + ary[9] == :top ? ary : nil + }, + proc{|ary| + RubyVM::InstructionSequence.iseq_load(ary) + }) if CHECK_TO_A && defined?(RubyVM::InstructionSequence.iseq_load) + + # check to_binary + i2_bin = compare_dump_and_load(i1, + proc{|iseq| + begin + iseq.to_binary + rescue RuntimeError # not a toplevel + # STDERR.puts [:failed, $!, iseq].inspect + nil + end + }, + proc{|bin| + iseq = RubyVM::InstructionSequence.load_from_binary(bin) + # STDERR.puts iseq.inspect + iseq + }) if CHECK_TO_BINARY + # return value + i2_bin if CHECK_TO_BINARY + end if CHECK_TO_A || CHECK_TO_BINARY + + if opt == "prism" + # If RUBY_ISEQ_DUMP_DEBUG is "prism", we'll set up + # InstructionSequence.load_iseq to intercept loading filepaths to compile + # using prism. + def self.load_iseq(filepath) + RubyVM::InstructionSequence.compile_file_prism(filepath) + end + end +end + +#require_relative 'x'; exit(1) diff --git a/tool/jisx0208.rb b/tool/lib/jisx0208.rb index 921f574816..30185fb81b 100644 --- a/tool/jisx0208.rb +++ b/tool/lib/jisx0208.rb @@ -1,3 +1,5 @@ +# Library used by tools/enc-emoji-citrus-gen.rb + module JISX0208 class Char class << self diff --git a/tool/lib/launchable.rb b/tool/lib/launchable.rb new file mode 100644 index 0000000000..38f4fe92b3 --- /dev/null +++ b/tool/lib/launchable.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +require 'json' +require 'uri' + +module Launchable + ## + # JsonStreamWriter writes a JSON file using a stream. + # By utilizing a stream, we can minimize memory usage, especially for large files. + class JsonStreamWriter + def initialize(path) + @file = File.open(path, "w") + @file.write("{") + @indent_level = 0 + @is_first_key_val = true + @is_first_obj = true + write_new_line + end + + def write_object obj + if @is_first_obj + @is_first_obj = false + else + write_comma + write_new_line + end + @indent_level += 1 + @file.write(to_json_str(obj)) + @indent_level -= 1 + @is_first_key_val = true + # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified. + # { + # "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds", + # "status": "TEST_PASSED", + # "duration": 2.7e-05, + # "createdAt": "2024-02-09 12:21:07 +0000", + # "stderr": null, + # "stdout": null + # }: null <- here + # }, + # To prevent this, IO#flush is called here. + @file.flush + end + + def write_array(key) + @indent_level += 1 + @file.write(to_json_str(key)) + write_colon + @file.write(" ", "[") + write_new_line + end + + def close + return if @file.closed? + close_array + @indent_level -= 1 + write_new_line + @file.write("}", "\n") + @file.flush + @file.close + end + + private + def to_json_str(obj) + json = JSON.pretty_generate(obj) + json.gsub(/^/, ' ' * (2 * @indent_level)) + end + + def write_indent + @file.write(" " * 2 * @indent_level) + end + + def write_new_line + @file.write("\n") + end + + def write_comma + @file.write(',') + end + + def write_colon + @file.write(":") + end + + def close_array + write_new_line + write_indent + @file.write("]") + @indent_level -= 1 + end + end +end diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb new file mode 100644 index 0000000000..69df9a64b8 --- /dev/null +++ b/tool/lib/leakchecker.rb @@ -0,0 +1,321 @@ +# frozen_string_literal: true +class LeakChecker + @@try_lsof = nil # not-tried-yet + + def initialize + @fd_info = find_fds + @@skip = false + @tempfile_info = find_tempfiles + @thread_info = find_threads + @env_info = find_env + @encoding_info = find_encodings + @old_verbose = $VERBOSE + @old_warning_flags = find_warning_flags + end + + def check(test_name) + if /i386-solaris/ =~ RUBY_PLATFORM && /TestGem/ =~ test_name + GC.verify_internal_consistency + end + + leaks = [ + check_fd_leak(test_name), + check_thread_leak(test_name), + check_tempfile_leak(test_name), + check_env(test_name), + check_encodings(test_name), + check_verbose(test_name), + check_warning_flags(test_name), + ] + GC.start if leaks.any? + end + + def check_verbose test_name + puts "#{test_name}: $VERBOSE == #{$VERBOSE}" unless @old_verbose == $VERBOSE + end + + def find_fds + if IO.respond_to?(:console) and (m = IO.method(:console)).arity.nonzero? + m[:close] + end + %w"/proc/self/fd /dev/fd".each do |fd_dir| + if File.directory?(fd_dir) + fds = Dir.open(fd_dir) {|d| + a = d.grep(/\A\d+\z/, &:to_i) + if d.respond_to? :fileno + a -= [d.fileno] + end + a + } + return fds.sort + end + end + [] + end + + def check_fd_leak(test_name) + leaked = false + live1 = @fd_info + live2 = find_fds + fd_closed = live1 - live2 + if !fd_closed.empty? + fd_closed.each {|fd| + puts "Closed file descriptor: #{test_name}: #{fd}" + } + end + fd_leaked = live2 - live1 + if !@@skip && !fd_leaked.empty? + leaked = true + h = {} + ObjectSpace.each_object(IO) {|io| + inspect = io.inspect + begin + autoclose = io.autoclose? + fd = io.fileno + rescue IOError # closed IO object + next + end + (h[fd] ||= []) << [io, autoclose, inspect] + } + fd_leaked.select! {|fd| + str = ''.dup + pos = nil + if h[fd] + str << ' :' + h[fd].map {|io, autoclose, inspect| + if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] + pos = "#{ObjectSpace.allocation_sourcefile(io)}:#{ObjectSpace.allocation_sourceline(io)}" + end + s = ' ' + inspect + s << "(not-autoclose)" if !autoclose + s + }.sort.each {|s| + str << s + } + else + begin + io = IO.for_fd(fd, autoclose: false) + s = io.stat + rescue Errno::EBADF + # something un-stat-able + next + else + next if /darwin/ =~ RUBY_PLATFORM and [0, -1].include?(s.dev) + str << ' ' << s.inspect + ensure + io&.close + end + end + puts "Leaked file descriptor: #{test_name}: #{fd}#{str}" + puts " The IO was created at #{pos}" if pos + true + } + unless fd_leaked.empty? + unless @@try_lsof == false + @@try_lsof |= system(*%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + end + end + h.each {|fd, list| + next if list.length <= 1 + if 1 < list.count {|io, autoclose, inspect| autoclose } + str = list.map {|io, autoclose, inspect| " #{inspect}" + (autoclose ? "(autoclose)" : "") }.sort.join + puts "Multiple autoclose IO objects for a file descriptor in: #{test_name}: #{str}" + end + } + end + @fd_info = live2 + @@skip = false + return leaked + end + + def extend_tempfile_counter + return if defined? LeakChecker::TempfileCounter + m = Module.new { + @count = 0 + class << self + attr_accessor :count + end + + def new(...) + LeakChecker::TempfileCounter.count += 1 + super + end + } + LeakChecker.const_set(:TempfileCounter, m) + + class << Tempfile + prepend LeakChecker::TempfileCounter + end + end + + def find_tempfiles(prev_count=-1) + return [prev_count, []] unless defined? Tempfile + extend_tempfile_counter + count = TempfileCounter.count + if prev_count == count + [prev_count, []] + else + tempfiles = ObjectSpace.each_object(Tempfile).reject {|t| + t.instance_variables.empty? || t.closed? + } + [count, tempfiles] + end + end + + def check_tempfile_leak(test_name) + return false unless defined? Tempfile + count1, initial_tempfiles = @tempfile_info + count2, current_tempfiles = find_tempfiles(count1) + leaked = false + tempfiles_leaked = current_tempfiles - initial_tempfiles + if !tempfiles_leaked.empty? + leaked = true + list = tempfiles_leaked.map {|t| t.inspect }.sort + list.each {|str| + puts "Leaked tempfile: #{test_name}: #{str}" + } + tempfiles_leaked.each {|t| t.close! } + end + @tempfile_info = [count2, initial_tempfiles] + return leaked + end + + def find_threads + Thread.list.find_all {|t| + t != Thread.current && t.alive? && + !(t.thread_variable?(:"\0__detached_thread__") && t.thread_variable_get(:"\0__detached_thread__")) + } + end + + def check_thread_leak(test_name) + live1 = @thread_info + live2 = find_threads + thread_finished = live1 - live2 + leaked = false + if !thread_finished.empty? + list = thread_finished.map {|t| t.inspect }.sort + list.each {|str| + puts "Finished thread: #{test_name}: #{str}" + } + end + thread_leaked = live2 - live1 + if !thread_leaked.empty? + leaked = true + list = thread_leaked.map {|t| t.inspect }.sort + list.each {|str| + puts "Leaked thread: #{test_name}: #{str}" + } + end + @thread_info = live2 + return leaked + end + + e = ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] + begin + ENV["_Ruby_Env_Ignorecase_"] = ENV["_RUBY_ENV_IGNORECASE_"] = nil + ENV["_RUBY_ENV_IGNORECASE_"] = "ENV_CASE_TEST" + ENV_IGNORECASE = ENV["_Ruby_Env_Ignorecase_"] == "ENV_CASE_TEST" + ensure + ENV["_Ruby_Env_Ignorecase_"], ENV["_RUBY_ENV_IGNORECASE_"] = e + end + + if ENV_IGNORECASE + def find_env + ENV.to_h {|k, v| [k.upcase, v]} + end + else + def find_env + ENV.to_h + end + end + + def check_env(test_name) + old_env = @env_info + new_env = find_env + return false if old_env == new_env + if defined?(Bundler::EnvironmentPreserver) + bundler_prefix = Bundler::EnvironmentPreserver::BUNDLER_PREFIX + end + (old_env.keys | new_env.keys).sort.each {|k| + # Don't report changed environment variables caused by Bundler's backups + next if bundler_prefix and k.start_with?(bundler_prefix) + + if old_env.has_key?(k) + if new_env.has_key?(k) + if old_env[k] != new_env[k] + puts "Environment variable changed: #{test_name} : #{k.inspect} changed : #{old_env[k].inspect} -> #{new_env[k].inspect}" + end + else + puts "Environment variable changed: #{test_name} : #{k.inspect} deleted" + end + else + if new_env.has_key?(k) + puts "Environment variable changed: #{test_name} : #{k.inspect} added" + else + flunk "unreachable" + end + end + } + @env_info = new_env + return true + end + + def find_encodings + { + 'Encoding.default_internal' => Encoding.default_internal, + 'Encoding.default_external' => Encoding.default_external, + 'STDIN.internal_encoding' => STDIN.internal_encoding, + 'STDIN.external_encoding' => STDIN.external_encoding, + 'STDOUT.internal_encoding' => STDOUT.internal_encoding, + 'STDOUT.external_encoding' => STDOUT.external_encoding, + 'STDERR.internal_encoding' => STDERR.internal_encoding, + 'STDERR.external_encoding' => STDERR.external_encoding, + } + end + + def check_encodings(test_name) + old_encoding_info = @encoding_info + @encoding_info = find_encodings + leaked = false + @encoding_info.each do |key, new_encoding| + old_encoding = old_encoding_info[key] + if new_encoding != old_encoding + leaked = true + puts "#{key} changed: #{test_name} : #{old_encoding.inspect} to #{new_encoding.inspect}" + end + end + leaked + end + + WARNING_CATEGORIES = (Warning.respond_to?(:[]) ? %i[deprecated experimental] : []).freeze + + def find_warning_flags + WARNING_CATEGORIES.to_h do |category| + [category, Warning[category]] + end + end + + def check_warning_flags(test_name) + new_warning_flags = find_warning_flags + leaked = false + WARNING_CATEGORIES.each do |category| + if new_warning_flags[category] != @old_warning_flags[category] + leaked = true + puts "Warning[#{category.inspect}] changed: #{test_name} : #{@old_warning_flags[category]} to #{new_warning_flags[category]}" + end + end + return leaked + end + + def puts(*a) + output = Test::Unit::Runner.output + if defined?(output.set_encoding) + output.set_encoding(nil, nil) + end + output.puts(*a) + end + + def self.skip + @@skip = true + end +end diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb new file mode 100644 index 0000000000..429e5f6a1d --- /dev/null +++ b/tool/lib/memory_status.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true +begin + require '-test-/memory_status.so' +rescue LoadError +end + +module Memory + keys = [] + + case + when File.exist?(procfile = "/proc/self/status") && (pat = /^Vm(\w+):\s+(\d+)/) =~ (data = File.binread(procfile)) + PROC_FILE = procfile + VM_PAT = pat + def self.read_status + File.foreach(PROC_FILE, encoding: Encoding::ASCII_8BIT) do |l| + yield($1.downcase.intern, $2.to_i * 1024) if VM_PAT =~ l + end + end + + data.scan(pat) {|k, v| keys << k.downcase.intern} + + when /mswin|mingw/ =~ RUBY_PLATFORM + keys.push(:size, :rss, :peak) + + begin + require 'fiddle/import' + require 'fiddle/types' + rescue LoadError + # Fallback to PowerShell command to get memory information for current process + def self.read_status + cmd = [ + "powershell.exe", "-NoProfile", "-Command", + "Get-Process -Id #{$$} | " \ + "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }" + ] + + IO.popen(cmd, "r", err: [:child, :out]) do |out| + if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read + yield :size, $1.to_i + yield :rss, $2.to_i + yield :peak, $3.to_i + end + end + end + else + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end + end + + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end + end + end + when (require_relative 'find_executable' + pat = /^\s*(\d+)\s+(\d+)$/ + pscmd = EnvUtil.find_executable("ps", "-ovsz=", "-orss=", "-p", $$.to_s) {|out| pat =~ out}) + pscmd.pop + PAT = pat + PSCMD = pscmd + + keys << :size << :rss + def self.read_status + if PAT =~ IO.popen(PSCMD + [$$.to_s], "r", err: [:child, :out], &:read) + yield :size, $1.to_i*1024 + yield :rss, $2.to_i*1024 + end + end + else + def self.read_status + raise NotImplementedError, "unsupported platform" + end + end + + if !keys.empty? + Status = Struct.new(*keys) + end +end unless defined?(Memory::Status) + +if defined?(Memory::Status) + class Memory::Status + def _update + Memory.read_status do |key, val| + self[key] = val + end + self + end unless method_defined?(:_update) + + Header = members.map {|k| k.to_s.upcase.rjust(6)}.join('') + Format = "%6d" + + def initialize + _update + end + + def to_s + status = each_pair.map {|n,v| + "#{n}:#{v}" + } + "{#{status.join(",")}}" + end + + def self.parse(str) + status = allocate + str.scan(/(?:\A\{|\G,)(#{members.join('|')}):(\d+)(?=,|\}\z)/) do + status[$1] = $2.to_i + end + status + end + end + + # On some platforms (e.g. Solaris), libc malloc does not return + # freed memory to OS because of efficiency, and linking with extra + # malloc library is needed to detect memory leaks. + # + case RUBY_PLATFORM + when /solaris2\.(?:9|[1-9][0-9])/i # Solaris 9, 10, 11,... + bits = [nil].pack('p').size == 8 ? 64 : 32 + if ENV['LD_PRELOAD'].to_s.empty? && + ENV["LD_PRELOAD_#{bits}"].to_s.empty? && + (ENV['UMEM_OPTIONS'].to_s.empty? || + ENV['UMEM_OPTIONS'] == 'backend=mmap') then + envs = { + 'LD_PRELOAD' => 'libumem.so', + 'UMEM_OPTIONS' => 'backend=mmap' + } + args = [ + envs, + "--disable=gems", + "-v", "-", + ] + _, err, status = EnvUtil.invoke_ruby(args, "exit(0)", true, true) + if status.exitstatus == 0 && err.to_s.empty? then + Memory::NO_MEMORY_LEAK_ENVS = envs + end + end + end #case RUBY_PLATFORM + +end diff --git a/tool/lib/output.rb b/tool/lib/output.rb new file mode 100644 index 0000000000..8cb426ae4a --- /dev/null +++ b/tool/lib/output.rb @@ -0,0 +1,70 @@ +require_relative 'vpath' +require_relative 'colorize' + +class Output + attr_reader :path, :vpath + + def initialize(path: nil, timestamp: nil, ifchange: nil, color: nil, + overwrite: false, create_only: false, vpath: VPath.new) + @path = path + @timestamp = timestamp + @ifchange = ifchange + @color = color + @overwrite = overwrite + @create_only = create_only + @vpath = vpath + end + + COLOR_WHEN = { + 'always' => true, 'auto' => nil, 'never' => false, + nil => true, false => false, + } + + def def_options(opt) + opt.separator(" Output common options:") + opt.on('-o', '--output=PATH') {|v| @path = v} + opt.on('-t', '--timestamp[=PATH]') {|v| @timestamp = v || true} + opt.on('-c', '--[no-]if-change') {|v| @ifchange = v} + opt.on('--[no-]color=[WHEN]', COLOR_WHEN.keys) {|v| @color = COLOR_WHEN[v]} + opt.on('--[no-]create-only') {|v| @create_only = v} + opt.on('--[no-]overwrite') {|v| @overwrite = v} + @vpath.def_options(opt) + end + + def write(data, overwrite: @overwrite, create_only: @create_only) + unless @path + $stdout.print data + return true + end + color = Colorize.new(@color) + unchanged = color.pass("unchanged") + updated = color.fail("updated") + outpath = nil + + if (@ifchange or overwrite or create_only) and (@vpath.open(@path, "rb") {|f| + outpath = f.path + if @ifchange or create_only + original = f.read + (@ifchange and original == data) or (create_only and !original.empty?) + end + } rescue false) + puts "#{outpath} #{unchanged}" + written = false + else + unless overwrite and outpath and (File.binwrite(outpath, data) rescue nil) + File.binwrite(outpath = @path, data) + end + puts "#{outpath} #{updated}" + written = true + end + if timestamp = @timestamp + if timestamp == true + dir, base = File.split(@path) + timestamp = File.join(dir, ".time." + base) + end + File.binwrite(timestamp, '') + File.utime(nil, nil, timestamp) + end + written + end +end diff --git a/tool/lib/path.rb b/tool/lib/path.rb new file mode 100644 index 0000000000..f16a164338 --- /dev/null +++ b/tool/lib/path.rb @@ -0,0 +1,101 @@ +module Path + module_function + + def clean(path) + path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') + nil while path.sub!(/[^\/]+\/\.\.\//, '') + path + end + + def relative(path, base) + path = clean(path) + base = clean(base) + path, base = [path, base].map{|s|s.split("/")} + until path.empty? or base.empty? or path[0] != base[0] + path.shift + base.shift + end + path, base = [path, base].map{|s|s.join("/")} + if base.empty? + path + elsif base.start_with?("../") or File.absolute_path?(base) + File.expand_path(path) + else + base.gsub!(/[^\/]+/, '..') + File.join(base, path) + end + end + + def clean_link(src, dest) + begin + link = File.readlink(dest) + rescue + else + return if link == src + File.unlink(dest) + end + yield src, dest + end + + # Extensions to FileUtils + + module Mswin + def ln_safe(src, dest, real_src, *opt) + cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] + cmd[1, 0] = opt + return if system("cmd", "/c", *cmd) + # TODO: use RUNAS or something + puts cmd.join(" ") + end + + def ln_dir_safe(src, dest, real_src) + ln_safe(src, dest, "/d") + end + end + + module HardlinkExcutable + def ln_exe(relative_src, dest, src) + ln(src, dest, force: true) + end + end + + def ln_safe(src, dest, real_src) + ln_sf(src, dest) + rescue Errno::ENOENT + # Windows disallows to create broken symboic links, probably because + # it is a kind of reparse points. + raise if File.exist?(real_src) + end + + alias ln_dir_safe ln_safe + alias ln_exe ln_safe + + def ln_relative(src, dest, executable = false) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + if executable + return (ln_exe(relative(src, parent), dest, src) if File.exist?(src)) + end + clean_link(relative(src, parent), dest) {|s, d| ln_safe(s, d, src)} + end + + def ln_dir_relative(src, dest) + return if File.identical?(src, dest) + parent = File.dirname(dest) + File.directory?(parent) or mkdir_p(parent) + clean_link(relative(src, parent), dest) {|s, d| ln_dir_safe(s, d, src)} + end + + case (CROSS_COMPILING || RUBY_PLATFORM) + when /linux|darwin|solaris/ + prepend HardlinkExcutable + extend HardlinkExcutable + when /mingw|mswin/ + unless File.respond_to?(:symlink) + prepend Mswin + extend Mswin + end + else + end +end diff --git a/tool/lib/profile_test_all.rb b/tool/lib/profile_test_all.rb new file mode 100644 index 0000000000..fb434e314d --- /dev/null +++ b/tool/lib/profile_test_all.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +# +# purpose: +# Profile memory usage of each tests. +# +# usage: +# RUBY_TEST_ALL_PROFILE=[file] make test-all +# +# output: +# [file] specified by RUBY_TEST_ALL_PROFILE +# If [file] is 'true', then it is ./test_all_profile +# +# collected information: +# - ObjectSpace.memsize_of_all +# - GC.stat +# - /proc/meminfo (some fields, if exists) +# - /proc/self/status (some fields, if exists) +# - /proc/self/statm (if exists) +# + +require 'objspace' + +class Test::Unit::TestCase + alias orig_run run + + file = ENV['RUBY_TEST_ALL_PROFILE'] + file = 'test-all-profile-result' if file == 'true' + TEST_ALL_PROFILE_OUT = open(file, 'w') + TEST_ALL_PROFILE_GC_STAT_HASH = {} + TEST_ALL_PROFILE_BANNER = ['name'] + TEST_ALL_PROFILE_PROCS = [] + + def self.add *name, &b + TEST_ALL_PROFILE_BANNER.concat name + TEST_ALL_PROFILE_PROCS << b + end + + add 'failed?' do |result, tc| + result << (tc.passed? ? 0 : 1) + end + + add 'memsize_of_all' do |result, *| + result << ObjectSpace.memsize_of_all + end + + add(*GC.stat.keys) do |result, *| + GC.stat(TEST_ALL_PROFILE_GC_STAT_HASH) + result.concat TEST_ALL_PROFILE_GC_STAT_HASH.values + end + + def self.add_proc_meminfo file, fields + return unless FileTest.exist?(file) + regexp = /(#{fields.join("|")}):\s*(\d+) kB/ + # check = {}; fields.each{|e| check[e] = true} + add(*fields) do |result, *| + text = File.read(file) + text.scan(regexp){ + # check.delete $1 + result << $2 + '' + } + # raise check.inspect unless check.empty? + end + end + + add_proc_meminfo '/proc/meminfo', %w(MemTotal MemFree) + add_proc_meminfo '/proc/self/status', %w(VmPeak VmSize VmHWM VmRSS) + + if FileTest.exist?('/proc/self/statm') + add 'size', 'resident', 'share', 'text', 'lib', 'data', 'dt' do |result, *| + result.concat File.read('/proc/self/statm').split(/\s+/) + end + end + + def memprofile_test_all_result_result + result = ["#{self.class}\##{self.__name__.to_s.gsub(/\s+/, '')}"] + TEST_ALL_PROFILE_PROCS.each{|proc| + proc.call(result, self) + } + result.join("\t") + end + + def run runner + result = orig_run(runner) + TEST_ALL_PROFILE_OUT.puts memprofile_test_all_result_result + TEST_ALL_PROFILE_OUT.flush + result + end + + TEST_ALL_PROFILE_OUT.puts TEST_ALL_PROFILE_BANNER.join("\t") +end diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb new file mode 100644 index 0000000000..7b889163b0 --- /dev/null +++ b/tool/lib/test/jobserver.rb @@ -0,0 +1,47 @@ +module Test + module JobServer + end +end + +class << Test::JobServer + def connect(makeflags = ENV["MAKEFLAGS"]) + return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags + begin + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end + rescue + r&.close + nil + else + return r, w + end + end + + def acquire_possible(r, w, max) + return unless tokens = r.read_nonblock(max - 1, exception: false) + if (jobs = tokens.size) > 0 + jobserver, w = w, nil + at_exit do + jobserver.print(tokens) + jobserver.close + end + end + return jobs + 1 + rescue Errno::EBADF + ensure + r&.close + w&.close + end + + def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"]) + if max > 1 and (r, w = connect(makeflags)) + acquire_possible(r, w, max) + end + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb new file mode 100644 index 0000000000..2663b7b76a --- /dev/null +++ b/tool/lib/test/unit.rb @@ -0,0 +1,1896 @@ +# frozen_string_literal: true + +# Enable deprecation warnings for test-all, so deprecated methods/constants/functions are dealt with early. +Warning[:deprecated] = true + +if ENV['BACKTRACE_FOR_DEPRECATION_WARNINGS'] + Warning.extend Module.new { + def warn(message, category: nil, **kwargs) + if category == :deprecated and $stderr.respond_to?(:puts) + $stderr.puts nil, message, caller, nil + else + super + end + end + } +end + +require_relative '../envutil' +require_relative '../colorize' +require_relative '../leakchecker' +require_relative '../test/unit/testcase' +require_relative '../test/jobserver' +require 'optparse' + +# See Test::Unit +module Test + + ## + # Test::Unit is an implementation of the xUnit testing framework for Ruby. + module Unit + ## + # Assertion base class + + class AssertionFailedError < Exception; end + + ## + # Assertion raised when skipping a test + + class PendedError < AssertionFailedError; end + + class << self + ## + # Extract the location where the last assertion method was + # called. Returns "<empty>" if _e_ does not have backtrace, or + # an empty string if no assertion method location was found. + + def location e + last_before_assertion = nil + + return '<empty>' unless e&.backtrace # SystemStackError can return nil. + + e.backtrace.reverse_each do |s| + break if s =~ /:in \W(?:.*\#)?(?:assert|refute|flunk|pass|fail|raise|must|wont)/ + last_before_assertion = s + end + return "" unless last_before_assertion + /:in / =~ last_before_assertion ? $` : last_before_assertion + end + end + + module Order + class NoSort + def initialize(seed) + end + + def sort_by_name(list) + list + end + + alias sort_by_string sort_by_name + + def group(list) + list + end + end + + class Alpha < NoSort + def sort_by_name(list) + list.sort_by(&:name) + end + + def sort_by_string(list) + list.sort + end + + end + + # shuffle test suites based on CRC32 of their names + Shuffle = Struct.new(:seed, :salt) do + def initialize(seed) + self.class::CRC_TBL ||= (0..255).map {|i| + (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } + }.freeze + + salt = [seed].pack("V").unpack1("H*") + super(seed, "\n#{salt}".freeze).freeze + end + + def sort_by_name(list) + list.sort_by {|e| randomize_key(e.name)} + end + + def sort_by_string(list) + list.sort_by {|e| randomize_key(e)} + end + + def group(list) + list + end + + private + + def crc32(str, crc32 = 0xffffffff) + crc_tbl = self.class::CRC_TBL + str.each_byte do |data| + crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) + end + crc32 + end + + def randomize_key(name) + crc32(salt, crc32(name)) ^ 0xffffffff + end + end + + Types = { + random: Shuffle, + alpha: Alpha, + sorted: Alpha, + nosort: NoSort, + } + Types.default_proc = proc {|_, order| + raise "Unknown test_order: #{order.inspect}" + } + end + + module RunCount # :nodoc: all + @@run_count = 0 + + def self.have_run? + @@run_count.nonzero? + end + + def run(*) + @@run_count += 1 + super + end + + def run_once + return if have_run? + return if $! # don't run if there was an exception + yield + end + module_function :run_once + end + + module Options # :nodoc: all + def initialize(*, &block) + @init_hook = block + @options = nil + super(&nil) + end + + def option_parser + @option_parser ||= OptionParser.new + end + + def process_args(args = []) + return @options if @options + orig_args = args.dup + options = {} + opts = option_parser + setup_options(opts, options) + opts.parse!(args) + orig_args -= args + args = @init_hook.call(args, options) if @init_hook + non_options(args, options) + @run_options = orig_args + + order = options[:test_order] + if seed = options[:seed] + order ||= :random + elsif (order ||= :random) == :random + seed = options[:seed] = rand(0x10000) + orig_args.unshift "--seed=#{seed}" + end + Test::Unit::TestCase.test_order = order if order + order = Test::Unit::TestCase.test_order + @order = Test::Unit::Order::Types[order].new(seed) + + @help = "\n" + orig_args.map { |s| + " " + (s =~ /[\s|&<>$()]/ ? s.inspect : s) + }.join("\n") + + @options = options + end + + private + def setup_options(opts, options) + opts.separator 'test-unit options:' + + opts.on '-h', '--help', 'Display this help.' do + puts opts + exit + end + + opts.on '-s', '--seed SEED', Integer, "Sets random seed" do |m| + options[:seed] = m.to_i + end + + opts.on '-v', '--verbose', "Verbose. Show progress processing files." do + options[:verbose] = true + self.verbose = options[:verbose] + end + + opts.on '-n', '--name PATTERN', "Filter test method names on pattern: /REGEXP/, !/REGEXP/ or STRING" do |a| + (options[:filter] ||= []) << a + end + + orders = Test::Unit::Order::Types.keys + opts.on "--test-order=#{orders.join('|')}", orders do |a| + options[:test_order] = a + end + end + + def non_options(files, options) + filter = options[:filter] + if filter + pos_pat = /\A\/(.*)\/\z/ + neg_pat = /\A!\/(.*)\/\z/ + negative, positive = filter.partition {|s| neg_pat =~ s} + if positive.empty? + filter = nil + elsif negative.empty? and positive.size == 1 and pos_pat !~ positive[0] + filter = positive[0] + unless /\A[A-Z]\w*(?:::[A-Z]\w*)*#/ =~ filter + filter = /##{Regexp.quote(filter)}\z/ + end + else + filter = Regexp.union(*positive.map! {|s| Regexp.new(s[pos_pat, 1] || "\\A#{Regexp.quote(s)}\\z")}) + end + unless negative.empty? + negative = Regexp.union(*negative.map! {|s| Regexp.new(s[neg_pat, 1])}) + filter = /\A(?=.*#{filter})(?!.*#{negative})/ + end + options[:filter] = filter + end + true + end + end + + module Parallel # :nodoc: all + attr_accessor :prefix + + def process_args(args = []) + return @options if @options + options = super + if @options[:parallel] + @files = args + end + options + end + + def non_options(files, options) + @jobserver = nil + if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS")) + options[:parallel] ||= 256 # number of tokens to acquire first + end + @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) + super + end + + def status(*args) + result = super + raise @interrupt if @interrupt + result + end + + private + def setup_options(opts, options) + super + + opts.separator "parallel test options:" + + options[:retry] = false + + opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| + options[:testing] = true & t # For testing + options[:parallel] = a.to_i + end + + opts.on '--worker-timeout=N', Integer, "Timeout workers not responding in N seconds" do |a| + options[:worker_timeout] = a + end + + opts.on '--separate', "Restart job process after one testcase has done" do + options[:parallel] ||= 1 + options[:separate] = true + end + + opts.on '--retry', "Retry running testcase when --jobs specified" do + options[:retry] = true + end + + opts.on '--no-retry', "Disable --retry" do + options[:retry] = false + end + + opts.on '--ruby VAL', "Path to ruby which is used at -j option", + "Also used as EnvUtil.rubybin by some assertion methods" do |a| + options[:ruby] = a.split(/ /).reject(&:empty?) + end + + opts.on '--timetable-data=FILE', "Path to timetable data" do |a| + options[:timetable_data] = a + end + end + + class Worker + def self.launch(ruby,args=[]) + scale = EnvUtil.timeout_scale + io = IO.popen([*ruby, "-W1", + "#{__dir__}/unit/parallel.rb", + *("--timeout-scale=#{scale}" if scale), + *args], "rb+") + new(io, io.pid, :waiting) + end + + attr_reader :quit_called + attr_accessor :start_time + attr_accessor :response_at + attr_accessor :current + + @@worker_number = 0 + + def initialize(io, pid, status) + @num = (@@worker_number += 1) + @io = io + @pid = pid + @status = status + @file = nil + @real_file = nil + @loadpath = [] + @hooks = {} + @quit_called = false + @response_at = nil + end + + def name + "Worker #{@num}" + end + + def puts(*args) + @io.puts(*args) + end + + def run(task, type, base = nil) + if base + @file = task.delete_prefix(base).chomp(".rb") + else + @file = File.basename(task, ".rb") + end + @real_file = task + begin + puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" + @loadpath = $:.dup + puts "run #{task} #{type}" + @status = :prepare + @start_time = Time.now + @response_at = @start_time + rescue Errno::EPIPE + died + rescue IOError + raise unless /stream closed|closed stream/ =~ $!.message + died + end + end + + def hook(id,&block) + @hooks[id] ||= [] + @hooks[id] << block + self + end + + def read + res = (@status == :quit) ? @io.read : @io.gets + @response_at = Time.now + res && res.chomp + end + + def close + @io.close unless @io.closed? + self + rescue IOError + end + + def quit(reason = :normal) + return if @io.closed? + @quit_called = true + @io.puts "quit #{reason}" + rescue Errno::EPIPE => e + warn "#{@pid}:#{@status.to_s.ljust(7)}:#{@file}: #{e.message}" + end + + def kill + EnvUtil::Debugger.search&.dump(@pid) + signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV + Process.kill(signal, @pid) + warn "worker #{to_s} does not respond; #{signal} is sent" + rescue Errno::ESRCH + end + + def died(*additional) + @status = :quit + @io.close + status = $? + if status and status.signaled? + additional[0] ||= SignalException.new(status.termsig) + end + + call_hook(:dead,*additional) + end + + def to_s + if @file and @status != :ready + "#{@pid}=#{@file}" + else + "#{@pid}:#{@status.to_s.ljust(7)}" + end + end + + attr_reader :io, :pid + attr_accessor :status, :file, :real_file, :loadpath + + private + + def call_hook(id,*additional) + @hooks[id] ||= [] + @hooks[id].each{|hook| hook[self,additional] } + self + end + + end + + def flush_job_tokens + if @jobserver + r, w = @jobserver.shift(2) + @jobserver = nil + w << @job_tokens.slice!(0..-1) + r.close + w.close + end + end + + def after_worker_down(worker, e=nil, c=false) + return unless @options[:parallel] + return if @interrupt + flush_job_tokens + warn e if e + real_file = worker.real_file and warn "running file: #{real_file}" + @need_quit = true + warn "" + warn "A test worker crashed. It might be an interpreter bug or" + warn "a bug in test/unit/parallel.rb. Try again without the -j" + warn "option." + warn "" + if File.exist?('core') + require 'fileutils' + require 'time' + Dir.glob('/tmp/test-unit-core.*').each do |f| + if Time.now - File.mtime(f) > 7 * 24 * 60 * 60 # 7 days + warn "Deleting an old core file: #{f}" + FileUtils.rm(f) + end + end + core_path = "/tmp/test-unit-core.#{Time.now.utc.iso8601}" + warn "A core file is found. Saving it at: #{core_path.dump}" + FileUtils.mv('core', core_path) + cmd = ['gdb', RbConfig.ruby, '-c', core_path, '-ex', 'bt', '-batch'] + p cmd # debugging why it's not working + system(*cmd) + end + STDERR.flush + exit c + end + + def after_worker_quit(worker) + return unless @options[:parallel] + return if @interrupt + worker.close + if @jobserver and (token = @job_tokens.slice!(0)) + @jobserver[1] << token + end + @workers.delete(worker) + @dead_workers << worker + @ios = @workers.map(&:io) + end + + def launch_worker + begin + worker = Worker.launch(@options[:ruby], @run_options) + rescue => e + abort "ERROR: Failed to launch job process - #{e.class}: #{e.message}" + end + worker.hook(:dead) do |w,info| + after_worker_quit w + after_worker_down w, *info if !info.empty? && !worker.quit_called + end + @workers << worker + @ios << worker.io + @workers_hash[worker.io] = worker + worker + end + + def delete_worker(worker) + @workers_hash.delete worker.io + @workers.delete worker + @ios.delete worker.io + end + + def quit_workers(&cond) + return if @workers.empty? + closed = [] if cond + @workers.reject! do |worker| + next unless cond&.call(worker) + begin + Timeout.timeout(5) do + worker.quit(cond ? :timeout : :normal) + end + rescue Errno::EPIPE + rescue Timeout::Error + end + closed&.push worker + begin + Timeout.timeout(1) do + worker.close + end + rescue Timeout::Error + worker.kill + retry + end + @ios.delete worker.io + end + + return if (closed ||= @workers).empty? + pids = closed.map(&:pid) + begin + Timeout.timeout(1 * closed.size) do + Process.waitall + end + rescue Timeout::Error + if pids + Process.kill(:KILL, *pids) rescue nil + pids = nil + retry + end + end + @workers.clear unless cond + closed + end + + FakeClass = Struct.new(:name) + def fake_class(name) + (@fake_classes ||= {})[name] ||= FakeClass.new(name) + end + + def deal(io, type, result, rep, shutting_down = false) + worker = @workers_hash[io] + cmd = worker.read + cmd.sub!(/\A\.+/, '') if cmd # read may return nil + + case cmd + when '' + # just only dots, ignore + when /^okay$/ + worker.status = :running + when /^ready(!)?$/ + bang = $1 + worker.status = :ready + + unless task = @tasks.shift + worker.quit + return nil + end + if @options[:separate] and not bang + worker.quit + worker = launch_worker + end + worker.run(task, type, (@prefix unless @options[:job_status] == :replace)) + @test_count += 1 + + jobs_status(worker) + when /^start (.+?)$/ + worker.current = Marshal.load($1.unpack1("m")) + when /^done (.+?)$/ + begin + r = Marshal.load($1.unpack1("m")) + rescue + print "unknown object: #{$1.unpack1("m").dump}" + return true + end + result << r[0..1] unless r[0..1] == [nil,nil] + rep << {file: worker.real_file, report: r[2], result: r[3], testcase: r[5]} + $:.push(*r[4]).uniq! + jobs_status(worker) if @options[:job_status] == :replace + + return true + when /^record (.+?)$/ + begin + r = Marshal.load($1.unpack1("m")) + + suite = r.first + key = [worker.name, suite] + if @records[key] + @records[key][1] = worker.start_time = Time.now + else + @records[key] = [worker.start_time, Time.now] + end + rescue => e + print "unknown record: #{e.message} #{$1.unpack1("m").dump}" + return true + end + record(fake_class(r[0]), *r[1..-1]) + when /^p (.+?)$/ + del_jobs_status + print $1.unpack1("m") + jobs_status(worker) if @options[:job_status] == :replace + when /^after (.+?)$/ + @warnings << Marshal.load($1.unpack1("m")) + when /^bye (.+?)$/ + after_worker_down worker, Marshal.load($1.unpack1("m")) + when /^bye$/, nil + if shutting_down || worker.quit_called + after_worker_quit worker + else + after_worker_down worker + end + else + print "unknown command: #{cmd.dump}\n" + end + return false + end + + def _run_parallel suites, type, result + @records = {} + + if @options[:parallel] < 1 + warn "Error: parameter of -j option should be greater than 0." + return + end + + # Require needed thing for parallel running + require 'timeout' + @tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames. + + @need_quit = false + @dead_workers = [] # Array of dead workers. + @warnings = [] + @total_tests = @tasks.size.to_s(10) + rep = [] # FIXME: more good naming + + @workers = [] # Array of workers. + @workers_hash = {} # out-IO => worker + @ios = [] # Array of worker IOs + @job_tokens = String.new(encoding: Encoding::ASCII_8BIT) if @jobserver + begin + while true + newjobs = [@tasks.size, @options[:parallel]].min - @workers.size + if newjobs > 0 + if @jobserver + t = @jobserver[0].read_nonblock(newjobs, exception: false) + @job_tokens << t if String === t + newjobs = @job_tokens.size + 1 - @workers.size + end + newjobs.times {launch_worker} + end + + timeout = [(@workers.filter_map {|w| w.response_at}.min&.-(Time.now) || 0), 0].max + @worker_timeout + + if !(_io = IO.select(@ios, nil, nil, timeout)) + timeout = Time.now - @worker_timeout + quit_workers {|w| w.response_at&.<(timeout) }&.map {|w| + rep << {file: w.real_file, result: nil, testcase: w.current[0], error: w.current} + } + elsif _io.first.any? {|io| + @need_quit or + (deal(io, type, result, rep).nil? and + !@workers.any? {|x| [:running, :prepare].include? x.status}) + } + break + end + if @tasks.empty? + break if @workers.empty? + next # wait for all workers to finish + end + end + rescue Interrupt => ex + @interrupt = ex + return result + ensure + if file = @options[:timetable_data] + File.open(file, 'w'){|f| + @records.each{|(worker, suite), (st, ed)| + f.puts '[' + [worker.dump, suite.dump, st.to_f * 1_000, ed.to_f * 1_000].join(", ") + '],' + } + } + end + + if @interrupt + @ios.select!{|x| @workers_hash[x].status == :running } + while !@ios.empty? && (__io = IO.select(@ios,[],[],10)) + __io[0].reject! {|io| deal(io, type, result, rep, true)} + end + end + + quit_workers + flush_job_tokens + + unless @interrupt || !@options[:retry] || @need_quit + parallel = @options[:parallel] + @options[:parallel] = false + suites, rep = rep.partition {|r| + r[:testcase] && r[:file] && + (!r.key?(:report) || r[:report].any? {|e| !e[2].is_a?(Test::Unit::PendedError)}) + } + suites.map {|r| File.realpath(r[:file])}.uniq.each {|file| require file} + del_status_line or puts + error, suites = suites.partition {|r| r[:error]} + unless suites.empty? + puts "\n" + @failed_output.puts "Failed tests:" + suites.each {|r| + r[:report].each {|c, m, e| + @failed_output.puts "#{c}##{m}: #{e&.class}: #{e&.message&.slice(/\A.*/)}" + } + } + @failed_output.puts "\n" + puts "Retrying..." + @verbose = options[:verbose] + suites.map! {|r| ::Object.const_get(r[:testcase])} + _run_suites(suites, type) + end + unless error.empty? + puts "\n""Retrying hung up testcases..." + error = error.map do |r| + begin + ::Object.const_get(r[:testcase]) + rescue NameError + # testcase doesn't specify the correct case, so show `r` for information + require 'pp' + + $stderr.puts "Retrying is failed because the file and testcase is not consistent:" + PP.pp r, $stderr + @errors += 1 + nil + end + end.compact + verbose = @verbose + job_status = options[:job_status] + options[:verbose] = @verbose = true + options[:job_status] = :normal + result.concat _run_suites(error, type) + options[:verbose] = @verbose = verbose + options[:job_status] = job_status + end + @options[:parallel] = parallel + end + unless @options[:retry] + del_status_line or puts + end + unless rep.empty? + rep.each do |r| + if r[:error] + puke(*r[:error], Timeout::Error.new) + next + end + r[:report]&.each do |f| + puke(*f) if f + end + end + if @options[:retry] + rep.each do |x| + (e, f, s = x[:result]) or next + @errors += e + @failures += f + @skips += s + end + end + end + unless @warnings.empty? + warn "" + @warnings.uniq! {|w| w[1].message} + @warnings.each do |w| + @errors += 1 + warn "#{w[0]}: #{w[1].message} (#{w[1].class})" + end + warn "" + end + end + end + + def _run_suites suites, type + _prepare_run(suites, type) + @interrupt = nil + result = [] + GC.start + if @options[:parallel] + _run_parallel suites, type, result + else + suites.each {|suite| + begin + result << _run_suite(suite, type) + rescue Interrupt => e + @interrupt = e + break + end + } + end + del_status_line + result + end + end + + module Skipping # :nodoc: all + def failed(s) + super if !s or @options[:hide_skip] + end + + private + def setup_options(opts, options) + super + + opts.separator "skipping options:" + + options[:hide_skip] = true + + opts.on '-q', '--hide-skip', 'Hide skipped tests' do + options[:hide_skip] = true + end + + opts.on '--show-skip', 'Show skipped tests' do + options[:hide_skip] = false + end + end + + def _run_suites(suites, type) + result = super + report.reject!{|r| r.start_with? "Skipped:" } if @options[:hide_skip] + report.sort_by!{|r| r.start_with?("Skipped:") ? 0 : \ + (r.start_with?("Failure:") ? 1 : 2) } + failed(nil) + result + end + end + + module Statistics + def update_list(list, rec, max) + if i = list.empty? ? 0 : list.bsearch_index {|*a| yield(*a)} + list[i, 0] = [rec] + list[max..-1] = [] if list.size >= max + end + end + + def record(suite, method, assertions, time, error, source_location = nil) + if @options.values_at(:longest, :most_asserted).any? + @tops ||= {} + rec = [suite.name, method, assertions, time, error] + if max = @options[:longest] + update_list(@tops[:longest] ||= [], rec, max) {|_,_,_,t,_|t<time} + end + if max = @options[:most_asserted] + update_list(@tops[:most_asserted] ||= [], rec, max) {|_,_,a,_,_|a<assertions} + end + end + # (((@record ||= {})[suite] ||= {})[method]) = [assertions, time, error] + super + end + + def run(*args) + result = super + if @tops ||= nil + @tops.each do |t, list| + if list + puts "#{t.to_s.tr('_', ' ')} tests:" + list.each {|suite, method, assertions, time, error| + printf "%5.2fsec(%d): %s#%s\n", time, assertions, suite, method + } + end + end + end + result + end + + private + def setup_options(opts, options) + super + opts.separator "statistics options:" + opts.on '--longest=N', Integer, 'Show longest N tests' do |n| + options[:longest] = n + end + opts.on '--most-asserted=N', Integer, 'Show most asserted N tests' do |n| + options[:most_asserted] = n + end + end + end + + module StatusLine # :nodoc: all + def terminal_width + unless @terminal_width ||= nil + begin + require 'io/console' + width = $stdout.winsize[1] + rescue LoadError, NoMethodError, Errno::ENOTTY, Errno::EBADF, Errno::EINVAL + width = ENV["COLUMNS"].to_i.nonzero? || 80 + end + width -= 1 if /mswin|mingw/ =~ RUBY_PLATFORM + @terminal_width = width + end + @terminal_width + end + + def del_status_line(flush = true) + @status_line_size ||= 0 + if @options[:job_status] == :replace + $stdout.print "\r"+" "*@status_line_size+"\r" + else + $stdout.puts if @status_line_size > 0 + end + $stdout.flush if flush + @status_line_size = 0 + end + + def add_status(line) + @status_line_size ||= 0 + if @options[:job_status] == :replace + line = line[0...(terminal_width-@status_line_size)] + end + print line + @status_line_size += line.size + end + + def jobs_status(worker) + return if !@options[:job_status] or @verbose + if @options[:job_status] == :replace + status_line = @workers.map(&:to_s).join(" ") + else + status_line = worker.to_s + end + update_status(status_line) or (puts; nil) + end + + def del_jobs_status + return unless @options[:job_status] == :replace && @status_line_size.nonzero? + del_status_line + end + + def output + (@output ||= nil) || super + end + + def _prepare_run(suites, type) + options[:job_status] ||= @tty ? :replace : :normal unless @verbose + case options[:color] + when :always + color = true + when :auto, nil + color = true if @tty || @options[:job_status] == :replace + else + color = false + end + @colorize = Colorize.new(color, colors_file: File.join(__dir__, "../../colors")) + if color or @options[:job_status] == :replace + @verbose = !options[:parallel] + end + @output = Output.new(self) unless @options[:testing] + filter = options[:filter] + type = "#{type}_methods" + total = suites.sum {|suite| + methods = suite.send(type) + if filter + methods.count {|method| filter === "#{suite}##{method}"} + else + methods.size + end + } + @test_count = 0 + @total_tests = total.to_s(10) + end + + def new_test(s) + @test_count += 1 + update_status(s) + end + + def update_status(s) + count = @test_count.to_s(10).rjust(@total_tests.size) + del_status_line(false) + add_status(@colorize.pass("[#{count}/#{@total_tests}]")) + add_status(" #{s}") + $stdout.print "\r" if @options[:job_status] == :replace and !@verbose + $stdout.flush + end + + def _print(s); $stdout.print(s); end + def succeed; del_status_line; end + + def failed(s) + return if s and @options[:job_status] != :replace + sep = "\n" + @report_count ||= 0 + report.each do |msg| + if msg.start_with? "Skipped:" + if @options[:hide_skip] + del_status_line + next + end + color = :skip + else + color = :fail + end + first, msg = msg.split(/$/, 2) + first = sprintf("%3d) %s", @report_count += 1, first) + @failed_output.print(sep, @colorize.decorate(first, color), msg, "\n") + sep = nil + end + report.clear + end + + def initialize + super + @tty = $stdout.tty? + end + + def run(*args) + result = super + puts "\nruby -v: #{RUBY_DESCRIPTION}" + result + end + + private + def setup_options(opts, options) + super + + opts.separator "status line options:" + + options[:job_status] = nil + + opts.on '--jobs-status [TYPE]', [:normal, :replace, :none], + "Show status of jobs every file; Disabled when --jobs isn't specified." do |type| + options[:job_status] = (type || :normal if type != :none) + end + + opts.on '--color[=WHEN]', + [:always, :never, :auto], + "colorize the output. WHEN defaults to 'always'", "or can be 'never' or 'auto'." do |c| + options[:color] = c || :always + end + + opts.on '--tty[=WHEN]', + [:yes, :no], + "force to output tty control. WHEN defaults to 'yes'", "or can be 'no'." do |c| + @tty = c != :no + end + end + + class Output < Struct.new(:runner) # :nodoc: all + def puts(*a) $stdout.puts(*a) unless a.empty? end + def respond_to_missing?(*a) $stdout.respond_to?(*a) end + def method_missing(*a, &b) $stdout.__send__(*a, &b) end + + def print(s) + case s + when /\A(.*\#.*) = \z/ + runner.new_test($1) + when /\A(.* s) = \z/ + runner.add_status(" = #$1") + when /\A\.+\z/ + runner.succeed + when /\A\.*[EFST][EFST.]*\z/ + runner.failed(s) + else + $stdout.print(s) + end + end + end + end + + module LoadPathOption # :nodoc: all + def non_options(files, options) + begin + require "rbconfig" + rescue LoadError + warn "#{caller(1, 1)[0]}: warning: Parallel running disabled because can't get path to ruby; run specify with --ruby argument" + options[:parallel] = nil + else + options[:ruby] ||= [RbConfig.ruby] + end + + super + end + + def setup_options(parser, options) + super + parser.separator "load path options:" + parser.on '-Idirectory', 'Add library load path' do |dirs| + dirs.split(':').each { |d| $LOAD_PATH.unshift d } + end + end + end + + module GlobOption # :nodoc: all + @@testfile_prefix = "test" + @@testfile_suffix = "test" + + def setup_options(parser, options) + super + parser.separator "globbing options:" + parser.on '-B', '--base-directory DIR', 'Base directory to glob.' do |dir| + raise OptionParser::InvalidArgument, "not a directory: #{dir}" unless File.directory?(dir) + options[:base_directory] = dir + end + parser.on '-x', '--exclude REGEXP', 'Exclude test files on pattern.' do |pattern| + (options[:reject] ||= []) << pattern + end + end + + def complement_test_name f, orig_f + basename = File.basename(f) + + if /\.rb\z/ !~ basename + return File.join(File.dirname(f), basename+'.rb') + elsif /\Atest_/ !~ basename + return File.join(File.dirname(f), 'test_'+basename) + end if f.end_with?(basename) # otherwise basename is dirname/ + + raise ArgumentError, "file not found: #{orig_f}" + end + + def non_options(files, options) + paths = [options.delete(:base_directory), nil].uniq + if reject = options.delete(:reject) + reject_pat = Regexp.union(reject.map {|r| %r"#{r}"}) + end + files.map! {|f| + f = f.tr(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR + orig_f = f + while true + ret = ((paths if /\A\.\.?(?:\z|\/)/ !~ f) || [nil]).any? do |prefix| + if prefix + path = f.empty? ? prefix : "#{prefix}/#{f}" + else + next if f.empty? + path = f + end + if f.end_with?(File::SEPARATOR) or !f.include?(File::SEPARATOR) or File.directory?(path) + match = (Dir["#{path}/**/#{@@testfile_prefix}_*.rb"] + Dir["#{path}/**/*_#{@@testfile_suffix}.rb"]).uniq + else + match = Dir[path] + end + if !match.empty? + if reject + match.reject! {|n| + n = n[(prefix.length+1)..-1] if prefix + reject_pat =~ n + } + end + break match + elsif !reject or reject_pat !~ f and File.exist? path + break path + end + end + if !ret + f = complement_test_name(f, orig_f) + else + break ret + end + end + } + files.flatten! + super(files, options) + end + end + + module OutputOption # :nodoc: all + def setup_options(parser, options) + super + parser.separator "output options:" + + options[:failed_output] = $stdout + parser.on '--stderr-on-failure', 'Use stderr to print failure messages' do + options[:failed_output] = $stderr + end + parser.on '--stdout-on-failure', 'Use stdout to print failure messages', '(default)' do + options[:failed_output] = $stdout + end + end + + def process_args(args = []) + return @options if @options + options = super + @failed_output = options[:failed_output] + options + end + end + + module GCOption # :nodoc: all + def setup_options(parser, options) + super + parser.separator "GC options:" + parser.on '--[no-]gc-stress', 'Set GC.stress as true' do |flag| + options[:gc_stress] = flag + end + parser.on '--[no-]gc-compact', 'GC.compact every time' do |flag| + options[:gc_compact] = flag + end + end + + def non_options(files, options) + if options.delete(:gc_stress) + Test::Unit::TestCase.class_eval do + oldrun = instance_method(:run) + define_method(:run) do |runner| + begin + gc_stress, GC.stress = GC.stress, true + oldrun.bind_call(self, runner) + ensure + GC.stress = gc_stress + end + end + end + end + if options.delete(:gc_compact) + Test::Unit::TestCase.class_eval do + oldrun = instance_method(:run) + define_method(:run) do |runner| + begin + oldrun.bind_call(self, runner) + ensure + GC.compact + end + end + end + end + super + end + end + + module RequireFiles # :nodoc: all + def non_options(files, options) + return false if !super + errors = {} + result = false + files.each {|f| + d = File.dirname(path = File.realpath(f)) + unless $:.include? d + $: << d + end + begin + require path unless options[:parallel] + result = true + rescue LoadError + next if errors[$!.message] + errors[$!.message] = true + puts "#{f}: #{$!}" + end + } + @load_failed = errors.size.nonzero? + result + end + + def run(*) + super or @load_failed + end + end + + module RepeatOption # :nodoc: all + def setup_options(parser, options) + super + options[:repeat_count] = nil + parser.separator "repeat options:" + parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| + options[:repeat_count] = n + end + options[:keep_repeating] = false + parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n| + options[:keep_repeating] = true + end + end + + def _run_anything(type) + @repeat_count = @options[:repeat_count] + @keep_repeating = @options[:keep_repeating] + super + end + end + + module ExcludesOption # :nodoc: all + class ExcludedMethods < Struct.new(:excludes) + def exclude(name, reason) + excludes[name] = reason + end + + def exclude_from(klass) + excludes = self.excludes + pattern = excludes.keys.grep(Regexp).tap {|k| + break (Regexp.new(k.join('|')) unless k.empty?) + } + klass.class_eval do + public_instance_methods(false).each do |method| + if excludes[method] or (pattern and pattern =~ method) + remove_method(method) + end + end + public_instance_methods(true).each do |method| + if excludes[method] or (pattern and pattern =~ method) + undef_method(method) + end + end + end + end + + def self.load(dirs, name) + return unless dirs and name + instance = nil + dirs.each do |dir| + path = File.join(dir, name.gsub(/::/, '/') + ".rb") + begin + src = File.read(path) + rescue Errno::ENOENT + nil + else + instance ||= new({}) + instance.instance_eval(src, path) + end + end + instance + end + end + + def setup_options(parser, options) + super + if excludes = ENV["EXCLUDES"] + excludes = excludes.split(File::PATH_SEPARATOR) + end + options[:excludes] = excludes || [] + parser.separator "excludes options:" + parser.on '-X', '--excludes-dir DIRECTORY', "Directory name of exclude files" do |d| + options[:excludes].concat d.split(File::PATH_SEPARATOR) + end + end + + def _run_suite(suite, type) + if ex = ExcludedMethods.load(@options[:excludes], suite.name) + ex.exclude_from(suite) + end + super + end + end + + module TimeoutOption + def setup_options(parser, options) + super + parser.separator "timeout options:" + parser.on '--timeout-scale NUM', '--subprocess-timeout-scale NUM', "Scale timeout", Float do |scale| + raise OptionParser::InvalidArgument, "timeout scale must be positive" unless scale > 0 + options[:timeout_scale] = scale + end + end + + def non_options(files, options) + if scale = options[:timeout_scale] or + (scale = ENV["RUBY_TEST_TIMEOUT_SCALE"] || ENV["RUBY_TEST_SUBPROCESS_TIMEOUT_SCALE"] and + (scale = scale.to_f) > 0) + EnvUtil.timeout_scale = scale + end + super + end + end + + module LaunchableOption + module Nothing + private + def setup_options(opts, options) + super + opts.define_tail 'Launchable options:' + # This is expected to be called by Test::Unit::Worker. + opts.on_tail '--launchable-test-reports=PATH', String, 'Do nothing' + end + end + + def record(suite, method, assertions, time, error, source_location = nil) + if writer = @options[:launchable_test_reports] + if loc = (source_location || suite.instance_method(method).source_location) + path, lineno = loc + # Launchable JSON schema is defined at + # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code. + e = case error + when nil + status = 'TEST_PASSED' + nil + when Test::Unit::PendedError + status = 'TEST_SKIPPED' + "Skipped:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Test::Unit::AssertionFailedError + status = 'TEST_FAILED' + "Failure:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n" + when Timeout::Error + status = 'TEST_FAILED' + "Timeout:\n#{suite.name}##{method}\n" + else + status = 'TEST_FAILED' + bt = Test::filter_backtrace(error.backtrace).join "\n " + "Error:\n#{suite.name}##{method}:\n#{error.class}: #{error.message.b}\n #{bt}\n" + end + repo_path = File.expand_path("#{__dir__}/../../../") + relative_path = path.delete_prefix("#{repo_path}/") + # The test path is a URL-encoded representation. + # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18 + test_path = {file: relative_path, class: suite.name, testcase: method}.map{|key, val| + "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}" + }.join('#') + end + end + super + ensure + if writer && test_path && status + # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified. + # In such cases, we proceed to execute the operation here. + writer.write_object( + { + testPath: test_path, + status: status, + duration: time, + createdAt: Time.now.to_s, + stderr: e, + stdout: nil, + data: { + lineNumber: lineno + } + } + ) + end + end + + private + def setup_options(opts, options) + super + opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path| + require_relative '../launchable' + options[:launchable_test_reports] = writer = Launchable::JsonStreamWriter.new(path) + writer.write_array('testCases') + main_pid = Process.pid + at_exit { + # This block is executed when the fork block in a test is completed. + # Therefore, we need to verify whether all tests have been completed. + stack = caller + if stack.size == 0 && main_pid == Process.pid && $!.is_a?(SystemExit) + writer.close + end + } + end + + def encode_test_path_component component + component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26') + end + end + end + + class Runner # :nodoc: all + + attr_accessor :report, :failures, :errors, :skips # :nodoc: + attr_accessor :assertion_count # :nodoc: + attr_writer :test_count # :nodoc: + attr_accessor :start_time # :nodoc: + attr_accessor :help # :nodoc: + attr_accessor :verbose # :nodoc: + attr_writer :options # :nodoc: + + ## + # :attr: + # + # if true, installs an "INFO" signal handler (only available to BSD and + # OS X users) which prints diagnostic information about the test run. + # + # This is auto-detected by default but may be overridden by custom + # runners. + + attr_accessor :info_signal + + ## + # Lazy accessor for options. + + def options + @options ||= {seed: 42} + end + + @@installed_at_exit ||= false + @@out = $stdout + @@after_tests = [] + @@current_repeat_count = 0 + + ## + # A simple hook allowing you to run a block of code after _all_ of + # the tests are done. Eg: + # + # Test::Unit::Runner.after_tests { p $debugging_info } + + def self.after_tests &block + @@after_tests << block + end + + ## + # Returns the stream to use for output. + + def self.output + @@out + end + + ## + # Sets Test::Unit::Runner to write output to +stream+. $stdout is the default + # output + + def self.output= stream + @@out = stream + end + + ## + # Tells Test::Unit::Runner to delegate to +runner+, an instance of a + # Test::Unit::Runner subclass, when Test::Unit::Runner#run is called. + + def self.runner= runner + @@runner = runner + end + + ## + # Returns the Test::Unit::Runner subclass instance that will be used + # to run the tests. A Test::Unit::Runner instance is the default + # runner. + + def self.runner + @@runner ||= self.new + end + + ## + # Return all plugins' run methods (methods that start with "run_"). + + def self.plugins + @@plugins ||= (["run_tests"] + + public_instance_methods(false). + grep(/^run_/).map { |s| s.to_s }).uniq + end + + ## + # Return the IO for output. + + def output + self.class.output + end + + def puts *a # :nodoc: + output.puts(*a) + end + + def print *a # :nodoc: + output.print(*a) + end + + def test_count # :nodoc: + @test_count ||= 0 + end + + ## + # Runner for a given +type+ (eg, test vs bench). + + def self.current_repeat_count + @@current_repeat_count + end + + def _run_anything type + suites = Test::Unit::TestCase.send "#{type}_suites" + return if suites.empty? + + suites = @order.sort_by_name(suites) + + puts + puts "# Running #{type}s:" + puts + + @test_count, @assertion_count = 0, 0 + test_count = assertion_count = 0 + sync = output.respond_to? :"sync=" # stupid emacs + old_sync, output.sync = output.sync, true if sync + + @@current_repeat_count = 0 + begin + start = Time.now + + results = _run_suites suites, type + + @test_count = results.inject(0) { |sum, (tc, _)| sum + tc } + @assertion_count = results.inject(0) { |sum, (_, ac)| sum + ac } + test_count += @test_count + assertion_count += @assertion_count + t = Time.now - start + @@current_repeat_count += 1 + unless @repeat_count + puts + puts + end + puts "Finished%s %ss in %.6fs, %.4f tests/s, %.4f assertions/s.\n" % + [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, + t, @test_count.fdiv(t), @assertion_count.fdiv(t)] + end while @repeat_count && @@current_repeat_count < @repeat_count && + (@keep_repeating || report.empty? && failures.zero? && errors.zero?) + + output.sync = old_sync if sync + + report.each_with_index do |msg, i| + puts "\n%3d) %s" % [i + 1, msg] + end + + puts + @test_count = test_count + @assertion_count = assertion_count + + status + end + + ## + # Run a single +suite+ for a given +type+. + + def _run_suite suite, type + header = "#{type}_suite_header" + puts send(header, suite) if respond_to? header + + filter = options[:filter] + + all_test_methods = suite.send "#{type}_methods" + if filter + all_test_methods.select! {|method| + filter === "#{suite}##{method}" + } + end + all_test_methods = @order.sort_by_name(all_test_methods) + + leakchecker = LeakChecker.new + if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] + require "objspace" + trace = true + end + + assertions = all_test_methods.map { |method| + + inst = suite.new method + _start_method(inst) + inst._assertions = 0 + + print "#{suite}##{method.inspect.sub(/\A:/, '')} = " if @verbose + + start_time = Time.now if @verbose + result = + if trace + ObjectSpace.trace_object_allocations {inst.run self} + else + inst.run self + end + + print "%.2f s = " % (Time.now - start_time) if @verbose + print result + puts if @verbose + $stdout.flush + + leakchecker.check("#{inst.class}\##{inst.__name__}") + + _end_method(inst) + + inst._assertions + } + return assertions.size, assertions.inject(0) { |sum, n| sum + n } + end + + def _start_method(inst) + end + def _end_method(inst) + end + + ## + # Record the result of a single test. Makes it very easy to gather + # information. Eg: + # + # class StatisticsRecorder < Test::Unit::Runner + # def record suite, method, assertions, time, error + # # ... record the results somewhere ... + # end + # end + # + # Test::Unit::Runner.runner = StatisticsRecorder.new + # + # NOTE: record might be sent more than once per test. It will be + # sent once with the results from the test itself. If there is a + # failure or error in teardown, it will be sent again with the + # error or failure. + + def record suite, method, assertions, time, error, source_location = nil + end + + def location e # :nodoc: + Test::Unit.location e + end + + ## + # Writes status for failed test +meth+ in +klass+ which finished with + # exception +e+ + + def initialize # :nodoc: + @report = [] + @errors = @failures = @skips = 0 + @verbose = false + @mutex = Thread::Mutex.new + @info_signal = Signal.list['INFO'] + @repeat_count = nil + end + + def synchronize # :nodoc: + if @mutex then + @mutex.synchronize { yield } + else + yield + end + end + + def inspect + "#<#{self.class.name}: " << + instance_variables.filter_map do |var| + next if var == :@option_parser # too big + "#{var}=#{instance_variable_get(var).inspect}" + end.join(", ") << ">" + end + + ## + # Top level driver, controls all output and filtering. + + def _run args = [] + args = process_args args # ARGH!! blame test/unit process_args + self.options.merge! args + + puts "Run options: #{help}" + + self.class.plugins.each do |plugin| + send plugin + break unless report.empty? + end + + return (failures + errors).nonzero? # or return nil... + rescue Interrupt + abort 'Interrupted' + end + + ## + # Runs test suites matching +filter+. + + def run_tests + _run_anything :test + end + + ## + # Writes status to +io+ + + def status io = self.output + format = "%d tests, %d assertions, %d failures, %d errors, %d skips" + io.puts format % [test_count, assertion_count, failures, errors, skips] + end + + prepend Test::Unit::Options + prepend Test::Unit::StatusLine + prepend Test::Unit::Parallel + prepend Test::Unit::Statistics + prepend Test::Unit::Skipping + prepend Test::Unit::GlobOption + prepend Test::Unit::OutputOption + prepend Test::Unit::RepeatOption + prepend Test::Unit::LoadPathOption + prepend Test::Unit::GCOption + prepend Test::Unit::ExcludesOption + prepend Test::Unit::TimeoutOption + prepend Test::Unit::RunCount + prepend Test::Unit::LaunchableOption::Nothing + + ## + # Begins the full test run. Delegates to +runner+'s #_run method. + + def run(argv = []) + self.class.runner._run(argv) + rescue NoMemoryError + system("cat /proc/meminfo") if File.exist?("/proc/meminfo") + system("ps x -opid,args,%cpu,%mem,nlwp,rss,vsz,wchan,stat,start,time,etime,blocked,caught,ignored,pending,f") if File.exist?("/bin/ps") + raise + end + + @@stop_auto_run = false + def self.autorun + at_exit { + Test::Unit::RunCount.run_once { + exit(Test::Unit::Runner.new.run(ARGV) || true) + } unless @@stop_auto_run + } unless @@installed_at_exit + @@installed_at_exit = true + end + + alias orig_run_suite _run_suite + + # Overriding of Test::Unit::Runner#puke + def puke klass, meth, e + n = report.size + e = case e + when Test::Unit::PendedError then + @skips += 1 + return "S" unless @verbose + "Skipped:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Test::Unit::AssertionFailedError then + @failures += 1 + "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n" + when Timeout::Error + @errors += 1 + "Timeout:\n#{klass}##{meth}\n" + else + @errors += 1 + bt = Test::filter_backtrace(e.backtrace).join "\n " + "Error:\n#{klass}##{meth}:\n#{e.class}: #{e.message.b}\n #{bt}\n" + end + @report << e + rep = e[0, 1] + if Test::Unit::PendedError === e and /no message given\z/ =~ e.message + report.slice!(n..-1) + rep = "." + end + rep + end + end + + class AutoRunner # :nodoc: all + class Runner < Test::Unit::Runner + include Test::Unit::RequireFiles + include Test::Unit::LaunchableOption + end + + attr_accessor :to_run, :options + + def initialize(force_standalone = false, default_dir = nil, argv = ARGV) + @force_standalone = force_standalone + @runner = Runner.new do |files, options| + base = options[:base_directory] ||= default_dir + @runner.prefix = base ? (base + "/") : nil + files << default_dir if files.empty? and default_dir + @to_run = files + yield self if block_given? + $LOAD_PATH.unshift base if base + files + end + Runner.runner = @runner + @options = @runner.option_parser + if @force_standalone + @options.banner.sub!(/\[options\]/, '\& tests...') + end + @argv = argv + end + + def process_args(*args) + @runner.process_args(*args) + !@to_run.empty? + end + + def run + if @force_standalone and not process_args(@argv) + abort @options.banner + end + @runner.run(@argv) || true + end + + def self.run(*args) + new(*args).run + end + end + + class ProxyError < StandardError # :nodoc: all + def initialize(ex) + @message = ex.message + @backtrace = ex.backtrace + end + + attr_accessor :message, :backtrace + end + end +end + +Test::Unit::Runner.autorun diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb new file mode 100644 index 0000000000..19581fc3ab --- /dev/null +++ b/tool/lib/test/unit/assertions.rb @@ -0,0 +1,844 @@ +# frozen_string_literal: true +require 'pp' + +module Test + module Unit + module Assertions + + ## + # Returns the diff command to use in #diff. Tries to intelligently + # figure out what diff to use. + + def self.diff + unless defined? @diff + exe = RbConfig::CONFIG['EXEEXT'] + @diff = %W"gdiff#{exe} diff#{exe}".find do |diff| + if system(diff, "-u", __FILE__, __FILE__) + break "#{diff} -u" + end + end + end + + @diff + end + + ## + # Set the diff command to use in #diff. + + def self.diff= o + @diff = o + end + + ## + # Returns a diff between +exp+ and +act+. If there is no known + # diff command or if it doesn't make sense to diff the output + # (single line, short output), then it simply returns a basic + # comparison between the two. + + def diff exp, act + require "tempfile" + + expect = mu_pp_for_diff exp + butwas = mu_pp_for_diff act + result = nil + + need_to_diff = + self.class.diff && + (expect.include?("\n") || + butwas.include?("\n") || + expect.size > 30 || + butwas.size > 30 || + expect == butwas) + + return "Expected: #{mu_pp exp}\n Actual: #{mu_pp act}" unless + need_to_diff + + tempfile_a = nil + tempfile_b = nil + + Tempfile.open("expect") do |a| + tempfile_a = a + a.puts expect + a.flush + + Tempfile.open("butwas") do |b| + tempfile_b = b + b.puts butwas + b.flush + + result = `#{self.class.diff} #{a.path} #{b.path}` + result.sub!(/^\-\-\- .+/, "--- expected") + result.sub!(/^\+\+\+ .+/, "+++ actual") + + if result.empty? then + klass = exp.class + result = [ + "No visible difference in the #{klass}#inspect output.\n", + "You should look at the implementation of #== on ", + "#{klass} or its members.\n", + expect, + ].join + end + end + end + + result + ensure + tempfile_a.close! if tempfile_a + tempfile_b.close! if tempfile_b + end + + ## + # This returns a diff-able human-readable version of +obj+. This + # differs from the regular mu_pp because it expands escaped + # newlines and makes hex-values generic (like object_ids). This + # uses mu_pp to do the first pass and then cleans it up. + + def mu_pp_for_diff obj + mu_pp(obj).gsub(/(?<!\\)(?:\\\\)*\K\\n/, "\n").gsub(/:0x[a-fA-F0-9]{4,}/m, ':0xXXXXXX') + end + + ## + # Fails unless +test+ is a true value. + + def assert test, msg = nil + msg ||= "Failed assertion, no message given." + self._assertions += 1 + unless test then + msg = msg.call if Proc === msg + raise Test::Unit::AssertionFailedError, msg + end + true + end + + ## + # Fails unless +obj+ is empty. + + def assert_empty obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be empty" } + assert_respond_to obj, :empty? + assert obj.empty?, msg + end + + ## + # For comparing Floats. Fails unless +exp+ and +act+ are within +delta+ + # of each other. + # + # assert_in_delta Math::PI, (22.0 / 7.0), 0.01 + + def assert_in_delta exp, act, delta = 0.001, msg = nil + n = (exp - act).abs + msg = message(msg) { + "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}" + } + assert delta >= n, msg + end + + ## + # For comparing Floats. Fails unless +exp+ and +act+ have a relative + # error less than +epsilon+. + + def assert_in_epsilon a, b, epsilon = 0.001, msg = nil + assert_in_delta a, b, [a.abs, b.abs].min * epsilon, msg + end + + ## + # Fails unless +collection+ includes +obj+. + + def assert_includes collection, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(collection)} to include #{mu_pp(obj)}" + } + assert_respond_to collection, :include? + assert collection.include?(obj), msg + end + + ## + # Fails unless +obj+ is an instance of +cls+. + + def assert_instance_of cls, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} to be an instance of #{cls}, not #{obj.class}" + } + + assert obj.instance_of?(cls), msg + end + + ## + # Fails unless +obj+ is a kind of +cls+. + + def assert_kind_of cls, obj, msg = nil # TODO: merge with instance_of + msg = message(msg) { + "Expected #{mu_pp(obj)} to be a kind of #{cls}, not #{obj.class}" } + + assert obj.kind_of?(cls), msg + end + + ## + # Fails unless +matcher+ <tt>=~</tt> +obj+. + + def assert_match matcher, obj, msg = nil + msg = message(msg) { "Expected #{mu_pp matcher} to match #{mu_pp obj}" } + assert_respond_to matcher, :"=~" + matcher = Regexp.new Regexp.escape matcher if String === matcher + assert matcher =~ obj, msg + end + + ## + # Fails unless +obj+ is nil + + def assert_nil obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be nil" } + assert obj.nil?, msg + end + + ## + # Fails unless +obj+ is true + + def assert_true obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be true" } + assert obj == true, msg + end + + ## + # Fails unless +obj+ is false + + def assert_false obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to be false" } + assert obj == false, msg + end + + ## + # For testing with binary operators. + # + # assert_operator 5, :<=, 4 + + def assert_operator o1, op, o2 = (predicate = true; nil), msg = nil + return assert_predicate o1, op, msg if predicate + msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op} #{mu_pp(o2)}" } + assert o1.__send__(op, o2), msg + end + + ## + # Fails if stdout or stderr do not output the expected results. + # Pass in nil if you don't care about that streams output. Pass in + # "" if you require it to be silent. Pass in a regexp if you want + # to pattern match. + # + # NOTE: this uses #capture_io, not #capture_subprocess_io. + # + # See also: #assert_silent + + def assert_output stdout = nil, stderr = nil + out, err = capture_output do + yield + end + + err_msg = Regexp === stderr ? :assert_match : :assert_equal if stderr + out_msg = Regexp === stdout ? :assert_match : :assert_equal if stdout + + y = send err_msg, stderr, err, "In stderr" if err_msg + x = send out_msg, stdout, out, "In stdout" if out_msg + + (!stdout || x) && (!stderr || y) + end + + ## + # For testing with predicates. + # + # assert_predicate str, :empty? + # + # This is really meant for specs and is front-ended by assert_operator: + # + # str.must_be :empty? + + def assert_predicate o1, op, msg = nil + msg = message(msg) { "Expected #{mu_pp(o1)} to be #{op}" } + assert o1.__send__(op), msg + end + + ## + # Fails unless +obj+ responds to +meth+. + + def assert_respond_to obj, meth, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} (#{obj.class}) to respond to ##{meth}" + } + assert obj.respond_to?(meth), msg + end + + ## + # Fails unless +exp+ and +act+ are #equal? + + def assert_same exp, act, msg = nil + msg = message(msg) { + data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] + "Expected %s (oid=%d) to be the same as %s (oid=%d)" % data + } + assert exp.equal?(act), msg + end + + ## + # Fails if the block outputs anything to stderr or stdout. + # + # See also: #assert_output + + def assert_silent + assert_output "", "" do + yield + end + end + + ## + # Fails unless the block throws +sym+ + + def assert_throws sym, msg = nil + default = "Expected #{mu_pp(sym)} to have been thrown" + caught = true + catch(sym) do + begin + yield + rescue ThreadError => e # wtf?!? 1.8 + threads == suck + default += ", not \:#{e.message[/uncaught throw \`(\w+?)\'/, 1]}" + rescue ArgumentError => e # 1.9 exception + default += ", not #{e.message.split(/ /).last}" + rescue NameError => e # 1.8 exception + default += ", not #{e.name.inspect}" + end + caught = false + end + + assert caught, message(msg) { default } + end + + def assert_path_exists(path, msg = nil) + msg = message(msg) { "Expected path '#{path}' to exist" } + assert File.exist?(path), msg + end + alias assert_path_exist assert_path_exists + alias refute_path_not_exist assert_path_exists + + def refute_path_exists(path, msg = nil) + msg = message(msg) { "Expected path '#{path}' to not exist" } + refute File.exist?(path), msg + end + alias refute_path_exist refute_path_exists + alias assert_path_not_exist refute_path_exists + + ## + # Captures $stdout and $stderr into strings: + # + # out, err = capture_output do + # puts "Some info" + # warn "You did a bad thing" + # end + # + # assert_match %r%info%, out + # assert_match %r%bad%, err + + def capture_output + require 'stringio' + + captured_stdout, captured_stderr = StringIO.new, StringIO.new + + synchronize do + orig_stdout, orig_stderr = $stdout, $stderr + $stdout, $stderr = captured_stdout, captured_stderr + + begin + yield + ensure + $stdout = orig_stdout + $stderr = orig_stderr + end + end + + return captured_stdout.string, captured_stderr.string + end + + def capture_io + raise NoMethodError, "use capture_output" + end + + ## + # Fails with +msg+ + + def flunk msg = nil + msg ||= "Epic Fail!" + assert false, msg + end + + ## + # used for counting assertions + + def pass msg = nil + assert true + end + + ## + # Fails if +test+ is a true value + + def refute test, msg = nil + msg ||= "Failed refutation, no message given" + not assert(! test, msg) + end + + ## + # Fails if +obj+ is empty. + + def refute_empty obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not be empty" } + assert_respond_to obj, :empty? + refute obj.empty?, msg + end + + ## + # Fails if <tt>exp == act</tt>. + # + # For floats use refute_in_delta. + + def refute_equal exp, act, msg = nil + msg = message(msg) { + "Expected #{mu_pp(act)} to not be equal to #{mu_pp(exp)}" + } + refute exp == act, msg + end + + ## + # For comparing Floats. Fails if +exp+ is within +delta+ of +act+. + # + # refute_in_delta Math::PI, (22.0 / 7.0) + + def refute_in_delta exp, act, delta = 0.001, msg = nil + n = (exp - act).abs + msg = message(msg) { + "Expected |#{exp} - #{act}| (#{n}) to not be <= #{delta}" + } + refute delta >= n, msg + end + + ## + # For comparing Floats. Fails if +exp+ and +act+ have a relative error + # less than +epsilon+. + + def refute_in_epsilon a, b, epsilon = 0.001, msg = nil + refute_in_delta a, b, a * epsilon, msg + end + + ## + # Fails if +collection+ includes +obj+. + + def refute_includes collection, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(collection)} to not include #{mu_pp(obj)}" + } + assert_respond_to collection, :include? + refute collection.include?(obj), msg + end + + ## + # Fails if +obj+ is an instance of +cls+. + + def refute_instance_of cls, obj, msg = nil + msg = message(msg) { + "Expected #{mu_pp(obj)} to not be an instance of #{cls}" + } + refute obj.instance_of?(cls), msg + end + + ## + # Fails if +obj+ is a kind of +cls+. + + def refute_kind_of cls, obj, msg = nil # TODO: merge with instance_of + msg = message(msg) { "Expected #{mu_pp(obj)} to not be a kind of #{cls}" } + refute obj.kind_of?(cls), msg + end + + ## + # Fails if +matcher+ <tt>=~</tt> +obj+. + + def refute_match matcher, obj, msg = nil + msg = message(msg) {"Expected #{mu_pp matcher} to not match #{mu_pp obj}"} + assert_respond_to matcher, :"=~" + matcher = Regexp.new Regexp.escape matcher if String === matcher + refute matcher =~ obj, msg + end + + ## + # Fails if +obj+ is nil. + + def refute_nil obj, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not be nil" } + refute obj.nil?, msg + end + + ## + # Fails if +o1+ is not +op+ +o2+. Eg: + # + # refute_operator 1, :>, 2 #=> pass + # refute_operator 1, :<, 2 #=> fail + + def refute_operator o1, op, o2 = (predicate = true; nil), msg = nil + return refute_predicate o1, op, msg if predicate + msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op} #{mu_pp(o2)}"} + refute o1.__send__(op, o2), msg + end + + ## + # For testing with predicates. + # + # refute_predicate str, :empty? + # + # This is really meant for specs and is front-ended by refute_operator: + # + # str.wont_be :empty? + + def refute_predicate o1, op, msg = nil + msg = message(msg) { "Expected #{mu_pp(o1)} to not be #{op}" } + refute o1.__send__(op), msg + end + + ## + # Fails if +obj+ responds to the message +meth+. + + def refute_respond_to obj, meth, msg = nil + msg = message(msg) { "Expected #{mu_pp(obj)} to not respond to #{meth}" } + + refute obj.respond_to?(meth), msg + end + + ## + # Fails if +exp+ is the same (by object identity) as +act+. + + def refute_same exp, act, msg = nil + msg = message(msg) { + data = [mu_pp(act), act.object_id, mu_pp(exp), exp.object_id] + "Expected %s (oid=%d) to not be the same as %s (oid=%d)" % data + } + refute exp.equal?(act), msg + end + + ## + # Skips the current test. Gets listed at the end of the run but + # doesn't cause a failure exit code. + + def pend msg = nil, bt = caller, &_ + msg ||= "Skipped, no message given" + @skip = true + raise Test::Unit::PendedError, msg, bt + end + alias omit pend + + def skip(msg = nil, bt = caller) + raise NoMethodError, "use omit or pend", caller + end + + ## + # Was this testcase skipped? Meant for #teardown. + + def skipped? + defined?(@skip) and @skip + end + + ## + # Takes a block and wraps it with the runner's shared mutex. + + def synchronize + Test::Unit::Runner.runner.synchronize do + yield + end + end + + # :call-seq: + # assert_block( failure_message = nil ) + # + #Tests the result of the given block. If the block does not return true, + #the assertion will fail. The optional +failure_message+ argument is the same as in + #Assertions#assert. + # + # assert_block do + # [1, 2, 3].any? { |num| num < 1 } + # end + def assert_block(*msgs) + assert yield, *msgs + end + + # :call-seq: + # assert_nothing_thrown( failure_message = nil, &block ) + # + #Fails if the given block uses a call to Kernel#throw, and + #returns the result of the block otherwise. + # + #An optional failure message may be provided as the final argument. + # + # assert_nothing_thrown "Something was thrown!" do + # throw :problem? + # end + def assert_nothing_thrown(msg=nil) + begin + ret = yield + rescue ArgumentError => error + raise error if /\Auncaught throw (.+)\z/m !~ error.message + msg = message(msg) { "<#{$1}> was thrown when nothing was expected" } + flunk(msg) + end + assert(true, "Expected nothing to be thrown") + ret + end + + # :call-seq: + # assert_equal( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is equal to +actual+. + # + #An optional failure message may be provided as the final argument. + def assert_equal(exp, act, msg = nil) + msg = message(msg) { + exp_str = mu_pp(exp) + act_str = mu_pp(act) + exp_comment = '' + act_comment = '' + if exp_str == act_str + if (exp.is_a?(String) && act.is_a?(String)) || + (exp.is_a?(Regexp) && act.is_a?(Regexp)) + exp_comment = " (#{exp.encoding})" + act_comment = " (#{act.encoding})" + elsif exp.is_a?(Float) && act.is_a?(Float) + exp_str = "%\#.#{Float::DIG+2}g" % exp + act_str = "%\#.#{Float::DIG+2}g" % act + elsif exp.is_a?(Time) && act.is_a?(Time) + if exp.subsec * 1000_000_000 == exp.nsec + exp_comment = " (#{exp.nsec}[ns])" + else + exp_comment = " (subsec=#{exp.subsec})" + end + if act.subsec * 1000_000_000 == act.nsec + act_comment = " (#{act.nsec}[ns])" + else + act_comment = " (subsec=#{act.subsec})" + end + elsif exp.class != act.class + # a subclass of Range, for example. + exp_comment = " (#{exp.class})" + act_comment = " (#{act.class})" + end + elsif !Encoding.compatible?(exp_str, act_str) + if exp.is_a?(String) && act.is_a?(String) + exp_str = exp.dump + act_str = act.dump + exp_comment = " (#{exp.encoding})" + act_comment = " (#{act.encoding})" + else + exp_str = exp_str.dump + act_str = act_str.dump + end + end + "<#{exp_str}>#{exp_comment} expected but was\n<#{act_str}>#{act_comment}" + } + assert(exp == act, msg) + end + + # :call-seq: + # assert_not_nil( expression, failure_message = nil ) + # + #Tests if +expression+ is not nil. + # + #An optional failure message may be provided as the final argument. + def assert_not_nil(exp, msg=nil) + msg = message(msg) { "<#{mu_pp(exp)}> expected to not be nil" } + assert(!exp.nil?, msg) + end + + # :call-seq: + # assert_not_equal( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is not equal to +actual+. + # + #An optional failure message may be provided as the final argument. + def assert_not_equal(exp, act, msg=nil) + msg = message(msg) { "<#{mu_pp(exp)}> expected to be != to\n<#{mu_pp(act)}>" } + assert(exp != act, msg) + end + + # :call-seq: + # assert_no_match( regexp, string, failure_message = nil ) + # + #Tests if the given Regexp does not match a given String. + # + #An optional failure message may be provided as the final argument. + def assert_no_match(regexp, string, msg=nil) + assert_instance_of(Regexp, regexp, "The first argument to assert_no_match should be a Regexp.") + self._assertions -= 1 + msg = message(msg) { "<#{mu_pp(regexp)}> expected to not match\n<#{mu_pp(string)}>" } + assert(regexp !~ string, msg) + end + + # :call-seq: + # assert_not_same( expected, actual, failure_message = nil ) + # + #Tests if +expected+ is not the same object as +actual+. + #This test uses Object#equal? to test equality. + # + #An optional failure message may be provided as the final argument. + # + # assert_not_same("x", "x") #Succeeds + def assert_not_same(expected, actual, message="") + msg = message(msg) { build_message(message, <<EOT, expected, expected.__id__, actual, actual.__id__) } +<?> +with id <?> expected to not be equal\\? to +<?> +with id <?>. +EOT + assert(!actual.equal?(expected), msg) + end + + # :call-seq: + # assert_send( +send_array+, failure_message = nil ) + # + # Passes if the method send returns a true value. + # + # +send_array+ is composed of: + # * A receiver + # * A method + # * Arguments to the method + # + # Example: + # assert_send(["Hello world", :include?, "Hello"]) # -> pass + # assert_send(["Hello world", :include?, "Goodbye"]) # -> fail + def assert_send send_ary, m = nil + recv, msg, *args = send_ary + m = message(m) { + if args.empty? + argsstr = "" + else + (argsstr = mu_pp(args)).sub!(/\A\[(.*)\]\z/m, '(\1)') + end + "Expected #{mu_pp(recv)}.#{msg}#{argsstr} to return true" + } + assert recv.__send__(msg, *args), m + end + + # :call-seq: + # assert_not_send( +send_array+, failure_message = nil ) + # + # Passes if the method send doesn't return a true value. + # + # +send_array+ is composed of: + # * A receiver + # * A method + # * Arguments to the method + # + # Example: + # assert_not_send([[1, 2], :member?, 1]) # -> fail + # assert_not_send([[1, 2], :member?, 4]) # -> pass + def assert_not_send send_ary, m = nil + recv, msg, *args = send_ary + m = message(m) { + if args.empty? + argsstr = "" + else + (argsstr = mu_pp(args)).sub!(/\A\[(.*)\]\z/m, '(\1)') + end + "Expected #{mu_pp(recv)}.#{msg}#{argsstr} to return false" + } + assert !recv.__send__(msg, *args), m + end + + ms = instance_methods(true).map {|sym| sym.to_s } + ms.grep(/\Arefute_/) do |m| + mname = ('assert_not_'.dup << m.to_s[/.*?_(.*)/, 1]) + alias_method(mname, m) unless ms.include? mname + end + alias assert_include assert_includes + alias assert_not_include assert_not_includes + + def assert_not_all?(obj, m = nil, &blk) + failed = [] + obj.each do |*a, &b| + if blk.call(*a, &b) + failed << (a.size > 1 ? a : a[0]) + end + end + assert(failed.empty?, message(m) {failed.pretty_inspect}) + end + + def assert_syntax_error(code, error, *args, **opt) + prepare_syntax_check(code, *args, **opt) do |src, fname, line, mesg| + yield if defined?(yield) + e = assert_raise(SyntaxError, mesg) do + syntax_check(src, fname, line) + end + + # Prism adds ANSI escape sequences to syntax error messages to + # colorize and format them. We strip them out here to make them easier + # to match against in tests. + message = e.message + message.gsub!(/\e\[.*?m/, "") + + assert_match(error, message, mesg) + e + end + end + + def assert_no_warning(pat, msg = nil) + result = nil + stderr = EnvUtil.verbose_warning { + EnvUtil.with_default_internal(pat.encoding) { + result = yield + } + } + msg = message(msg) {diff pat, stderr} + refute(pat === stderr, msg) + result + end + + # kernel resolution can limit the minimum time we can measure + # [ruby-core:81540] + MIN_HZ = /mswin|mingw/ =~ RUBY_PLATFORM ? 67 : 100 + MIN_MEASURABLE = 1.0 / MIN_HZ + + def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) + wait = EnvUtil.apply_timeout_scale(wait) + if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c + warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" + end + + t0, r0 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error + end + end + + t1, r1 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + total = t1.utime - t0.utime + t1.stime - t0.stime + t1.cutime - t0.cutime + t1.cstime - t0.cstime + real = r1 - r0 + + max = pct * real + min_measurable = MIN_MEASURABLE + min_measurable *= 1.30 # add a little (30%) to account for misc. overheads + if max < min_measurable + max = min_measurable + end + + assert_operator total, :<=, max, msg + end + + def assert_is_minus_zero(f) + assert(1.0/f == -Float::INFINITY, "#{f} is not -0.0") + end + + def build_message(head, template=nil, *arguments) #:nodoc: + template &&= template.chomp + template.gsub(/\G((?:[^\\]|\\.)*?)(\\)?\?/) { $1 + ($2 ? "?" : mu_pp(arguments.shift)) } + end + end + end +end diff --git a/tool/lib/test/unit/parallel.rb b/tool/lib/test/unit/parallel.rb new file mode 100644 index 0000000000..188a0d1a19 --- /dev/null +++ b/tool/lib/test/unit/parallel.rb @@ -0,0 +1,221 @@ +# frozen_string_literal: true + +require_relative "../../../test/init" + +module Test + module Unit + class Worker < Runner # :nodoc: + class << self + undef autorun + end + + undef _run_suite + undef _run_suites + undef run + + def increment_io(orig) # :nodoc: + *rest, io = 32.times.inject([orig.dup]){|ios, | ios << ios.last.dup } + rest.each(&:close) + io + end + + def _run_suites(suites, type) # :nodoc: + suites.map do |suite| + _run_suite(suite, type) + end + end + + def _start_method(inst) + _report "start", Marshal.dump([inst.class.name, inst.__name__]) + end + + def _run_suite(suite, type) # :nodoc: + @partial_report = [] + orig_testout = Test::Unit::Runner.output + i,o = IO.pipe + + Test::Unit::Runner.output = o + orig_stdin, orig_stdout = $stdin, $stdout + + th = Thread.new do + begin + while buf = (self.verbose ? i.gets : i.readpartial(1024)) + _report "p", buf or break + end + rescue IOError + end + end + + e, f, s = @errors, @failures, @skips + + begin + result = orig_run_suite(suite, type) + rescue Interrupt + @need_exit = true + result = [nil,nil] + end + + Test::Unit::Runner.output = orig_testout + $stdin = orig_stdin + $stdout = orig_stdout + + o.close + begin + th.join + rescue IOError + raise unless /stream closed|closed stream/ =~ $!.message + end + i.close + + result << @partial_report + @partial_report = nil + result << [@errors-e,@failures-f,@skips-s] + result << ($: - @old_loadpath) + result << suite.name + + _report "done", Marshal.dump(result) + return result + ensure + Test::Unit::Runner.output = orig_stdout + $stdin = orig_stdin if orig_stdin + $stdout = orig_stdout if orig_stdout + o.close if o && !o.closed? + i.close if i && !i.closed? + end + + def run(args = []) # :nodoc: + process_args args + @@stop_auto_run = true + @opts = @options.dup + @need_exit = false + + @old_loadpath = [] + begin + begin + @stdout = increment_io(STDOUT) + @stdin = increment_io(STDIN) + rescue + exit 2 + end + exit 2 unless @stdout && @stdin + + @stdout.sync = true + _report "ready!" + while buf = @stdin.gets + case buf.chomp + when /^loadpath (.+?)$/ + @old_loadpath = $:.dup + $:.push(*Marshal.load($1.unpack1("m").force_encoding("ASCII-8BIT"))).uniq! + when /^run (.+?) (.+?)$/ + _report "okay" + + @options = @opts.dup + suites = Test::Unit::TestCase.test_suites + + begin + require File.realpath($1) + rescue LoadError + _report "after", Marshal.dump([$1, ProxyError.new($!)]) + _report "ready" + next + end + _run_suites Test::Unit::TestCase.test_suites-suites, $2.to_sym + + if @need_exit + _report "bye" + exit + else + _report "ready" + end + when /^quit (.+?)$/, "quit" + if $1 == "timeout" + err = ["", "!!! worker #{$$} killed due to timeout:"] + Thread.list.each do |th| + err << "#{ th.inspect }:" + th.backtrace.each do |s| + err << " #{ s }" + end + end + err << "" + STDERR.puts err.join("\n") + end + _report "bye" + exit + end + end + rescue Exception => e + trace = e.backtrace || ['unknown method'] + err = ["#{trace.shift}: #{e.message} (#{e.class})"] + trace.map{|t| "\t" + t } + + if @stdout + _report "bye", Marshal.dump(err.join("\n")) + else + raise "failed to report a failure due to lack of @stdout" + end + exit + ensure + @stdin.close if @stdin + @stdout.close if @stdout + end + end + + def _report(res, *args) # :nodoc: + @stdout.write(args.empty? ? "#{res}\n" : "#{res} #{args.pack("m0")}\n") + true + rescue Errno::EPIPE + rescue TypeError => e + abort("#{e.inspect} in _report(#{res.inspect}, #{args.inspect})\n#{e.backtrace.join("\n")}") + end + + def puke(klass, meth, e) # :nodoc: + if e.is_a?(Test::Unit::PendedError) + new_e = Test::Unit::PendedError.new(e.message) + new_e.set_backtrace(e.backtrace) + e = new_e + end + @partial_report << [klass.name, meth, e.is_a?(Test::Unit::AssertionFailedError) ? e : ProxyError.new(e)] + super + end + + def record(suite, method, assertions, time, error) # :nodoc: + case error + when nil + when Test::Unit::AssertionFailedError, Test::Unit::PendedError + case error.cause + when nil, Test::Unit::AssertionFailedError, Test::Unit::PendedError + else + bt = error.backtrace + error = error.class.new(error.message) + error.set_backtrace(bt) + end + else + error = ProxyError.new(error) + end + _report "record", Marshal.dump([suite.name, method, assertions, time, error, suite.instance_method(method).source_location]) + super + end + end + end +end + +if $0 == __FILE__ + module Test + module Unit + class TestCase # :nodoc: all + undef on_parallel_worker? + def on_parallel_worker? + true + end + def self.on_parallel_worker? + true + end + end + end + end + require 'rubygems' + begin + require 'rake' + rescue LoadError + end + Test::Unit::Worker.new.run(ARGV) +end diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb new file mode 100644 index 0000000000..51ffff37eb --- /dev/null +++ b/tool/lib/test/unit/testcase.rb @@ -0,0 +1,298 @@ +# frozen_string_literal: true +require_relative 'assertions' +require_relative '../../core_assertions' + +module Test + module Unit + + ## + # Provides a simple set of guards that you can use in your tests + # to skip execution if it is not applicable. These methods are + # mixed into TestCase as both instance and class methods so you + # can use them inside or outside of the test methods. + # + # def test_something_for_mri + # skip "bug 1234" if jruby? + # # ... + # end + # + # if windows? then + # # ... lots of test methods ... + # end + + module Guard + + ## + # Is this running on jruby? + + def jruby? platform = RUBY_PLATFORM + "java" == platform + end + + ## + # Is this running on mri? + + def mri? platform = RUBY_DESCRIPTION + /^ruby/ =~ platform + end + + ## + # Is this running on windows? + + def windows? platform = RUBY_PLATFORM + /mswin|mingw/ =~ platform + end + + ## + # Is this running on mingw? + + def mingw? platform = RUBY_PLATFORM + /mingw/ =~ platform + end + + end + + ## + # Provides before/after hooks for setup and teardown. These are + # meant for library writers, NOT for regular test authors. See + # #before_setup for an example. + + module LifecycleHooks + ## + # Runs before every test, after setup. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def after_setup; end + + ## + # Runs before every test, before setup. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # As a simplistic example: + # + # module MyTestUnitPlugin + # def before_setup + # super + # # ... stuff to do before setup is run + # end + # + # def after_setup + # # ... stuff to do after setup is run + # super + # end + # + # def before_teardown + # super + # # ... stuff to do before teardown is run + # end + # + # def after_teardown + # # ... stuff to do after teardown is run + # super + # end + # end + # + # class Test::Unit::Runner::TestCase + # include MyTestUnitPlugin + # end + + def before_setup; end + + ## + # Runs after every test, before teardown. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def before_teardown; end + + ## + # Runs after every test, after teardown. This hook is meant for + # libraries to extend Test::Unit. It is not meant to be used by + # test developers. + # + # See #before_setup for an example. + + def after_teardown; end + end + + ## + # Subclass TestCase to create your own tests. Typically you'll want a + # TestCase subclass per implementation class. + # + # See <code>Test::Unit::AssertionFailedError</code>s + + class TestCase + include Assertions + include CoreAssertions + + include LifecycleHooks + include Guard + extend Guard + + attr_reader :__name__ # :nodoc: + + # Method name of this test. + alias method_name __name__ + + PASSTHROUGH_EXCEPTIONS = [NoMemoryError, SignalException, + Interrupt, SystemExit] # :nodoc: + + ## + # Runs the tests reporting the status to +runner+ + + def run runner + @__runner_options__ = runner.options + trap "INFO" do + runner.report.each_with_index do |msg, i| + warn "\n%3d) %s" % [i + 1, msg] + end + warn '' + time = runner.start_time ? Time.now - runner.start_time : 0 + warn "Current Test: %s#%s %.2fs" % [self.class, self.__name__, time] + runner.status $stderr + end if runner.info_signal + + start_time = Time.now + + result = "" + + begin + @__passed__ = nil + self.before_setup + self.setup + self.after_setup + self.run_test self.__name__ + result = "." unless io? + time = Time.now - start_time + runner.record self.class, self.__name__, self._assertions, time, nil + @__passed__ = true + rescue *PASSTHROUGH_EXCEPTIONS + raise + rescue Exception => e + @__passed__ = Test::Unit::PendedError === e + time = Time.now - start_time + runner.record self.class, self.__name__, self._assertions, time, e + result = runner.puke self.class, self.__name__, e + ensure + %w{ before_teardown teardown after_teardown }.each do |hook| + begin + self.send hook + rescue *PASSTHROUGH_EXCEPTIONS + raise + rescue Exception => e + @__passed__ = false + runner.record self.class, self.__name__, self._assertions, time, e + result = runner.puke self.class, self.__name__, e + end + end + trap 'INFO', 'DEFAULT' if runner.info_signal + end + result + end + + RUN_TEST_TRACE = "#{__FILE__}:#{__LINE__+3}:in `run_test'".freeze + def run_test(name) + progname, $0 = $0, "#{$0}: #{self.class}##{name}" + self.__send__(name) + ensure + $@.delete(RUN_TEST_TRACE) if $@ + $0 = progname + end + + def initialize name # :nodoc: + @__name__ = name + @__io__ = nil + @__passed__ = nil + @@__current__ = self # FIX: make thread local + end + + def self.current # :nodoc: + @@__current__ # FIX: make thread local + end + + ## + # Return the output IO object + + def io + @__io__ = true + Test::Unit::Runner.output + end + + ## + # Have we hooked up the IO yet? + + def io? + @__io__ + end + + def self.reset # :nodoc: + @@test_suites = {} + @@test_suites[self] = true + end + + reset + + def self.inherited klass # :nodoc: + @@test_suites[klass] = true + super + end + + @test_order = :sorted + + class << self + attr_writer :test_order + end + + def self.test_order + defined?(@test_order) ? @test_order : superclass.test_order + end + + def self.test_suites # :nodoc: + @@test_suites.keys + end + + def self.test_methods # :nodoc: + public_instance_methods(true).grep(/^test/) + end + + ## + # Returns true if the test passed. + + def passed? + @__passed__ + end + + ## + # Runs before every test. Use this to set up before each test + # run. + + def setup; end + + ## + # Runs after every test. Use this to clean up after each test + # run. + + def teardown; end + + def on_parallel_worker? + false + end + + def self.method_added(name) + super + return unless name.to_s.start_with?("test_") + @test_methods ||= {} + if @test_methods[name] + raise AssertionFailedError, "test/unit: method #{ self }##{ name } is redefined" + end + @test_methods[name] = true + end + end + end +end diff --git a/tool/lib/tracepointchecker.rb b/tool/lib/tracepointchecker.rb new file mode 100644 index 0000000000..3254e59357 --- /dev/null +++ b/tool/lib/tracepointchecker.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true +module TracePointChecker + STATE = { + count: 0, + running: false, + } + + module ZombieTraceHunter + def tracepoint_capture_stat_get + TracePoint.stat.map{|k, (activated, deleted)| + deleted = 0 unless @tracepoint_captured_singlethread + [k, activated, deleted] + } + end + + def before_setup + @tracepoint_captured_singlethread = (Thread.list.size == 1) + @tracepoint_captured_stat = tracepoint_capture_stat_get() + super + end + + def after_teardown + super + + # detect zombie traces. + assert_equal( + @tracepoint_captured_stat, + tracepoint_capture_stat_get(), + "The number of active/deleted trace events was changed" + ) + # puts "TracePoint - deleted: #{deleted}" if deleted > 0 + + TracePointChecker.check if STATE[:running] + end + end + + MAIN_THREAD = Thread.current + TRACES = [] + + def self.prefix event + case event + when :call, :return + :n + when :c_call, :c_return + :c + when :b_call, :b_return + :b + end + end + + def self.clear_call_stack + Thread.current[:call_stack] = [] + end + + def self.call_stack + stack = Thread.current[:call_stack] + stack = clear_call_stack unless stack + stack + end + + def self.verbose_out label, method + puts label => call_stack, :count => STATE[:count], :method => method + end + + def self.method_label tp + "#{prefix(tp.event)}##{tp.method_id}" + end + + def self.start verbose: false, stop_at_failure: false + call_events = %i(a_call) + return_events = %i(a_return) + clear_call_stack + + STATE[:running] = true + + TRACES << TracePoint.new(*call_events){|tp| + next if Thread.current != MAIN_THREAD + + method = method_label(tp) + call_stack.push method + STATE[:count] += 1 + + verbose_out :push, method if verbose + } + + TRACES << TracePoint.new(*return_events){|tp| + next if Thread.current != MAIN_THREAD + STATE[:count] += 1 + + method = "#{prefix(tp.event)}##{tp.method_id}" + verbose_out :pop1, method if verbose + + stored_method = call_stack.pop + next if stored_method.nil? + + verbose_out :pop2, method if verbose + + if stored_method != method + stop if stop_at_failure + RubyVM::SDR() if defined? RubyVM::SDR() + call_stack.clear + raise "#{stored_method} is expected, but #{method} (count: #{STATE[:count]})" + end + } + + TRACES.each{|trace| trace.enable} + end + + def self.stop + STATE[:running] = true + TRACES.each{|trace| trace.disable} + TRACES.clear + end + + def self.check + TRACES.each{|trace| + raise "trace #{trace} should not be deactivated" unless trace.enabled? + } + end +end if defined?(TracePoint.stat) + +class ::Test::Unit::TestCase + include TracePointChecker::ZombieTraceHunter +end if defined?(TracePointChecker) + +# TracePointChecker.start verbose: false diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb new file mode 100644 index 0000000000..26c9763c13 --- /dev/null +++ b/tool/lib/vcs.rb @@ -0,0 +1,656 @@ +# vcs +require 'fileutils' +require 'optparse' +require 'pp' +require 'tempfile' + +# This library is used by several other tools/ scripts to detect the current +# VCS in use (e.g. SVN, Git) or to interact with that VCS. + +ENV.delete('PWD') + +class VCS + DEBUG_OUT = STDERR.dup + + def self.dump(obj, pre = nil) + out = DEBUG_OUT + @pp ||= PP.new(out) + @pp.guard_inspect_key do + if pre + @pp.group(pre.size, pre) { + obj.pretty_print(@pp) + } + else + obj.pretty_print(@pp) + end + @pp.flush + out << "\n" + end + end +end + +unless File.respond_to? :realpath + require 'pathname' + def File.realpath(arg) + Pathname(arg).realpath.to_s + end +end + +def IO.pread(*args) + VCS.dump(args, "args: ") if $DEBUG + popen(*args) {|f|f.read} +end + +module DebugPOpen + refine IO.singleton_class do + def popen(*args) + VCS.dump(args, "args: ") if $DEBUG + super + end + end +end +using DebugPOpen +module DebugSystem + def system(*args, exception: true, **opts) + VCS.dump(args, "args: ") if $DEBUG + super(*args, exception: exception, **opts) + end +end + +class VCS + prepend(DebugSystem) if defined?(DebugSystem) + class NotFoundError < RuntimeError; end + + @@dirs = [] + def self.register(dir, &pred) + @@dirs << [dir, self, pred] + end + + def self.detect(path = '.', options = {}, parser = nil, **opts) + options.update(opts) + uplevel_limit = options.fetch(:uplevel_limit, 0) + curr = path + begin + @@dirs.each do |dir, klass, pred| + if pred ? pred[curr, dir] : File.directory?(File.join(curr, dir)) + if klass.const_defined?(:COMMAND) + IO.pread([{'LANG' => 'C', 'LC_ALL' => 'C'}, klass::COMMAND, "--version"]) rescue next + end + vcs = klass.new(curr) + vcs.define_options(parser) if parser + vcs.set_options(options) + return vcs + end + end + if uplevel_limit + break if uplevel_limit.zero? + uplevel_limit -= 1 + end + prev, curr = curr, File.realpath(File.join(curr, '..')) + end until curr == prev # stop at the root directory + raise VCS::NotFoundError, "does not seem to be under a vcs: #{path}" + end + + def self.local_path?(path) + String === path or path.respond_to?(:to_path) + end + + def self.define_options(parser, opts = {}) + parser.separator(" VCS common options:") + parser.define("--[no-]dryrun") {|v| opts[:dryrun] = v} + parser.define("--[no-]debug") {|v| opts[:debug] = v} + parser.define("-z", "--zone=OFFSET", /\A[-+]\d\d:\d\d\z/) {|v| opts[:zone] = v} + opts + end + + def release_date(time) + t = time.getlocal(@zone) + [ + t.strftime('#define RUBY_RELEASE_YEAR %Y'), + t.strftime('#define RUBY_RELEASE_MONTH %-m'), + t.strftime('#define RUBY_RELEASE_DAY %-d'), + ] + end + + def self.short_revision(rev) + rev + end + + attr_reader :srcdir + + def initialize(path) + @srcdir = path + super() + end + + def chdir(path) + @srcdir = path + end + + def define_options(parser) + end + + def set_options(opts) + @debug = opts.fetch(:debug) {$DEBUG} + @dryrun = opts.fetch(:dryrun) {@debug} + @zone = opts.fetch(:zone) {'+09:00'} + end + + attr_reader :dryrun, :debug + alias dryrun? dryrun + alias debug? debug + + NullDevice = IO::NULL + + # returns + # * the last revision of the current branch + # * the last revision in which +path+ was modified + # * the last modified time of +path+ + # * the last commit title since the latest upstream + def get_revisions(path) + if self.class.local_path?(path) + path = relative_to(path) + end + last, changed, modified, *rest = ( + begin + if NullDevice and !debug? + save_stderr = STDERR.dup + STDERR.reopen NullDevice, 'w' + end + _get_revisions(path, @srcdir) + rescue Errno::ENOENT => e + raise VCS::NotFoundError, e.message + ensure + if save_stderr + STDERR.reopen save_stderr + save_stderr.close + end + end + ) + last or raise VCS::NotFoundError, "last revision not found" + changed or raise VCS::NotFoundError, "changed revision not found" + if modified + /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ modified or + raise "unknown time format - #{modified}" + match = $~[1..6].map { |x| x.to_i } + off = $7 ? "#{$7}:#{$8}" : "+00:00" + match << off + begin + modified = Time.new(*match) + rescue ArgumentError + modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 + end + modified = modified.getlocal(@zone) + end + return last, changed, modified, *rest + end + + def modified(path) + _, _, modified, * = get_revisions(path) + modified + end + + def relative_to(path) + if path + srcdir = File.realpath(@srcdir) + path = File.realdirpath(path) + list1 = srcdir.split(%r{/}) + list2 = path.split(%r{/}) + while !list1.empty? && !list2.empty? && list1.first == list2.first + list1.shift + list2.shift + end + if list1.empty? && list2.empty? + "." + else + ([".."] * list1.length + list2).join("/") + end + else + '.' + end + end + + def after_export(dir) + FileUtils.rm_rf(Dir.glob("#{dir}/.git*")) + FileUtils.rm_rf(Dir.glob("#{dir}/.mailmap")) + end + + def revision_handler(rev) + self.class + end + + def revision_name(rev) + revision_handler(rev).revision_name(rev) + end + + def short_revision(rev) + revision_handler(rev).short_revision(rev) + end + + # make-snapshot generates only release_date whereas file2lastrev generates both release_date and release_datetime + def revision_header(last, release_date, release_datetime = nil, branch = nil, title = nil, limit: 20) + short = short_revision(last) + if /[^\x00-\x7f]/ =~ title and title.respond_to?(:force_encoding) + title = title.dup.force_encoding("US-ASCII") + end + code = [ + "#define RUBY_REVISION #{short.inspect}", + ] + unless short == last + code << "#define RUBY_FULL_REVISION #{last.inspect}" + end + if branch + e = '..' + name = branch.sub(/\A(.{#{limit-e.size}}).{#{e.size+1},}/o) {$1+e} + name = name.dump.sub(/\\#/, '#') + code << "#define RUBY_BRANCH_NAME #{name}" + end + if title + title = title.dump.sub(/\\#/, '#') + code << "#define RUBY_LAST_COMMIT_TITLE #{title}" + end + if release_datetime + t = release_datetime.utc + code << t.strftime('#define RUBY_RELEASE_DATETIME "%FT%TZ"') + end + code += self.release_date(release_date) + code + end + + class GIT < self + register(".git") do |path, dir| + SAFE_DIRECTORIES ||= + begin + command = ENV["GIT"] || 'git' + dirs = IO.popen(%W"#{command} config --global --get-all safe.directory", &:read).split("\n") + rescue + command = nil + dirs = [] + ensure + VCS.dump(dirs, "safe.directory: ") if $DEBUG + COMMAND = command + end + + COMMAND and File.exist?(File.join(path, dir)) + end + + def cmd_args(cmds, srcdir = nil) + (opts = cmds.last).kind_of?(Hash) or cmds << (opts = {}) + opts[:external_encoding] ||= "UTF-8" + if srcdir + opts[:chdir] ||= srcdir + end + VCS.dump(cmds, "cmds: ") if debug? and !$DEBUG + cmds + end + + def cmd_pipe_at(srcdir, cmds, &block) + without_gitconfig { IO.popen(*cmd_args(cmds, srcdir), &block) } + end + + def cmd_read_at(srcdir, cmds) + result = without_gitconfig { IO.pread(*cmd_args(cmds, srcdir)) } + VCS.dump(result, "result: ") if debug? + result + end + + def cmd_pipe(*cmds, &block) + cmd_pipe_at(@srcdir, cmds, &block) + end + + def cmd_read(*cmds) + cmd_read_at(@srcdir, cmds) + end + + def svn_revision(log) + if /^ *git-svn-id: .*@(\d+) .*\n+\z/ =~ log + $1.to_i + end + end + + def _get_revisions(path, srcdir = nil) + ref = Branch === path ? path.to_str : 'HEAD' + gitcmd = [COMMAND] + last = nil + IO.pipe do |r, w| + last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref, err: w]]).rstrip + w.close + unless r.eof? + raise VCS::NotFoundError, "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" + end + end + log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) + changed = log[/\Acommit (\h+)/, 1] + modified = log[/^CommitDate:\s+(.*)/, 1] + if rev = svn_revision(log) + if changed == last + last = rev + else + svn_rev = svn_revision(cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--format=%B', last]])) + last = svn_rev if svn_rev + end + changed = rev + end + branch = cmd_read_at(srcdir, [gitcmd + %W[symbolic-ref --short #{ref}]]) + if branch.empty? + branch = cmd_read_at(srcdir, [gitcmd + %W[describe --contains #{ref}]]).strip + end + if branch.empty? + branch_list = cmd_read_at(srcdir, [gitcmd + %W[branch --list --contains #{ref}]]).lines.to_a + branch, = branch_list.grep(/\A\*/) + case branch + when /\A\* *\(\S+ detached at (.*)\)\Z/ + branch = $1 + branch = nil if last.start_with?(branch) + when /\A\* (\S+)\Z/ + branch = $1 + else + branch = nil + end + unless branch + branch_list.each {|b| b.strip!} + branch_list.delete_if {|b| / / =~ b} + branch = branch_list.min_by(&:length) || "" + end + end + branch.chomp! + branch = ":detached:" if branch.empty? + upstream = cmd_read_at(srcdir, [gitcmd + %W[branch --list --format=%(upstream:short) #{branch}]]) + upstream.chomp! + title = cmd_read_at(srcdir, [gitcmd + %W[log --format=%s -n1 #{upstream}..#{ref}]]) + title = nil if title.empty? + [last, changed, modified, branch, title] + end + + def self.revision_name(rev) + short_revision(rev) + end + + def self.short_revision(rev) + rev[0, 10] + end + + def without_gitconfig + envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| + h[v] = ENV.delete(v) + end + ENV['GIT_CONFIG_SYSTEM'] = NullDevice + ENV['GIT_CONFIG_GLOBAL'] = global_config + yield + ensure + ENV.update(envs) + end + + def global_config + return NullDevice if SAFE_DIRECTORIES.empty? + unless @gitconfig + @gitconfig = Tempfile.new(%w"vcs_ .gitconfig") + @gitconfig.close + ENV['GIT_CONFIG_GLOBAL'] = @gitconfig.path + SAFE_DIRECTORIES.each do |dir| + system(*%W[#{COMMAND} config --global --add safe.directory #{dir}]) + end + VCS.dump(`#{COMMAND} config --global --get-all safe.directory`, "safe.directory: ") if debug? + end + @gitconfig.path + end + + def initialize(*) + super + @srcdir = File.realpath(@srcdir) + @gitconfig = nil + VCS.dump(@srcdir, "srcdir: ") if debug? + self + end + + Branch = Struct.new(:to_str) + + def branch(name) + Branch.new(name) + end + + alias tag branch + + def master + branch("master") + end + alias trunk master + + def stable + cmd = %W"#{COMMAND} for-each-ref --format=\%(refname:short) refs/heads/ruby_[0-9]*" + branch(cmd_read(cmd)[/.*^(ruby_\d+_\d+)$/m, 1]) + end + + def branch_list(pat) + cmd = %W"#{COMMAND} for-each-ref --format=\%(refname:short) refs/heads/#{pat}" + cmd_pipe(cmd) {|f| + f.each {|line| + line.chomp! + yield line + } + } + end + + def grep(pat, tag, *files, &block) + cmd = %W[#{COMMAND} grep -h --perl-regexp #{tag} --] + set = block.binding.eval("proc {|match| $~ = match}") + cmd_pipe(cmd+files) do |f| + f.grep(pat) do |s| + set[$~] + yield s + end + end + end + + def export(revision, url, dir, keep_temp = false) + system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return + GIT.new(File.expand_path(dir)) + end + + def branch_beginning(url) + year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i + cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25 + --author=matz --committer=matz --grep=started\\.$ + #{url} -- version.h include/ruby/version.h])[/.*/] + end + + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true) + from, to = [from, to].map do |rev| + rev or next + rev unless rev.empty? + end + unless from&.match?(/./) or (from = branch_beginning(url))&.match?(/./) + warn "no starting commit found", uplevel: 1 + from = nil + end + if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + chdir: @srcdir, exception: false) + system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", + chdir: @srcdir, exception: false) + else + warn "Could not fetch notes/commits tree", uplevel: 1 + end + to ||= url.to_str + if from + arg = ["#{from}^..#{to}"] + else + arg = ["--since=25 Dec 00:00:00", to] + end + if base_url == true + env = CHANGELOG_ENV + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') + end + base_url = remote + end + writer = changelog_formatter(path, arg, base_url) + if !path or path == '-' + writer[$stdout] + else + File.open(path, 'wb', &writer) + end + end + + LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' + CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + + def changelog_formatter(path, arg, base_url = nil) + env = CHANGELOG_ENV + cmd = %W[#{COMMAND} log + --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges + --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] + ] + date = "--date=iso-local" + unless system(env, *cmd, date, "-1", chdir: @srcdir, out: NullDevice, exception: false) + date = "--date=iso" + end + cmd << date + cmd.concat(arg) + proc do |w| + w.print "-*- coding: utf-8 -*-\n" + w.print "\n""base-url = #{base_url}\n" if base_url + + begin + ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true) + .grep_v(/^ *(?:#|$)/) + .to_h {|v| [v, true]} + ignore_revs = nil if ignore_revs.empty? + rescue Errno::ENOENT + end + + cmd_pipe(env, cmd, chdir: @srcdir) do |r| + r.gets(sep = "commit ") + sep = "\n" + sep + while s = r.gets(sep, chomp: true) + h, s = s.split(/^$/, 2) + if ignore_revs&.key?(h[/\A\h{40}/]) + next + end + + next if /^Author: *dependabot\[bot\]/ =~ h + + h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') + if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') + fix = $1 + next if /\A *skip\Z/ =~ fix + s = s.lines + fix.each_line do |x| + next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') + b = ($2&.to_i || (s.size - 1 + $3.to_i)) + sp = $1 + if x.sub!(/^,(?:(\d+)|\$(?:-\d+)?)/, '') + range = b..($1&.to_i || (s.size - 1 + $2.to_i)) + else + range = b..b + end + case x + when %r[^s([#{LOG_FIX_REGEXP_SEPARATORS}])(.+)\1(.*)\1([gr]+)?]o + wrong = $2 + correct = $3 + if opt = $4 and opt.include?("r") # regexp + wrong = Regexp.new(wrong) + correct.gsub!(/(?<!\\)(?:\\\\)*\K(?:\\n)+/) {"\n" * ($&.size / 2)} + sub = opt.include?("g") ? :gsub! : :sub! + else + sub = false + end + range.each do |n| + if sub + ss = s[n].sub(/^#{sp}/, "") # un-indent for /^/ + if ss.__send__(sub, wrong, correct) + s[n, 1] = ss.lines.map {|l| "#{sp}#{l}"} + next + end + else + begin + s[n][wrong] = correct + rescue IndexError + else + next + end + end + message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] + from = [1, n-2].max + to = [s.size-1, n+2].min + s.each_with_index do |e, i| + next if i < from + break if to < i + message << "#{i}:#{e}" + end + raise message.join('') + end + when %r[^i([#{LOG_FIX_REGEXP_SEPARATORS}])(.*)\1]o + insert = "#{sp}#{$2}\n" + range.reverse_each do |n| + s[n, 0] = insert + end + when %r[^d] + s[range] = [] + end + end + s = s.join('') + end + + s.gsub!(%r[(?!<\w)([-\w]+/[-\w]+)(?:@(\h{8,40})|#(\d{5,}))\b]) do + path = defined?($2) ? "commit/#{$2}" : "pull/#{$3}" + "[#$&](https://github.com/#{$1}/#{path})" + end + if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s + issue = "#{$1}pull/" + s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} + end + + s.gsub!(/ +\n/, "\n") + s.sub!(/^Notes:/, ' \&') + w.print sep, h, s + end + end + end + end + + def upstream + (branch = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD")).chomp! + (upstream = cmd_read(%W"#{COMMAND} branch --list --format=%(upstream) #{branch}")).chomp! + while ref = upstream[%r"\Arefs/heads/(.*)", 1] + upstream = cmd_read(%W"#{COMMAND} branch --list --format=%(upstream) #{ref}") + end + unless %r"\Arefs/remotes/([^/]+)/(.*)" =~ upstream + raise "Upstream not found" + end + [$1, $2] + end + + def commit(opts = {}) + args = [COMMAND, "push"] + args << "-n" if dryrun? + remote, branch = upstream + args << remote + branches = %W[refs/notes/commits:refs/notes/commits HEAD:#{branch}] + if dryrun? + branches.each do |b| + VCS.dump(args + [b], "commit: ") + end + return true + end + branches.each do |b| + system(*(args + [b])) or return false + end + true + end + end + + class Null < self + def get_revisions(path, srcdir = nil) + @modified ||= Time.now - 10 + return nil, nil, @modified + end + + def revision_header(last, release_date, release_datetime = nil, branch = nil, title = nil, limit: 20) + self.release_date(release_date) + end + end +end diff --git a/tool/vpath.rb b/tool/lib/vpath.rb index 48ab148405..fa819f3242 100644 --- a/tool/vpath.rb +++ b/tool/lib/vpath.rb @@ -53,10 +53,11 @@ class VPath end def def_options(opt) + opt.separator(" VPath common options:") opt.on("-I", "--srcdir=DIR", "add a directory to search path") {|dir| @additional << dir } - opt.on("-L", "--vpath=PATH LIST", "add directories to search path") {|dirs| + opt.on("-L", "--vpath=PATH-LIST", "add directories to search path") {|dirs| @additional << [dirs] } opt.on("--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, "separator for vpath") {|sep, vsep| @@ -80,6 +81,10 @@ class VPath @list end + def add(path) + @additional << path + end + def strip(path) prefix = list.map {|dir| Regexp.quote(dir)} path.sub(/\A#{prefix.join('|')}(?:\/|\z)/, '') diff --git a/tool/lib/zombie_hunter.rb b/tool/lib/zombie_hunter.rb new file mode 100644 index 0000000000..33bc467941 --- /dev/null +++ b/tool/lib/zombie_hunter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module ZombieHunter + def after_teardown + super + assert_empty(Process.waitall) + end +end + +Test::Unit::TestCase.include ZombieHunter diff --git a/tool/ln_sr.rb b/tool/ln_sr.rb new file mode 100755 index 0000000000..e1b5b6f76b --- /dev/null +++ b/tool/ln_sr.rb @@ -0,0 +1,131 @@ +#!/usr/bin/ruby + +target_directory = true +noop = false +force = false +quiet = false + +until ARGV.empty? + case ARGV[0] + when '-n' + noop = true + when '-f' + force = true + when '-T' + target_directory = false + when '-q' + quiet = true + else + break + end + ARGV.shift +end + +unless ARGV.size == 2 + abort "usage: #{$0} src destdir" +end +src, dest = ARGV + +require 'fileutils' + +include FileUtils +unless respond_to?(:ln_sr) + def ln_sr(src, dest, target_directory: true, force: nil, noop: nil, verbose: nil) + options = "#{force ? 'f' : ''}#{target_directory ? '' : 'T'}" + dest = File.path(dest) + srcs = Array(src) + link = proc do |s, target_dir_p = true| + s = File.path(s) + if target_dir_p + d = File.join(destdirs = dest, File.basename(s)) + else + destdirs = File.dirname(d = dest) + end + destdirs = fu_split_path(File.realpath(destdirs)) + if fu_starting_path?(s) + srcdirs = fu_split_path((File.realdirpath(s) rescue File.expand_path(s))) + base = fu_relative_components_from(srcdirs, destdirs) + s = File.join(*base) + else + srcdirs = fu_clean_components(*fu_split_path(s)) + base = fu_relative_components_from(fu_split_path(Dir.pwd), destdirs) + while srcdirs.first&. == ".." and base.last&.!=("..") and !fu_starting_path?(base.last) + srcdirs.shift + base.pop + end + s = File.join(*base, *srcdirs) + end + fu_output_message "ln -s#{options} #{s} #{d}" if verbose + next if noop + remove_file d, true if force + File.symlink s, d + end + case srcs.size + when 0 + when 1 + link[srcs[0], target_directory && File.directory?(dest)] + else + srcs.each(&link) + end + end + + def fu_split_path(path) + path = File.path(path) + list = [] + until (parent, base = File.split(path); parent == path or parent == ".") + list << base + path = parent + end + list << path + list.reverse! + end + + def fu_relative_components_from(target, base) #:nodoc: + i = 0 + while target[i]&.== base[i] + i += 1 + end + Array.new(base.size-i, '..').concat(target[i..-1]) + end + + def fu_clean_components(*comp) + comp.shift while comp.first == "." + return comp if comp.empty? + clean = [comp.shift] + path = File.join(*clean, "") # ending with File::SEPARATOR + while c = comp.shift + if c == ".." and clean.last != ".." and !(fu_have_symlink? && File.symlink?(path)) + clean.pop + path.sub!(%r((?<=\A|/)[^/]+/\z), "") + else + clean << c + path << c << "/" + end + end + clean + end + + if fu_windows? + def fu_starting_path?(path) + path&.start_with?(%r(\w:|/)) + end + else + def fu_starting_path?(path) + path&.start_with?("/") + end + end +end + +if File.respond_to?(:symlink) + if quiet and File.identical?(src, dest) + exit + end + begin + ln_sr(src, dest, verbose: true, target_directory: target_directory, force: force, noop: noop) + rescue NotImplementedError, Errno::EPERM, Errno::EACCES + else + exit + end +end + +cp_r(src, dest) diff --git a/tool/lrama/LEGAL.md b/tool/lrama/LEGAL.md new file mode 100644 index 0000000000..a3ef848514 --- /dev/null +++ b/tool/lrama/LEGAL.md @@ -0,0 +1,12 @@ +# LEGAL NOTICE INFORMATION + +All the files in this distribution are covered under the MIT License except some files +mentioned below. + +## GNU General Public License version 3 + +These files are licensed under the GNU General Public License version 3 or later. See these files for more information. + +* template/bison/_yacc.h +* template/bison/yacc.c +* template/bison/yacc.h diff --git a/tool/lrama/MIT b/tool/lrama/MIT new file mode 100644 index 0000000000..b23d5210d5 --- /dev/null +++ b/tool/lrama/MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 Yuichiro Kaneko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md new file mode 100644 index 0000000000..f71118a913 --- /dev/null +++ b/tool/lrama/NEWS.md @@ -0,0 +1,1032 @@ +# NEWS for Lrama + +## Lrama 0.7.1 (2025-12-24) + +### Optimize IELR + +Optimized performance to a level that allows for IELR testing in practical applications. + +https://github.com/ruby/lrama/pull/595 +https://github.com/ruby/lrama/pull/605 +https://github.com/ruby/lrama/pull/685 +https://github.com/ruby/lrama/pull/700 + +### Introduce counterexamples timeout + +Counterexample searches can sometimes take a long time, so we've added a timeout to abort the process after a set period. The current limits are: + +* 10 seconds per case +* 120 seconds total (cumulative) + +Please note that these are hard-coded and cannot be modified by the user in the current version. + +https://github.com/ruby/lrama/pull/623 + +### Optimize Counterexamples + +Optimized counterexample search performance. + +https://github.com/ruby/lrama/pull/607 +https://github.com/ruby/lrama/pull/610 +https://github.com/ruby/lrama/pull/614 +https://github.com/ruby/lrama/pull/622 +https://github.com/ruby/lrama/pull/627 +https://github.com/ruby/lrama/pull/629 +https://github.com/ruby/lrama/pull/659 + +### Support parameterized rule's arguments include inline + +Allow to use %inline directive with Parameterized rules arguments. When an inline rule is used as an argument to a Parameterized rule, it expands inline at the point of use. + +```yacc +%rule %inline op : '+' + | '-' + ; +%% +operation : op? + ; +``` + +This expands to: + +```yacc +operation : /* empty */ + | '+' + | '-' + ; +``` + +https://github.com/ruby/lrama/pull/637 + +### Render conflicts of each state on output file + +Added token information for conflicts in the output file. +These information are useful when a state has many actions. + +``` +State 1 + + 4 class: keyword_class • tSTRING "end" + 5 $@1: ε • [tSTRING] + 7 class: keyword_class • $@1 tSTRING '!' "end" $@2 + 8 $@3: ε • [tSTRING] + 10 class: keyword_class • $@3 tSTRING '?' "end" $@4 + + Conflict on tSTRING. shift/reduce($@1) + Conflict on tSTRING. shift/reduce($@3) + Conflict on tSTRING. reduce($@1)/reduce($@3) + + tSTRING shift, and go to state 6 + + tSTRING reduce using rule 5 ($@1) + tSTRING reduce using rule 8 ($@3) + + $@1 go to state 7 + $@3 go to state 8 +``` + +https://github.com/ruby/lrama/pull/541 + +### Render the origin of conflicted tokens on output file + +For example, for the grammar file like below: + +``` +%% + +program: expr + ; + +expr: expr '+' expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where `"plus"` (`'+'`) look ahead tokens come from: + +``` +State 6 + + 2 expr: expr • "plus" expr + 2 | expr "plus" expr • ["end of file", "plus"] + + Conflict on "plus". shift/reduce(expr) + "plus" comes from state 0 goto by expr + "plus" comes from state 5 goto by expr +``` + +state 0 and state 5 look like below: + +``` +State 0 + + 0 $accept: • program "end of file" + 1 program: • expr + 2 expr: • expr "plus" expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + program go to state 2 + expr go to state 3 + +State 5 + + 2 expr: • expr "plus" expr + 2 | expr "plus" • expr + 3 | • tNUMBER + + tNUMBER shift, and go to state 1 + + expr go to state 6 +``` + +https://github.com/ruby/lrama/pull/726 + +### Render precedences usage information on output file + +For example, for the grammar file like below: + +``` +%left tPLUS +%right tUPLUS + +%% + +program: expr ; + +expr: tUPLUS expr + | expr tPLUS expr + | tNUMBER + ; + +%% +``` + +Lrama generates output file which describes where these precedences are used to resolve conflicts: + +``` +Precedences + precedence on "unary+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + precedence on "+" is used to resolve conflict on + LALR + state 5. Conflict between reduce by "expr -> tUPLUS expr" and shift "+" resolved as reduce ("+" < "unary+"). + state 8. Conflict between reduce by "expr -> expr tPLUS expr" and shift "+" resolved as reduce (%left "+"). +``` + +https://github.com/ruby/lrama/pull/741 + +### Add support for reporting Rule Usage Frequency + +Support to report rule usage frequency statistics for analyzing grammar characteristics. +Run `exe/lrama --report=rules` to show how frequently each terminal and non-terminal symbol is used in the grammar rules. + +```console +$ exe/lrama --report=rules sample/calc.y +Rule Usage Frequency + 0 tSTRING (4 times) + 1 keyword_class (3 times) + 2 keyword_end (3 times) + 3 '+' (2 times) + 4 string (2 times) + 5 string_1 (2 times) + 6 '!' (1 times) + 7 '-' (1 times) + 8 '?' (1 times) + 9 EOI (1 times) + 10 class (1 times) + 11 program (1 times) + 12 string_2 (1 times) + 13 strings_1 (1 times) + 14 strings_2 (1 times) + 15 tNUMBER (1 times) +``` + +This feature provides insights into the language characteristics by showing: +- Which symbols are most frequently used in the grammar +- The distribution of terminal and non-terminal usage +- Potential areas for grammar optimization or refactoring + +The frequency statistics help developers understand the grammar structure and can be useful for: +- Grammar complexity analysis +- Performance optimization hints +- Language design decisions +- Documentation and educational purposes + +https://github.com/ruby/lrama/pull/677 + +### Render Split States information on output file + +For example, for the grammar file like below: + +``` +%token a +%token b +%token c +%define lr.type ielr + +%precedence tLOWEST +%precedence a +%precedence tHIGHEST + +%% + +S: a A B a + | b A B b + ; + +A: a C D E + ; + +B: c + | // empty + ; + +C: D + ; + +D: a + ; + +E: a + | %prec tHIGHEST // empty + ; + +%% +``` + +Lrama generates output file which describes where which new states are created when IELR is enabled: + +``` +Split States + + State 19 is split from state 4 + State 20 is split from state 9 + State 21 is split from state 14 +``` + +https://github.com/ruby/lrama/pull/624 + +### Add ioption support to the Standard library + +Support `ioption` (inline option) rule, which is expanded inline without creating intermediate rules. + +Unlike the regular `option` rule that generates a separate rule, `ioption` directly expands at the point of use: + +```yacc +program: ioption(number) expr + +// Expanded inline to: + +program: expr + | number expr +``` + +This differs from the regular `option` which would generate: + +```yacc +program: option(number) expr + +// Expanded to: + +program: option_number expr +option_number: %empty + | number +``` + +The `ioption` rule provides more compact grammar generation by avoiding intermediate rule creation, which can be beneficial for reducing the parser's rule count and potentially improving performance. + +This feature is inspired by Menhir's standard library and maintains compatibility with [Menhir's `ioption` behavior](https://github.com/let-def/menhir/blob/e8ba7bef219acd355798072c42abbd11335ecf09/src/standard.mly#L33-L41). + +https://github.com/ruby/lrama/pull/666 + +### Syntax Diagrams + +Lrama provides an API for generating HTML syntax diagrams. These visual diagrams are highly useful as grammar development tools and can also serve as a form of automatic self-documentation. + + + +If you use syntax diagrams, you add `--diagram` option. + +```console +$ exe/lrama --diagram sample.y +``` + +https://github.com/ruby/lrama/pull/523 + +### Support `--profile` option + +You can profile parser generation process without modification for Lrama source code. +Currently `--profile=call-stack` and `--profile=memory` are supported. + +```console +$ exe/lrama --profile=call-stack sample/calc.y +``` + +Then "tmp/stackprof-cpu-myapp.dump" is generated. + +https://github.com/ruby/lrama/pull/525 + +### Add support Start-Symbol: `%start` + +https://github.com/ruby/lrama/pull/576 + +## Lrama 0.7.0 (2025-01-21) + +### [EXPERIMENTAL] Support the generation of the IELR(1) parser described in this paper + +Support the generation of the IELR(1) parser described in this paper. +https://www.sciencedirect.com/science/article/pii/S0167642309001191 + +If you use IELR(1) parser, you can write the following directive in your grammar file. + +```yacc +%define lr.type ielr +``` + +But, currently IELR(1) parser is experimental feature. If you find any bugs, please report it to us. Thank you. + +### Support `-t` option as same as `--debug` option + +Support to `-t` option as same as `--debug` option. +These options align with Bison behavior. So same as `--debug` option. + +### Trace only explicit rules + +Support to trace only explicit rules. +If you use `--trace=rules` option, it shows include mid-rule actions. If you want to show only explicit rules, you can use `--trace=only-explicit-rules` option. + +Example: + +```yacc +%{ +%} +%union { + int i; +} +%token <i> number +%type <i> program +%% +program : number { printf("%d", $1); } number { $$ = $1 + $3; } + ; +%% +``` + +Result of `--trace=rules`: + +```console +$ exe/lrama --trace=rules sample.y +Grammar rules: +$accept -> program YYEOF +$@1 -> ε +program -> number $@1 number +``` + +Result of `--trace=only-explicit-rules`: + +```console +$ exe/lrama --trace=explicit-rules sample.y +Grammar rules: +$accept -> program YYEOF +program -> number number +``` + +## Lrama 0.6.11 (2024-12-23) + +### Add support for %type declarations using %nterm in Nonterminal Symbols + +Allow to use `%nterm` in Nonterminal Symbols for `%type` declarations. + +```yacc +%nterm <type> nonterminal… +``` + +This directive is also supported for compatibility with Bison, and only non-terminal symbols are allowed. In other words, definitions like the following will result in an error: + +```yacc +%{ +// Prologue +%} + +%token EOI 0 "EOI" +%nterm EOI + +%% + +program: /* empty */ + ; +``` + +It show an error message like the following: + +```command +❯ exe/lrama nterm.y +nterm.y:6:7: symbol EOI redeclared as a nonterminal +%nterm EOI + ^^^ +``` + +## Lrama 0.6.10 (2024-09-11) + +### Aliased Named References for actions of RHS in Parameterizing rules + +Allow to use aliased named references for actions of RHS in Parameterizing rules. + +```yacc +%rule sum(X, Y): X[summand] '+' Y[addend] { $$ = $summand + $addend } + ; +``` + +https://github.com/ruby/lrama/pull/410 + + +### Named References for actions of RHS in Parameterizing rules caller side + +Allow to use named references for actions of RHS in Parameterizing rules caller side. + +```yacc +opt_nl: '\n'?[nl] <str> { $$ = $nl; } + ; +``` + +https://github.com/ruby/lrama/pull/414 + +### Widen the definable position of Parameterizing rules + +Allow to define Parameterizing rules in the middle of the grammar. + +```yacc +%rule defined_option(X): /* empty */ + | X + ; + +%% + +program : defined_option(number) <i> + | defined_list(number) <i> + ; + +%rule defined_list(X): /* empty */ /* <--- here */ + | defined_list(X) number + ; +``` + +https://github.com/ruby/lrama/pull/420 + +### Report unused terminal symbols + +Support to report unused terminal symbols. +Run `exe/lrama --report=terms` to show unused terminal symbols. + +```console +$ exe/lrama --report=terms sample/calc.y + 11 Unused Terms + 0 YYerror + 1 YYUNDEF + 2 '\\\\' + 3 '\\13' + 4 keyword_class2 + 5 tNUMBER + 6 tPLUS + 7 tMINUS + 8 tEQ + 9 tEQEQ + 10 '>' +``` +https://github.com/ruby/lrama/pull/439 + +### Report unused rules + +Support to report unused rules. +Run `exe/lrama --report=rules` to show unused rules. + +```console +$ exe/lrama --report=rules sample/calc.y + 3 Unused Rules + 0 unused_option + 1 unused_list + 2 unused_nonempty_list +``` + +https://github.com/ruby/lrama/pull/441 + +### Ensure compatibility with Bison for `%locations` directive + +Support `%locations` directive to ensure compatibility with Bison. +Change to `%locations` directive not set by default. + +https://github.com/ruby/lrama/pull/446 + +### Diagnostics report for parameterized rules redefine + +Support to warning redefined parameterized rules. +Run `exe/lrama -W` or `exe/lrama --warnings` to show redefined parameterized rules. + +```console +$ exe/lrama -W sample/calc.y +parameterized rule redefined: redefined_method(X) +parameterized rule redefined: redefined_method(X) +``` + +https://github.com/ruby/lrama/pull/448 + +### Support `-v` and `--verbose` option + +Support to `-v` and `--verbose` option. +These options align with Bison behavior. So same as '--report=state' option. + +https://github.com/ruby/lrama/pull/457 + +## Lrama 0.6.9 (2024-05-02) + +### Callee side tag specification of Parameterizing rules + +Allow to specify tag on callee side of Parameterizing rules. + +```yacc +%union { + int i; +} + +%rule with_tag(X) <i>: X { $$ = $1; } + ; +``` + +### Named References for actions of RHS in Parameterizing rules + +Allow to use named references for actions of RHS in Parameterizing rules. + +```yacc +%rule option(number): /* empty */ + | number { $$ = $number; } + ; +``` + +## Lrama 0.6.8 (2024-04-29) + +### Nested Parameterizing rules with tag + +Allow to nested Parameterizing rules with tag. + +```yacc +%union { + int i; +} + +%rule nested_nested_option(X): /* empty */ + | X + ; + +%rule nested_option(X): /* empty */ + | nested_nested_option(X) <i> + ; + +%rule option(Y): /* empty */ + | nested_option(Y) <i> + ; +``` + +## Lrama 0.6.7 (2024-04-28) + +### RHS of user defined Parameterizing rules contains `'symbol'?`, `'symbol'+` and `'symbol'*`. + +User can use `'symbol'?`, `'symbol'+` and `'symbol'*` in RHS of user defined Parameterizing rules. + +``` +%rule with_word_seps(X): /* empty */ + | X ' '+ + ; +``` + +## Lrama 0.6.6 (2024-04-27) + +### Trace actions + +Support trace actions for debugging. +Run `exe/lrama --trace=actions` to show grammar rules with actions. + +```console +$ exe/lrama --trace=actions sample/calc.y +Grammar rules with actions: +$accept -> list, YYEOF {} +list -> ε {} +list -> list, LF {} +list -> list, expr, LF { printf("=> %d\n", $2); } +expr -> NUM {} +expr -> expr, '+', expr { $$ = $1 + $3; } +expr -> expr, '-', expr { $$ = $1 - $3; } +expr -> expr, '*', expr { $$ = $1 * $3; } +expr -> expr, '/', expr { $$ = $1 / $3; } +expr -> '(', expr, ')' { $$ = $2; } +``` + +### Inlining + +Support inlining for rules. +The `%inline` directive causes all references to symbols to be replaced with its definition. + +```yacc +%rule %inline op: PLUS { + } + | TIMES { * } + ; + +%% + +expr : number { $$ = $1; } + | expr op expr { $$ = $1 $2 $3; } + ; +``` + +as same as + +```yacc +expr : number { $$ = $1; } + | expr '+' expr { $$ = $1 + $3; } + | expr '*' expr { $$ = $1 * $3; } + ; +``` + +## Lrama 0.6.5 (2024-03-25) + +### Typed Midrule Actions + +User can specify the type of mid-rule action by tag (`<bar>`) instead of specifying it with in an action. + +```yacc +primary: k_case expr_value terms? + { + $<val>$ = p->case_labels; + p->case_labels = Qnil; + } + case_body + k_end + { + ... + } +``` + +can be written as + +```yacc +primary: k_case expr_value terms? + { + $$ = p->case_labels; + p->case_labels = Qnil; + }<val> + case_body + k_end + { + ... + } +``` + +`%destructor` for midrule action is invoked only when tag is specified by Typed Midrule Actions. + +Difference from Bison's Typed Midrule Actions is that tag is postposed in Lrama however it's preposed in Bison. + +Bison supports this feature from 3.1. + +## Lrama 0.6.4 (2024-03-22) + +### Parameterizing rules (preceded, terminated, delimited) + +Support `preceded`, `terminated` and `delimited` rules. + +```text +program: preceded(opening, X) + +// Expanded to + +program: preceded_opening_X +preceded_opening_X: opening X +``` + +``` +program: terminated(X, closing) + +// Expanded to + +program: terminated_X_closing +terminated_X_closing: X closing +``` + +``` +program: delimited(opening, X, closing) + +// Expanded to + +program: delimited_opening_X_closing +delimited_opening_X_closing: opening X closing +``` + +https://github.com/ruby/lrama/pull/382 + +### Support `%destructor` declaration + +User can set codes for freeing semantic value resources by using `%destructor`. +In general, these resources are freed by actions or after parsing. +However, if syntax error happens in parsing, these codes may not be executed. +Codes associated to `%destructor` are executed when semantic value is popped from the stack by an error. + +```yacc +%token <val1> NUM +%type <val2> expr2 +%type <val3> expr + +%destructor { + printf("destructor for val1: %d\n", $$); +} <val1> // printer for TAG + +%destructor { + printf("destructor for val2: %d\n", $$); +} <val2> + +%destructor { + printf("destructor for expr: %d\n", $$); +} expr // printer for symbol +``` + +Bison supports this feature from 1.75b. + +https://github.com/ruby/lrama/pull/385 + +## Lrama 0.6.3 (2024-02-15) + +### Bring Your Own Stack + +Provide functionalities for Bring Your Own Stack. + +Ruby’s Ripper library requires their own semantic value stack to manage Ruby Objects returned by user defined callback method. Currently Ripper uses semantic value stack (`yyvsa`) which is used by parser to manage Node. This hack introduces some limitation on Ripper. For example, Ripper can not execute semantic analysis depending on Node structure. + +Lrama introduces two features to support another semantic value stack by parser generator users. + +1. Callback entry points + +User can emulate semantic value stack by these callbacks. +Lrama provides these five callbacks. Registered functions are called when each event happens. For example %after-shift function is called when shift happens on original semantic value stack. + +* `%after-shift` function_name +* `%before-reduce` function_name +* `%after-reduce` function_name +* `%after-shift-error-token` function_name +* `%after-pop-stack` function_name + +2. `$:n` variable to access index of each grammar symbols + +User also needs to access semantic value of their stack in grammar action. `$:n` provides the way to access to it. `$:n` is translated to the minus index from the top of the stack. +For example + +```yacc +primary: k_if expr_value then compstmt if_tail k_end + { + /*% ripper: if!($:2, $:4, $:5) %*/ + /* $:2 = -5, $:4 = -3, $:5 = -2. */ + } +``` + +https://github.com/ruby/lrama/pull/367 + +## Lrama 0.6.2 (2024-01-27) + +### %no-stdlib directive + +If `%no-stdlib` directive is set, Lrama doesn't load Lrama standard library for +parameterized rules, stdlib.y. + +https://github.com/ruby/lrama/pull/344 + +## Lrama 0.6.1 (2024-01-13) + +### Nested Parameterizing rules + +Allow to pass an instantiated rule to other Parameterizing rules. + +```yacc +%rule constant(X) : X + ; + +%rule option(Y) : /* empty */ + | Y + ; + +%% + +program : option(constant(number)) // Nested rule + ; +%% +``` + +Allow to use nested Parameterizing rules when define Parameterizing rules. + +```yacc +%rule option(x) : /* empty */ + | X + ; + +%rule double(Y) : Y Y + ; + +%rule double_opt(A) : option(double(A)) // Nested rule + ; + +%% + +program : double_opt(number) + ; + +%% +``` + +https://github.com/ruby/lrama/pull/337 + +## Lrama 0.6.0 (2023-12-25) + +### User defined Parameterizing rules + +Allow to define Parameterizing rule by `%rule` directive. + +```yacc +%rule pair(X, Y): X Y { $$ = $1 + $2; } + ; + +%% + +program: stmt + ; + +stmt: pair(ODD, EVEN) <num> + | pair(EVEN, ODD) <num> + ; +``` + +https://github.com/ruby/lrama/pull/285 + +## Lrama 0.5.11 (2023-12-02) + +### Type specification of Parameterizing rules + +Allow to specify type of rules by specifying tag, `<i>` in below example. +Tag is post-modification style. + +```yacc +%union { + int i; +} + +%% + +program : option(number) <i> + | number_alias? <i> + ; +``` + +https://github.com/ruby/lrama/pull/272 + + +## Lrama 0.5.10 (2023-11-18) + +### Parameterizing rules (option, nonempty_list, list) + +Support function call style Parameterizing rules for `option`, `nonempty_list` and `list`. + +https://github.com/ruby/lrama/pull/197 + +### Parameterizing rules (separated_list) + +Support `separated_list` and `separated_nonempty_list` Parameterizing rules. + +```text +program: separated_list(',', number) + +// Expanded to + +program: separated_list_number +separated_list_number: ε +separated_list_number: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +``` +program: separated_nonempty_list(',', number) + +// Expanded to + +program: separated_nonempty_list_number +separated_nonempty_list_number: number +separated_nonempty_list_number: separated_nonempty_list_number ',' number +``` + +https://github.com/ruby/lrama/pull/204 + +## Lrama 0.5.9 (2023-11-05) + +### Parameterizing rules (suffix) + +Parameterizing rules are template of rules. +It's very common pattern to write "list" grammar rule like: + +```yacc +opt_args: /* none */ + | args + ; + +args: arg + | args arg +``` + +Lrama supports these suffixes: + +* `?`: option +* `+`: nonempty list +* `*`: list + +Idea of Parameterizing rules comes from Menhir LR(1) parser generator (https://gallium.inria.fr/~fpottier/menhir/manual.html#sec32). + +https://github.com/ruby/lrama/pull/181 + +## Lrama 0.5.7 (2023-10-23) + +### Racc parser + +Replace Lrama's parser from handwritten parser to LR parser generated by Racc. +Lrama uses `--embedded` option to generate LR parser because Racc is changed from default gem to bundled gem by Ruby 3.3 (https://github.com/ruby/lrama/pull/132). + +https://github.com/ruby/lrama/pull/62 + +## Lrama 0.5.4 (2023-08-17) + +### Runtime configuration for error recovery + +Make error recovery function configurable on runtime by two new macros. + +* `YYMAXREPAIR`: Expected to return max length of repair operations. `%parse-param` is passed to this function. +* `YYERROR_RECOVERY_ENABLED`: Expected to return bool value to determine error recovery is enabled or not. `%parse-param` is passed to this function. + +https://github.com/ruby/lrama/pull/74 + +## Lrama 0.5.3 (2023-08-05) + +### Error Recovery + +Support token insert base Error Recovery. +`-e` option is needed to generate parser with error recovery functions. + +https://github.com/ruby/lrama/pull/44 + +## Lrama 0.5.2 (2023-06-14) + +### Named References + +Instead of positional references like `$1` or `$$`, +named references allow to access to symbol by name. + +```yacc +primary: k_class cpath superclass bodystmt k_end + { + $primary = new_class($cpath, $bodystmt, $superclass); + } +``` + +Alias name can be declared. + +```yacc +expr[result]: expr[ex-left] '+' expr[ex.right] + { + $result = $[ex-left] + $[ex.right]; + } +``` + +Bison supports this feature from 2.5. + +### Add parse params to some macros and functions + +`%parse-param` are added to these macros and functions to remove ytab.sed hack from Ruby. + +* `YY_LOCATION_PRINT` +* `YY_SYMBOL_PRINT` +* `yy_stack_print` +* `YY_STACK_PRINT` +* `YY_REDUCE_PRINT` +* `yysyntax_error` + +https://github.com/ruby/lrama/pull/40 + +See also: https://github.com/ruby/ruby/pull/7807 + +## Lrama 0.5.0 (2023-05-17) + +### stdin mode + +When `-` is given as grammar file name, reads the grammar source from STDIN, and takes the next argument as the input file name. This mode helps pre-process a grammar source. + +https://github.com/ruby/lrama/pull/8 + +## Lrama 0.4.0 (2023-05-13) + +This is the first version migrated to Ruby. +This version generates "parse.c" compatible with Bison 3.8.2. diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama new file mode 100755 index 0000000000..710ac0cb96 --- /dev/null +++ b/tool/lrama/exe/lrama @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +$LOAD_PATH << File.join(__dir__, "../lib") +require "lrama" + +Lrama::Command.new(ARGV.dup).run diff --git a/tool/lrama/lib/lrama.rb b/tool/lrama/lib/lrama.rb new file mode 100644 index 0000000000..56ba0044d4 --- /dev/null +++ b/tool/lrama/lib/lrama.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative "lrama/bitmap" +require_relative "lrama/command" +require_relative "lrama/context" +require_relative "lrama/counterexamples" +require_relative "lrama/diagram" +require_relative "lrama/digraph" +require_relative "lrama/erb" +require_relative "lrama/grammar" +require_relative "lrama/lexer" +require_relative "lrama/logger" +require_relative "lrama/option_parser" +require_relative "lrama/options" +require_relative "lrama/output" +require_relative "lrama/parser" +require_relative "lrama/reporter" +require_relative "lrama/state" +require_relative "lrama/states" +require_relative "lrama/tracer" +require_relative "lrama/version" +require_relative "lrama/warnings" diff --git a/tool/lrama/lib/lrama/bitmap.rb b/tool/lrama/lib/lrama/bitmap.rb new file mode 100644 index 0000000000..88b255b012 --- /dev/null +++ b/tool/lrama/lib/lrama/bitmap.rb @@ -0,0 +1,47 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + module Bitmap + # @rbs! + # type bitmap = Integer + + # @rbs (Array[Integer] ary) -> bitmap + def self.from_array(ary) + bit = 0 + + ary.each do |int| + bit |= (1 << int) + end + + bit + end + + # @rbs (Integer int) -> bitmap + def self.from_integer(int) + 1 << int + end + + # @rbs (bitmap int) -> Array[Integer] + def self.to_array(int) + a = [] #: Array[Integer] + i = 0 + + len = int.bit_length + while i < len do + if int[i] == 1 + a << i + end + + i += 1 + end + + a + end + + # @rbs (bitmap int, Integer size) -> Array[bool] + def self.to_bool_array(int, size) + Array.new(size) { |i| int[i] == 1 } + end + end +end diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb new file mode 100644 index 0000000000..17aad1a1c1 --- /dev/null +++ b/tool/lrama/lib/lrama/command.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Lrama + class Command + LRAMA_LIB = File.realpath(File.join(File.dirname(__FILE__))) + STDLIB_FILE_PATH = File.join(LRAMA_LIB, 'grammar', 'stdlib.y') + + def initialize(argv) + @logger = Lrama::Logger.new + @options = OptionParser.parse(argv) + @tracer = Tracer.new(STDERR, **@options.trace_opts) + @reporter = Reporter.new(**@options.report_opts) + @warnings = Warnings.new(@logger, @options.warnings) + rescue => e + abort format_error_message(e.message) + end + + def run + Lrama::Reporter::Profile::CallStack.report(@options.profile_opts[:call_stack]) do + Lrama::Reporter::Profile::Memory.report(@options.profile_opts[:memory]) do + execute_command_workflow + end + end + end + + private + + def execute_command_workflow + @tracer.enable_duration + text = read_input + grammar = build_grammar(text) + states, context = compute_status(grammar) + render_reports(states) if @options.report_file + @tracer.trace(grammar) + render_diagram(grammar) + render_output(context, grammar) + states.validate!(@logger) + @warnings.warn(grammar, states) + end + + def read_input + text = @options.y.read + @options.y.close unless @options.y == STDIN + text + end + + def build_grammar(text) + grammar = + Lrama::Parser.new(text, @options.grammar_file, @options.debug, @options.locations, @options.define).parse + merge_stdlib(grammar) + prepare_grammar(grammar) + grammar + rescue => e + raise e if @options.debug + abort format_error_message(e.message) + end + + def format_error_message(message) + return message unless Exception.to_tty? + + message.gsub(/.+/, "\e[1m\\&\e[m") + end + + def merge_stdlib(grammar) + return if grammar.no_stdlib + + stdlib_text = File.read(STDLIB_FILE_PATH) + stdlib_grammar = Lrama::Parser.new( + stdlib_text, + STDLIB_FILE_PATH, + @options.debug, + @options.locations, + @options.define, + ).parse + + grammar.prepend_parameterized_rules(stdlib_grammar.parameterized_rules) + end + + def prepare_grammar(grammar) + grammar.prepare + grammar.validate! + end + + def compute_status(grammar) + states = Lrama::States.new(grammar, @tracer) + states.compute + states.compute_ielr if grammar.ielr_defined? + [states, Lrama::Context.new(states)] + end + + def render_reports(states) + File.open(@options.report_file, "w+") do |f| + @reporter.report(f, states) + end + end + + def render_diagram(grammar) + return unless @options.diagram + + File.open(@options.diagram_file, "w+") do |f| + Lrama::Diagram.render(out: f, grammar: grammar) + end + end + + def render_output(context, grammar) + File.open(@options.outfile, "w+") do |f| + Lrama::Output.new( + out: f, + output_file_path: @options.outfile, + template_name: @options.skeleton, + grammar_file_path: @options.grammar_file, + header_file_path: @options.header_file, + context: context, + grammar: grammar, + error_recovery: @options.error_recovery, + ).render + end + end + end +end diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb new file mode 100644 index 0000000000..eb068c1b9e --- /dev/null +++ b/tool/lrama/lib/lrama/context.rb @@ -0,0 +1,497 @@ +# frozen_string_literal: true + +require_relative "tracer/duration" + +module Lrama + # This is passed to a template + class Context + include Tracer::Duration + + ErrorActionNumber = -Float::INFINITY + BaseMin = -Float::INFINITY + + # TODO: It might be better to pass `states` to Output directly? + attr_reader :states, :yylast, :yypact_ninf, :yytable_ninf, :yydefact, :yydefgoto + + def initialize(states) + @states = states + @yydefact = nil + @yydefgoto = nil + # Array of array + @_actions = [] + + compute_tables + end + + # enum yytokentype + def yytokentype + @states.terms.reject do |term| + 0 < term.token_id && term.token_id < 128 + end.map do |term| + [term.id.s_value, term.token_id, term.display_name] + end.unshift(["YYEMPTY", -2, nil]) + end + + # enum yysymbol_kind_t + def yysymbol_kind_t + @states.symbols.map do |sym| + [sym.enum_name, sym.number, sym.comment] + end.unshift(["YYSYMBOL_YYEMPTY", -2, nil]) + end + + # State number of final (accepted) state + def yyfinal + @states.states.find do |state| + state.items.find do |item| + item.lhs.accept_symbol? && item.end_of_rule? + end + end.id + end + + # Number of terms + def yyntokens + @states.terms.count + end + + # Number of nterms + def yynnts + @states.nterms.count + end + + # Number of rules + def yynrules + @states.rules.count + end + + # Number of states + def yynstates + @states.states.count + end + + # Last token number + def yymaxutok + @states.terms.map(&:token_id).max + end + + # YYTRANSLATE + # + # yytranslate is a mapping from token id to symbol number + def yytranslate + # 2 is YYSYMBOL_YYUNDEF + a = Array.new(yymaxutok, 2) + + @states.terms.each do |term| + a[term.token_id] = term.number + end + + return a + end + + def yytranslate_inverted + a = Array.new(@states.symbols.count, @states.undef_symbol.token_id) + + @states.terms.each do |term| + a[term.number] = term.token_id + end + + return a + end + + # Mapping from rule number to line number of the rule is defined. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyrline + a = [0] + + @states.rules.each do |rule| + a << rule.lineno + end + + return a + end + + # Mapping from symbol number to its name + def yytname + @states.symbols.sort_by(&:number).map do |sym| + sym.display_name + end + end + + def yypact + @base[0...yynstates] + end + + def yypgoto + @base[yynstates..-1] + end + + def yytable + @table + end + + def yycheck + @check + end + + def yystos + @states.states.map do |state| + state.accessing_symbol.number + end + end + + # Mapping from rule number to symbol number of LHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr1 + a = [0] + + @states.rules.each do |rule| + a << rule.lhs.number + end + + return a + end + + # Mapping from rule number to length of RHS. + # Dummy rule is appended as the first element whose value is 0 + # because 0 means error in yydefact. + def yyr2 + a = [0] + + @states.rules.each do |rule| + a << rule.rhs.count + end + + return a + end + + private + + # Compute these + # + # See also: "src/tables.c" of Bison. + # + # * yydefact + # * yydefgoto + # * yypact and yypgoto + # * yytable + # * yycheck + # * yypact_ninf + # * yytable_ninf + def compute_tables + report_duration(:compute_yydefact) { compute_yydefact } + report_duration(:compute_yydefgoto) { compute_yydefgoto } + report_duration(:sort_actions) { sort_actions } + # debug_sorted_actions + report_duration(:compute_packed_table) { compute_packed_table } + end + + def vectors_count + @states.states.count + @states.nterms.count + end + + # In compressed table, rule 0 is appended as an error case + # and reduce is represented as minus number. + def rule_id_to_action_number(rule_id) + (rule_id + 1) * -1 + end + + # Symbol number is assigned to term first then nterm. + # This method calculates sequence_number for nterm. + def nterm_number_to_sequence_number(nterm_number) + nterm_number - @states.terms.count + end + + # Vector is states + nterms + def nterm_number_to_vector_number(nterm_number) + @states.states.count + (nterm_number - @states.terms.count) + end + + def compute_yydefact + # Default action (shift/reduce/error) for each state. + # Index is state id, value is `rule id + 1` of a default reduction. + @yydefact = Array.new(@states.states.count, 0) + + @states.states.each do |state| + # Action number means + # + # * number = 0, default action + # * number = -Float::INFINITY, error by %nonassoc + # * number > 0, shift then move to state "number" + # * number < 0, reduce by "-number" rule. Rule "number" is already added by 1. + actions = Array.new(@states.terms.count, 0) + + if state.reduces.map(&:selected_look_ahead).any? {|la| !la.empty? } + # Iterate reduces with reverse order so that first rule is used. + state.reduces.reverse_each do |reduce| + reduce.look_ahead.each do |term| + actions[term.number] = rule_id_to_action_number(reduce.rule.id) + end + end + end + + # Shift is selected when S/R conflict exists. + state.selected_term_transitions.each do |shift| + actions[shift.next_sym.number] = shift.to_state.id + end + + state.resolved_conflicts.select do |conflict| + conflict.which == :error + end.each do |conflict| + actions[conflict.symbol.number] = ErrorActionNumber + end + + # If default_reduction_rule, replace default_reduction_rule in + # actions with zero. + if state.default_reduction_rule + actions.map! do |e| + if e == rule_id_to_action_number(state.default_reduction_rule.id) + 0 + else + e + end + end + end + + # If no default_reduction_rule, default behavior is an + # error then replace ErrorActionNumber with zero. + unless state.default_reduction_rule + actions.map! do |e| + if e == ErrorActionNumber + 0 + else + e + end + end + end + + s = actions.each_with_index.map do |n, i| + [i, n] + end.reject do |i, n| + # Remove default_reduction_rule entries + n == 0 + end + + if s.count != 0 + # Entry of @_actions is an array of + # + # * State id + # * Array of tuple, [from, to] where from is term number and to is action. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [state.id, s, s.count, s.last[0] - s.first[0] + 1] + end + + @yydefact[state.id] = state.default_reduction_rule ? state.default_reduction_rule.id + 1 : 0 + end + end + + def compute_yydefgoto + # Default GOTO (nterm transition) for each nterm. + # Index is sequence number of nterm, value is state id + # of a default nterm transition destination. + @yydefgoto = Array.new(@states.nterms.count, 0) + # Mapping from nterm to next_states + nterm_to_to_states = {} + + @states.states.each do |state| + state.nterm_transitions.each do |goto| + key = goto.next_sym + nterm_to_to_states[key] ||= [] + nterm_to_to_states[key] << [state, goto.to_state] # [from_state, to_state] + end + end + + @states.nterms.each do |nterm| + if (states = nterm_to_to_states[nterm]) + default_state = states.map(&:last).group_by {|s| s }.max_by {|_, v| v.count }.first + default_goto = default_state.id + not_default_gotos = [] + states.each do |from_state, to_state| + next if to_state.id == default_goto + not_default_gotos << [from_state.id, to_state.id] + end + else + default_goto = 0 + not_default_gotos = [] + end + + k = nterm_number_to_sequence_number(nterm.number) + @yydefgoto[k] = default_goto + + if not_default_gotos.count != 0 + v = nterm_number_to_vector_number(nterm.number) + + # Entry of @_actions is an array of + # + # * Nterm number as vector number + # * Array of tuple, [from, to] where from is state number and to is state number. + # * The number of "Array of tuple" used by sort_actions + # * "width" used by sort_actions + @_actions << [v, not_default_gotos, not_default_gotos.count, not_default_gotos.last[0] - not_default_gotos.first[0] + 1] + end + end + end + + def sort_actions + # This is not same with #sort_actions + # + # @sorted_actions = @_actions.sort_by do |_, _, count, width| + # [-width, -count] + # end + + @sorted_actions = [] + + @_actions.each do |action| + if @sorted_actions.empty? + @sorted_actions << action + next + end + + j = @sorted_actions.count - 1 + _state_id, _froms_and_tos, count, width = action + + while (j >= 0) do + case + when @sorted_actions[j][3] < width + j -= 1 + when @sorted_actions[j][3] == width && @sorted_actions[j][2] < count + j -= 1 + else + break + end + end + + @sorted_actions.insert(j + 1, action) + end + end + + def debug_sorted_actions + ary = Array.new + @sorted_actions.each do |state_id, froms_and_tos, count, width| + ary[state_id] = [state_id, froms_and_tos, count, width] + end + + print sprintf("table_print:\n\n") + + print sprintf("order [\n") + vectors_count.times do |i| + print sprintf("%d, ", @sorted_actions[i] ? @sorted_actions[i][0] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("width [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][3] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + + print sprintf("tally [\n") + vectors_count.times do |i| + print sprintf("%d, ", ary[i] ? ary[i][2] : 0) + print "\n" if i % 10 == 9 + end + print sprintf("]\n\n") + end + + def compute_packed_table + # yypact and yypgoto + @base = Array.new(vectors_count, BaseMin) + # yytable + @table = [] + # yycheck + @check = [] + # Key is froms_and_tos, value is index position + pushed = {} + used_res = {} + lowzero = 0 + high = 0 + + @sorted_actions.each do |state_id, froms_and_tos, _, _| + if (res = pushed[froms_and_tos]) + @base[state_id] = res + next + end + + res = lowzero - froms_and_tos.first[0] + + # Find the smallest `res` such that `@table[res + from]` is empty for all `from` in `froms_and_tos` + while true do + advanced = false + + while used_res[res] + res += 1 + advanced = true + end + + froms_and_tos.each do |from, to| + while @table[res + from] + res += 1 + advanced = true + end + end + + unless advanced + # no advance means that the current `res` satisfies the condition + break + end + end + + loc = 0 + + froms_and_tos.each do |from, to| + loc = res + from + + @table[loc] = to + @check[loc] = from + end + + while (@table[lowzero]) do + lowzero += 1 + end + + high = loc if high < loc + + @base[state_id] = res + pushed[froms_and_tos] = res + used_res[res] = true + end + + @yylast = high + + # replace_ninf + @yypact_ninf = (@base.reject {|i| i == BaseMin } + [0]).min - 1 + @base.map! do |i| + case i + when BaseMin + @yypact_ninf + else + i + end + end + + @yytable_ninf = (@table.compact.reject {|i| i == ErrorActionNumber } + [0]).min - 1 + @table.map! do |i| + case i + when nil + 0 + when ErrorActionNumber + @yytable_ninf + else + i + end + end + + @check.map! do |i| + case i + when nil + -1 + else + i + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb new file mode 100644 index 0000000000..60d830d048 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -0,0 +1,426 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "set" +require "timeout" + +require_relative "counterexamples/derivation" +require_relative "counterexamples/example" +require_relative "counterexamples/node" +require_relative "counterexamples/path" +require_relative "counterexamples/state_item" +require_relative "counterexamples/triple" + +module Lrama + # See: https://www.cs.cornell.edu/andru/papers/cupex/cupex.pdf + # 4. Constructing Nonunifying Counterexamples + class Counterexamples + PathSearchTimeLimit = 10 # 10 sec + CumulativeTimeLimit = 120 # 120 sec + + # @rbs! + # @states: States + # @iterate_count: Integer + # @total_duration: Float + # @exceed_cumulative_time_limit: bool + # @state_items: Hash[[State, State::Item], StateItem] + # @triples: Hash[Integer, Triple] + # @transitions: Hash[[StateItem, Grammar::Symbol], StateItem] + # @reverse_transitions: Hash[[StateItem, Grammar::Symbol], Set[StateItem]] + # @productions: Hash[StateItem, Set[StateItem]] + # @reverse_productions: Hash[[State, Grammar::Symbol], Set[StateItem]] # Grammar::Symbol is nterm + # @state_item_shift: Integer + + attr_reader :transitions #: Hash[[StateItem, Grammar::Symbol], StateItem] + attr_reader :productions #: Hash[StateItem, Set[StateItem]] + + # @rbs (States states) -> void + def initialize(states) + @states = states + @iterate_count = 0 + @total_duration = 0 + @exceed_cumulative_time_limit = false + @triples = {} + setup_state_items + setup_transitions + setup_productions + end + + # @rbs () -> "#<Counterexamples>" + def to_s + "#<Counterexamples>" + end + alias :inspect :to_s + + # @rbs (State conflict_state) -> Array[Example] + def compute(conflict_state) + conflict_state.conflicts.flat_map do |conflict| + # Check cumulative time limit for not each path search method call but each conflict + # to avoid one of example's path to be nil. + next if @exceed_cumulative_time_limit + + case conflict.type + when :shift_reduce + # @type var conflict: State::ShiftReduceConflict + shift_reduce_example(conflict_state, conflict) + when :reduce_reduce + # @type var conflict: State::ReduceReduceConflict + reduce_reduce_examples(conflict_state, conflict) + end + rescue Timeout::Error => e + STDERR.puts "Counterexamples calculation for state #{conflict_state.id} #{e.message} with #{@iterate_count} iteration" + increment_total_duration(PathSearchTimeLimit) + nil + end.compact + end + + private + + # @rbs (State state, State::Item item) -> StateItem + def get_state_item(state, item) + @state_items[[state, item]] + end + + # For optimization, create all StateItem in advance + # and use them by fetching an instance from `@state_items`. + # Do not create new StateItem instance in the shortest path search process + # to avoid miss hash lookup. + # + # @rbs () -> void + def setup_state_items + @state_items = {} + count = 0 + + @states.states.each do |state| + state.items.each do |item| + @state_items[[state, item]] = StateItem.new(count, state, item) + count += 1 + end + end + + @state_item_shift = Math.log(count, 2).ceil + end + + # @rbs () -> void + def setup_transitions + @transitions = {} + @reverse_transitions = {} + + @states.states.each do |src_state| + trans = {} #: Hash[Grammar::Symbol, State] + + src_state.transitions.each do |transition| + trans[transition.next_sym] = transition.to_state + end + + src_state.items.each do |src_item| + next if src_item.end_of_rule? + sym = src_item.next_sym + dest_state = trans[sym] + + dest_state.kernels.each do |dest_item| + next unless (src_item.rule == dest_item.rule) && (src_item.position + 1 == dest_item.position) + src_state_item = get_state_item(src_state, src_item) + dest_state_item = get_state_item(dest_state, dest_item) + + @transitions[[src_state_item, sym]] = dest_state_item + + # @type var key: [StateItem, Grammar::Symbol] + key = [dest_state_item, sym] + @reverse_transitions[key] ||= Set.new + @reverse_transitions[key] << src_state_item + end + end + end + end + + # @rbs () -> void + def setup_productions + @productions = {} + @reverse_productions = {} + + @states.states.each do |state| + # Grammar::Symbol is LHS + h = {} #: Hash[Grammar::Symbol, Set[StateItem]] + + state.closure.each do |item| + sym = item.lhs + + h[sym] ||= Set.new + h[sym] << get_state_item(state, item) + end + + state.items.each do |item| + next if item.end_of_rule? + next if item.next_sym.term? + + sym = item.next_sym + state_item = get_state_item(state, item) + @productions[state_item] = h[sym] + + # @type var key: [State, Grammar::Symbol] + key = [state, sym] + @reverse_productions[key] ||= Set.new + @reverse_productions[key] << state_item + end + end + end + + # For optimization, use same Triple if it's already created. + # Do not create new Triple instance anywhere else + # to avoid miss hash lookup. + # + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> Triple + def get_triple(state_item, precise_lookahead_set) + key = (precise_lookahead_set << @state_item_shift) | state_item.id + @triples[key] ||= Triple.new(state_item, precise_lookahead_set) + end + + # @rbs (State conflict_state, State::ShiftReduceConflict conflict) -> Example + def shift_reduce_example(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + # @type var shift_conflict_item: ::Lrama::State::Item + shift_conflict_item = conflict_state.items.find { |item| item.next_sym == conflict_symbol } + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce.item, conflict_symbol) + end + path1 = with_timeout("#find_shift_conflict_shortest_path:") do + find_shift_conflict_shortest_path(path2, conflict_state, shift_conflict_item) + end + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + # @rbs (State conflict_state, State::ReduceReduceConflict conflict) -> Example + def reduce_reduce_examples(conflict_state, conflict) + conflict_symbol = conflict.symbols.first + path1 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce1.item, conflict_symbol) + end + path2 = with_timeout("#shortest_path:") do + shortest_path(conflict_state, conflict.reduce2.item, conflict_symbol) + end + + Example.new(path1, path2, conflict, conflict_symbol, self) + end + + # @rbs (Array[StateItem]? reduce_state_items, State conflict_state, State::Item conflict_item) -> Array[StateItem] + def find_shift_conflict_shortest_path(reduce_state_items, conflict_state, conflict_item) + time1 = Time.now.to_f + @iterate_count = 0 + + target_state_item = get_state_item(conflict_state, conflict_item) + result = [target_state_item] + reversed_state_items = reduce_state_items.to_a.reverse + # Index for state_item + i = 0 + + while (state_item = reversed_state_items[i]) + # Index for prev_state_item + j = i + 1 + _j = j + + while (prev_state_item = reversed_state_items[j]) + if prev_state_item.type == :production + j += 1 + else + break + end + end + + if target_state_item == state_item || target_state_item.item.start_item? + result.concat( + reversed_state_items[_j..-1] #: Array[StateItem] + ) + break + end + + if target_state_item.type == :production + queue = [] #: Array[Node[StateItem]] + queue << Node.new(target_state_item, nil) + + # Find reverse production + while (sis = queue.shift) + @iterate_count += 1 + si = sis.elem + + # Reach to start state + if si.item.start_item? + a = Node.to_a(sis).reverse + a.shift + result.concat(a) + target_state_item = si + break + end + + if si.type == :production + # @type var key: [State, Grammar::Symbol] + key = [si.state, si.item.lhs] + @reverse_productions[key].each do |state_item| + queue << Node.new(state_item, sis) + end + else + # @type var key: [StateItem, Grammar::Symbol] + key = [si, si.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item&.state + a = Node.to_a(sis).reverse + a.shift + result.concat(a) + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + queue.clear + break + end + end + end + else + # Find reverse transition + # @type var key: [StateItem, Grammar::Symbol] + key = [target_state_item, target_state_item.item.previous_sym] + @reverse_transitions[key].each do |prev_target_state_item| + next if prev_target_state_item.state != prev_state_item&.state + result << prev_target_state_item + target_state_item = prev_target_state_item + i = j + break + end + end + end + + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "find_shift_conflict_shortest_path #{@iterate_count} iteration", duration) + end + + result.reverse + end + + # @rbs (StateItem target) -> Set[StateItem] + def reachable_state_items(target) + result = Set.new + queue = [target] + + while (state_item = queue.shift) + next if result.include?(state_item) + result << state_item + + @reverse_transitions[[state_item, state_item.item.previous_sym]]&.each do |prev_state_item| + queue << prev_state_item + end + + if state_item.item.beginning_of_rule? + @reverse_productions[[state_item.state, state_item.item.lhs]]&.each do |si| + queue << si + end + end + end + + result + end + + # @rbs (State conflict_state, State::Item conflict_reduce_item, Grammar::Symbol conflict_term) -> ::Array[StateItem]? + def shortest_path(conflict_state, conflict_reduce_item, conflict_term) + time1 = Time.now.to_f + @iterate_count = 0 + + queue = [] #: Array[[Triple, Path]] + visited = {} #: Hash[Triple, true] + start_state = @states.states.first #: Lrama::State + conflict_term_bit = Bitmap::from_integer(conflict_term.number) + raise "BUG: Start state should be just one kernel." if start_state.kernels.count != 1 + reachable = reachable_state_items(get_state_item(conflict_state, conflict_reduce_item)) + start = get_triple(get_state_item(start_state, start_state.kernels.first), Bitmap::from_integer(@states.eof_symbol.number)) + + queue << [start, Path.new(start.state_item, nil)] + + while (triple, path = queue.shift) + @iterate_count += 1 + + # Found + if (triple.state == conflict_state) && (triple.item == conflict_reduce_item) && (triple.l & conflict_term_bit != 0) + state_items = [path.state_item] + + while (path = path.parent) + state_items << path.state_item + end + + time2 = Time.now.to_f + duration = time2 - time1 + increment_total_duration(duration) + + if Tracer::Duration.enabled? + STDERR.puts sprintf(" %s %10.5f s", "shortest_path #{@iterate_count} iteration", duration) + end + + return state_items.reverse + end + + # transition + next_state_item = @transitions[[triple.state_item, triple.item.next_sym]] + if next_state_item && reachable.include?(next_state_item) + # @type var t: Triple + t = get_triple(next_state_item, triple.l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end + end + + # production step + @productions[triple.state_item]&.each do |si| + next unless reachable.include?(si) + + l = follow_l(triple.item, triple.l) + # @type var t: Triple + t = get_triple(si, l) + unless visited[t] + visited[t] = true + queue << [t, Path.new(t.state_item, path)] + end + end + end + + return nil + end + + # @rbs (State::Item item, Bitmap::bitmap current_l) -> Bitmap::bitmap + def follow_l(item, current_l) + # 1. follow_L (A -> X1 ... Xn-1 • Xn) = L + # 2. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = {Xk+2} if Xk+2 is a terminal + # 3. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) if Xk+2 is a nonnullable nonterminal + # 4. follow_L (A -> X1 ... Xk • Xk+1 Xk+2 ... Xn) = FIRST(Xk+2) + follow_L (A -> X1 ... Xk+1 • Xk+2 ... Xn) if Xk+2 is a nullable nonterminal + case + when item.number_of_rest_symbols == 1 + current_l + when item.next_next_sym.term? + item.next_next_sym.number_bitmap + when !item.next_next_sym.nullable + item.next_next_sym.first_set_bitmap + else + item.next_next_sym.first_set_bitmap | follow_l(item.new_by_next_position, current_l) + end + end + + # @rbs [T] (String message) { -> T } -> T + def with_timeout(message) + Timeout.timeout(PathSearchTimeLimit, Timeout::Error, message + " timeout of #{PathSearchTimeLimit} sec exceeded") do + yield + end + end + + # @rbs (Float|Integer duration) -> void + def increment_total_duration(duration) + @total_duration += duration + + if !@exceed_cumulative_time_limit && @total_duration > CumulativeTimeLimit + @exceed_cumulative_time_limit = true + STDERR.puts "CumulativeTimeLimit #{CumulativeTimeLimit} sec exceeded then skip following Counterexamples calculation" + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/derivation.rb b/tool/lrama/lib/lrama/counterexamples/derivation.rb new file mode 100644 index 0000000000..a2b74767a9 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/derivation.rb @@ -0,0 +1,76 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Derivation + # @rbs! + # @item: State::Item + # @left: Derivation? + + attr_reader :item #: State::Item + attr_reader :left #: Derivation? + attr_accessor :right #: Derivation? + + # @rbs (State::Item item, Derivation? left) -> void + def initialize(item, left) + @item = item + @left = left + end + + # @rbs () -> ::String + def to_s + "#<Derivation(#{item.display_name})>" + end + alias :inspect :to_s + + # @rbs () -> Array[String] + def render_strings_for_report + result = [] #: Array[String] + _render_for_report(self, 0, result, 0) + result.map(&:rstrip) + end + + # @rbs () -> String + def render_for_report + render_strings_for_report.join("\n") + end + + private + + # @rbs (Derivation derivation, Integer offset, Array[String] strings, Integer index) -> Integer + def _render_for_report(derivation, offset, strings, index) + item = derivation.item + if strings[index] + strings[index] << " " * (offset - strings[index].length) + else + strings[index] = " " * offset + end + str = strings[index] + str << "#{item.rule_id}: #{item.symbols_before_dot.map(&:display_name).join(" ")} " + + if derivation.left + len = str.length + str << "#{item.next_sym.display_name}" + length = _render_for_report(derivation.left, len, strings, index + 1) + # I want String#ljust! + str << " " * (length - str.length) if length > str.length + else + str << " • #{item.symbols_after_dot.map(&:display_name).join(" ")} " + return str.length + end + + if derivation.right&.left + left = derivation.right&.left #: Derivation + length = _render_for_report(left, str.length, strings, index + 1) + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore + str << " " * (length - str.length) if length > str.length + elsif item.next_next_sym + str << "#{item.symbols_after_dot[1..-1].map(&:display_name).join(" ")} " # steep:ignore + end + + return str.length + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/example.rb b/tool/lrama/lib/lrama/counterexamples/example.rb new file mode 100644 index 0000000000..c007f45af4 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/example.rb @@ -0,0 +1,154 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Example + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @path1: ::Array[StateItem] + # @path2: ::Array[StateItem] + # @conflict: State::conflict + # @conflict_symbol: Grammar::Symbol + # @counterexamples: Counterexamples + # @derivations1: Derivation + # @derivations2: Derivation + + attr_reader :path1 #: ::Array[StateItem] + attr_reader :path2 #: ::Array[StateItem] + attr_reader :conflict #: State::conflict + attr_reader :conflict_symbol #: Grammar::Symbol + + # path1 is shift conflict when S/R conflict + # path2 is always reduce conflict + # + # @rbs (Array[StateItem]? path1, Array[StateItem]? path2, State::conflict conflict, Grammar::Symbol conflict_symbol, Counterexamples counterexamples) -> void + def initialize(path1, path2, conflict, conflict_symbol, counterexamples) + @path1 = path1 + @path2 = path2 + @conflict = conflict + @conflict_symbol = conflict_symbol + @counterexamples = counterexamples + end + + # @rbs () -> (:shift_reduce | :reduce_reduce) + def type + @conflict.type + end + + # @rbs () -> State::Item + def path1_item + @path1.last.item + end + + # @rbs () -> State::Item + def path2_item + @path2.last.item + end + + # @rbs () -> Derivation + def derivations1 + @derivations1 ||= _derivations(path1) + end + + # @rbs () -> Derivation + def derivations2 + @derivations2 ||= _derivations(path2) + end + + private + + # @rbs (Array[StateItem] state_items) -> Derivation + def _derivations(state_items) + derivation = nil #: Derivation + current = :production + last_state_item = state_items.last #: StateItem + lookahead_sym = last_state_item.item.end_of_rule? ? @conflict_symbol : nil + + state_items.reverse_each do |si| + item = si.item + + case current + when :production + case si.type + when :start + derivation = Derivation.new(item, derivation) + current = :start + when :transition + derivation = Derivation.new(item, derivation) + current = :transition + when :production + derivation = Derivation.new(item, derivation) + current = :production + else + raise "Unexpected. #{si}" + end + + if lookahead_sym && item.next_next_sym && item.next_next_sym.first_set.include?(lookahead_sym) + si2 = @counterexamples.transitions[[si, item.next_sym]] + derivation2 = find_derivation_for_symbol(si2, lookahead_sym) + derivation.right = derivation2 # steep:ignore + lookahead_sym = nil + end + + when :transition + case si.type + when :start + derivation = Derivation.new(item, derivation) + current = :start + when :transition + # ignore + current = :transition + when :production + # ignore + current = :production + end + else + raise "BUG: Unknown #{current}" + end + + break if current == :start + end + + derivation + end + + # @rbs (StateItem state_item, Grammar::Symbol sym) -> Derivation? + def find_derivation_for_symbol(state_item, sym) + queue = [] #: Array[Array[StateItem]] + queue << [state_item] + + while (sis = queue.shift) + si = sis.last + next_sym = si.item.next_sym + + if next_sym == sym + derivation = nil + + sis.reverse_each do |si| + derivation = Derivation.new(si.item, derivation) + end + + return derivation + end + + if next_sym.nterm? && next_sym.first_set.include?(sym) + @counterexamples.productions[si].each do |next_si| + next if next_si.item.empty_rule? + next if sis.include?(next_si) + queue << (sis + [next_si]) + end + + if next_sym.nullable + next_si = @counterexamples.transitions[[si, next_sym]] + queue << (sis + [next_si]) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/node.rb b/tool/lrama/lib/lrama/counterexamples/node.rb new file mode 100644 index 0000000000..9214a0e7f1 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/node.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + # @rbs generic E < Object -- Type of an element + class Node + attr_reader :elem #: E + attr_reader :next_node #: Node[E]? + + # @rbs [E < Object] (Node[E] node) -> Array[E] + def self.to_a(node) + a = [] # steep:ignore UnannotatedEmptyCollection + + while (node) + a << node.elem + node = node.next_node + end + + a + end + + # @rbs (E elem, Node[E]? next_node) -> void + def initialize(elem, next_node) + @elem = elem + @next_node = next_node + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/path.rb b/tool/lrama/lib/lrama/counterexamples/path.rb new file mode 100644 index 0000000000..6b1325f73b --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/path.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Path + # @rbs! + # @state_item: StateItem + # @parent: Path? + + attr_reader :state_item #: StateItem + attr_reader :parent #: Path? + + # @rbs (StateItem state_item, Path? parent) -> void + def initialize(state_item, parent) + @state_item = state_item + @parent = parent + end + + # @rbs () -> ::String + def to_s + "#<Path>" + end + alias :inspect :to_s + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/state_item.rb b/tool/lrama/lib/lrama/counterexamples/state_item.rb new file mode 100644 index 0000000000..8c2481d793 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/state_item.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class StateItem + attr_reader :id #: Integer + attr_reader :state #: State + attr_reader :item #: State::Item + + # @rbs (Integer id, State state, State::Item item) -> void + def initialize(id, state, item) + @id = id + @state = state + @item = item + end + + # @rbs () -> (:start | :transition | :production) + def type + case + when item.start_item? + :start + when item.beginning_of_rule? + :production + else + :transition + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/counterexamples/triple.rb b/tool/lrama/lib/lrama/counterexamples/triple.rb new file mode 100644 index 0000000000..98fe051f53 --- /dev/null +++ b/tool/lrama/lib/lrama/counterexamples/triple.rb @@ -0,0 +1,41 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Counterexamples + class Triple + attr_reader :precise_lookahead_set #: Bitmap::bitmap + + alias :l :precise_lookahead_set + + # @rbs (StateItem state_item, Bitmap::bitmap precise_lookahead_set) -> void + def initialize(state_item, precise_lookahead_set) + @state_item = state_item + @precise_lookahead_set = precise_lookahead_set + end + + # @rbs () -> State + def state + @state_item.state + end + alias :s :state + + # @rbs () -> State::Item + def item + @state_item.item + end + alias :itm :item + + # @rbs () -> StateItem + def state_item + @state_item + end + + # @rbs () -> ::String + def inspect + "#{state.inspect}. #{item.display_name}. #{l.to_s(2)}" + end + alias :to_s :inspect + end + end +end diff --git a/tool/lrama/lib/lrama/diagram.rb b/tool/lrama/lib/lrama/diagram.rb new file mode 100644 index 0000000000..985808933f --- /dev/null +++ b/tool/lrama/lib/lrama/diagram.rb @@ -0,0 +1,77 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Diagram + class << self + # @rbs (IO out, Grammar grammar, String template_name) -> void + def render(out:, grammar:, template_name: 'diagram/diagram.html') + return unless require_railroad_diagrams + new(out: out, grammar: grammar, template_name: template_name).render + end + + # @rbs () -> bool + def require_railroad_diagrams + require "railroad_diagrams" + true + rescue LoadError + warn "railroad_diagrams is not installed. Please run `bundle install`." + false + end + end + + # @rbs (IO out, Grammar grammar, String template_name) -> void + def initialize(out:, grammar:, template_name: 'diagram/diagram.html') + @grammar = grammar + @out = out + @template_name = template_name + end + + # @rbs () -> void + def render + RailroadDiagrams::TextDiagram.set_formatting(RailroadDiagrams::TextDiagram::PARTS_UNICODE) + @out << ERB.render(template_file, output: self) + end + + # @rbs () -> string + def default_style + RailroadDiagrams::Style::default_style + end + + # @rbs () -> string + def diagrams + result = +'' + @grammar.unique_rule_s_values.each do |s_value| + diagrams = + @grammar.select_rules_by_s_value(s_value).map { |r| r.to_diagrams } + add_diagram( + s_value, + RailroadDiagrams::Diagram.new( + RailroadDiagrams::Choice.new(0, *diagrams), + ), + result + ) + end + result + end + + private + + # @rbs () -> string + def template_dir + File.expand_path('../../template', __dir__) + end + + # @rbs () -> string + def template_file + File.join(template_dir, @template_name) + end + + # @rbs (String name, RailroadDiagrams::Diagram diagram, String result) -> void + def add_diagram(name, diagram, result) + result << "\n<h2 class=\"diagram-header\">#{RailroadDiagrams.escape_html(name)}</h2>" + diagram.write_svg(result.method(:<<)) + result << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/digraph.rb b/tool/lrama/lib/lrama/digraph.rb new file mode 100644 index 0000000000..52865f52dd --- /dev/null +++ b/tool/lrama/lib/lrama/digraph.rb @@ -0,0 +1,104 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Digraph Algorithm of https://dl.acm.org/doi/pdf/10.1145/69622.357187 (P. 625) + # + # Digraph is an algorithm for graph data structure. + # The algorithm efficiently traverses SCC (Strongly Connected Component) of graph + # and merges nodes attributes within the same SCC. + # + # `compute_read_sets` and `compute_follow_sets` have the same structure. + # Graph of gotos and attributes of gotos are given then compute propagated attributes for each node. + # + # In the case of `compute_read_sets`: + # + # * Set of gotos is nodes of graph + # * `reads_relation` is edges of graph + # * `direct_read_sets` is nodes attributes + # + # In the case of `compute_follow_sets`: + # + # * Set of gotos is nodes of graph + # * `includes_relation` is edges of graph + # * `read_sets` is nodes attributes + # + # + # @rbs generic X < Object -- Type of a node + # @rbs generic Y < _Or -- Type of attribute sets assigned to a node which should support merge operation (#| method) + class Digraph + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # interface _Or + # def |: (self) -> self + # end + # @sets: Array[X] + # @relation: Hash[X, Array[X]] + # @base_function: Hash[X, Y] + # @stack: Array[X] + # @h: Hash[X, (Integer|Float)?] + # @result: Hash[X, Y] + + # @rbs sets: Array[X] -- Nodes of graph + # @rbs relation: Hash[X, Array[X]] -- Edges of graph + # @rbs base_function: Hash[X, Y] -- Attributes of nodes + # @rbs return: void + def initialize(sets, relation, base_function) + + # X in the paper + @sets = sets + + # R in the paper + @relation = relation + + # F' in the paper + @base_function = base_function + + # S in the paper + @stack = [] + + # N in the paper + @h = Hash.new(0) + + # F in the paper + @result = {} + end + + # @rbs () -> Hash[X, Y] + def compute + @sets.each do |x| + next if @h[x] != 0 + traverse(x) + end + + return @result + end + + private + + # @rbs (X x) -> void + def traverse(x) + @stack.push(x) + d = @stack.count + @h[x] = d + @result[x] = @base_function[x] # F x = F' x + + @relation[x]&.each do |y| + traverse(y) if @h[y] == 0 + @h[x] = [@h[x], @h[y]].min + @result[x] |= @result[y] # F x = F x + F y + end + + if @h[x] == d + while (z = @stack.pop) do + @h[z] = Float::INFINITY + break if z == x + @result[z] = @result[x] # F (Top of S) = F x + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/erb.rb b/tool/lrama/lib/lrama/erb.rb new file mode 100644 index 0000000000..8f8be54811 --- /dev/null +++ b/tool/lrama/lib/lrama/erb.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "erb" + +module Lrama + class ERB + # @rbs (String file, **untyped kwargs) -> String + def self.render(file, **kwargs) + new(file).render(**kwargs) + end + + # @rbs (String file) -> void + def initialize(file) + input = File.read(file) + if ::ERB.instance_method(:initialize).parameters.last.first == :key + @erb = ::ERB.new(input, trim_mode: '-') + else + @erb = ::ERB.new(input, nil, '-') # steep:ignore UnexpectedPositionalArgument + end + @erb.filename = file + end + + # @rbs (**untyped kwargs) -> String + def render(**kwargs) + @erb.result_with_hash(kwargs) + end + end +end diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb new file mode 100644 index 0000000000..95a80bb01c --- /dev/null +++ b/tool/lrama/lib/lrama/grammar.rb @@ -0,0 +1,603 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "grammar/auxiliary" +require_relative "grammar/binding" +require_relative "grammar/code" +require_relative "grammar/counter" +require_relative "grammar/destructor" +require_relative "grammar/error_token" +require_relative "grammar/inline" +require_relative "grammar/parameterized" +require_relative "grammar/percent_code" +require_relative "grammar/precedence" +require_relative "grammar/printer" +require_relative "grammar/reference" +require_relative "grammar/rule" +require_relative "grammar/rule_builder" +require_relative "grammar/symbol" +require_relative "grammar/symbols" +require_relative "grammar/type" +require_relative "grammar/union" +require_relative "lexer" + +module Lrama + # Grammar is the result of parsing an input grammar file + class Grammar + # @rbs! + # + # interface _DelegatedMethods + # def rules: () -> Array[Rule] + # def accept_symbol: () -> Grammar::Symbol + # def eof_symbol: () -> Grammar::Symbol + # def undef_symbol: () -> Grammar::Symbol + # def precedences: () -> Array[Precedence] + # + # # delegate to @symbols_resolver + # def symbols: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def ielr_defined?: () -> bool + # end + # + # include Symbols::Resolver::_DelegatedMethods + # + # @rule_counter: Counter + # @percent_codes: Array[PercentCode] + # @printers: Array[Printer] + # @destructors: Array[Destructor] + # @error_tokens: Array[ErrorToken] + # @symbols_resolver: Symbols::Resolver + # @types: Array[Type] + # @rule_builders: Array[RuleBuilder] + # @rules: Array[Rule] + # @sym_to_rules: Hash[Integer, Array[Rule]] + # @parameterized_resolver: Parameterized::Resolver + # @empty_symbol: Grammar::Symbol + # @eof_symbol: Grammar::Symbol + # @error_symbol: Grammar::Symbol + # @undef_symbol: Grammar::Symbol + # @accept_symbol: Grammar::Symbol + # @aux: Auxiliary + # @no_stdlib: bool + # @locations: bool + # @define: Hash[String, String] + # @required: bool + # @union: Union + # @precedences: Array[Precedence] + # @start_nterm: Lrama::Lexer::Token::Base? + + extend Forwardable + + attr_reader :percent_codes #: Array[PercentCode] + attr_reader :eof_symbol #: Grammar::Symbol + attr_reader :error_symbol #: Grammar::Symbol + attr_reader :undef_symbol #: Grammar::Symbol + attr_reader :accept_symbol #: Grammar::Symbol + attr_reader :aux #: Auxiliary + attr_reader :parameterized_resolver #: Parameterized::Resolver + attr_reader :precedences #: Array[Precedence] + attr_accessor :union #: Union + attr_accessor :expect #: Integer + attr_accessor :printers #: Array[Printer] + attr_accessor :error_tokens #: Array[ErrorToken] + attr_accessor :lex_param #: String + attr_accessor :parse_param #: String + attr_accessor :initial_action #: Grammar::Code::InitialActionCode + attr_accessor :after_shift #: Lexer::Token::Base + attr_accessor :before_reduce #: Lexer::Token::Base + attr_accessor :after_reduce #: Lexer::Token::Base + attr_accessor :after_shift_error_token #: Lexer::Token::Base + attr_accessor :after_pop_stack #: Lexer::Token::Base + attr_accessor :symbols_resolver #: Symbols::Resolver + attr_accessor :types #: Array[Type] + attr_accessor :rules #: Array[Rule] + attr_accessor :rule_builders #: Array[RuleBuilder] + attr_accessor :sym_to_rules #: Hash[Integer, Array[Rule]] + attr_accessor :no_stdlib #: bool + attr_accessor :locations #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :required #: bool + + def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value, + :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, + :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type, + :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number! + + # @rbs (Counter rule_counter, bool locations, Hash[String, String] define) -> void + def initialize(rule_counter, locations, define = {}) + @rule_counter = rule_counter + + # Code defined by "%code" + @percent_codes = [] + @printers = [] + @destructors = [] + @error_tokens = [] + @symbols_resolver = Grammar::Symbols::Resolver.new + @types = [] + @rule_builders = [] + @rules = [] + @sym_to_rules = {} + @parameterized_resolver = Parameterized::Resolver.new + @empty_symbol = nil + @eof_symbol = nil + @error_symbol = nil + @undef_symbol = nil + @accept_symbol = nil + @aux = Auxiliary.new + @no_stdlib = false + @locations = locations + @define = define + @required = false + @precedences = [] + @start_nterm = nil + + append_special_symbols + end + + # @rbs (Counter rule_counter, Counter midrule_action_counter) -> RuleBuilder + def create_rule_builder(rule_counter, midrule_action_counter) + RuleBuilder.new(rule_counter, midrule_action_counter, @parameterized_resolver) + end + + # @rbs (id: Lexer::Token::Base, code: Lexer::Token::UserCode) -> Array[PercentCode] + def add_percent_code(id:, code:) + @percent_codes << PercentCode.new(id.s_value, code.s_value) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Destructor] + def add_destructor(ident_or_tags:, token_code:, lineno:) + @destructors << Destructor.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[Printer] + def add_printer(ident_or_tags:, token_code:, lineno:) + @printers << Printer.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> Array[ErrorToken] + def add_error_token(ident_or_tags:, token_code:, lineno:) + @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno) + end + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> Array[Type] + def add_type(id:, tag:) + @types << Type.new(id: id, tag: tag) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_nonassoc(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :nonassoc, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_left(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :left, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_right(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :right, precedence: precedence, lineno: lineno)) + end + + # @rbs (Grammar::Symbol sym, Integer precedence, String s_value, Integer lineno) -> Precedence + def add_precedence(sym, precedence, s_value, lineno) + set_precedence(sym, Precedence.new(symbol: sym, s_value: s_value, type: :precedence, precedence: precedence, lineno: lineno)) + end + + # @rbs (Lrama::Lexer::Token::Base id) -> Lrama::Lexer::Token::Base + def set_start_nterm(id) + # When multiple `%start` directives are defined, Bison does not generate an error, + # whereas Lrama does generate an error. + # Related Bison's specification are + # refs: https://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html + if @start_nterm.nil? + @start_nterm = id + else + start = @start_nterm #: Lrama::Lexer::Token::Base + raise "Start non-terminal is already set to #{start.s_value} (line: #{start.first_line}). Cannot set to #{id.s_value} (line: #{id.first_line})." + end + end + + # @rbs (Grammar::Symbol sym, Precedence precedence) -> (Precedence | bot) + def set_precedence(sym, precedence) + @precedences << precedence + sym.precedence = precedence + end + + # @rbs (Grammar::Code::NoReferenceCode code, Integer lineno) -> Union + def set_union(code, lineno) + @union = Union.new(code: code, lineno: lineno) + end + + # @rbs (RuleBuilder builder) -> Array[RuleBuilder] + def add_rule_builder(builder) + @rule_builders << builder + end + + # @rbs (Parameterized::Rule rule) -> Array[Parameterized::Rule] + def add_parameterized_rule(rule) + @parameterized_resolver.add_rule(rule) + end + + # @rbs () -> Array[Parameterized::Rule] + def parameterized_rules + @parameterized_resolver.rules + end + + # @rbs (Array[Parameterized::Rule] rules) -> Array[Parameterized::Rule] + def prepend_parameterized_rules(rules) + @parameterized_resolver.rules = rules + @parameterized_resolver.rules + end + + # @rbs (Integer prologue_first_lineno) -> Integer + def prologue_first_lineno=(prologue_first_lineno) + @aux.prologue_first_lineno = prologue_first_lineno + end + + # @rbs (String prologue) -> String + def prologue=(prologue) + @aux.prologue = prologue + end + + # @rbs (Integer epilogue_first_lineno) -> Integer + def epilogue_first_lineno=(epilogue_first_lineno) + @aux.epilogue_first_lineno = epilogue_first_lineno + end + + # @rbs (String epilogue) -> String + def epilogue=(epilogue) + @aux.epilogue = epilogue + end + + # @rbs () -> void + def prepare + resolve_inline_rules + normalize_rules + collect_symbols + set_lhs_and_rhs + fill_default_precedence + fill_symbols + fill_sym_to_rules + sort_precedence + compute_nullable + compute_first_set + set_locations + end + + # TODO: More validation methods + # + # * Validation for no_declared_type_reference + # + # @rbs () -> void + def validate! + @symbols_resolver.validate! + validate_no_precedence_for_nterm! + validate_rule_lhs_is_nterm! + validate_duplicated_precedence! + end + + # @rbs (Grammar::Symbol sym) -> Array[Rule] + def find_rules_by_symbol!(sym) + find_rules_by_symbol(sym) || (raise "Rules for #{sym} not found") + end + + # @rbs (Grammar::Symbol sym) -> Array[Rule]? + def find_rules_by_symbol(sym) + @sym_to_rules[sym.number] + end + + # @rbs (String s_value) -> Array[Rule] + def select_rules_by_s_value(s_value) + @rules.select {|rule| rule.lhs.id.s_value == s_value } + end + + # @rbs () -> Array[String] + def unique_rule_s_values + @rules.map {|rule| rule.lhs.id.s_value }.uniq + end + + # @rbs () -> bool + def ielr_defined? + @define.key?('lr.type') && @define['lr.type'] == 'ielr' + end + + private + + # @rbs () -> void + def sort_precedence + @precedences.sort_by! do |prec| + prec.symbol.number + end + @precedences.freeze + end + + # @rbs () -> Array[Grammar::Symbol] + def compute_nullable + @rules.each do |rule| + case + when rule.empty_rule? + rule.nullable = true + when rule.rhs.any?(&:term) + rule.nullable = false + else + # noop + end + end + + while true do + rs = @rules.select {|e| e.nullable.nil? } + nts = nterms.select {|e| e.nullable.nil? } + rule_count_1 = rs.count + nterm_count_1 = nts.count + + rs.each do |rule| + if rule.rhs.all?(&:nullable) + rule.nullable = true + end + end + + nts.each do |nterm| + find_rules_by_symbol!(nterm).each do |rule| + if rule.nullable + nterm.nullable = true + end + end + end + + rule_count_2 = @rules.count {|e| e.nullable.nil? } + nterm_count_2 = nterms.count {|e| e.nullable.nil? } + + if (rule_count_1 == rule_count_2) && (nterm_count_1 == nterm_count_2) + break + end + end + + rules.select {|r| r.nullable.nil? }.each do |rule| + rule.nullable = false + end + + nterms.select {|e| e.nullable.nil? }.each do |nterm| + nterm.nullable = false + end + end + + # @rbs () -> Array[Grammar::Symbol] + def compute_first_set + terms.each do |term| + term.first_set = Set.new([term]).freeze + term.first_set_bitmap = Lrama::Bitmap.from_array([term.number]) + end + + nterms.each do |nterm| + nterm.first_set = Set.new([]).freeze + nterm.first_set_bitmap = Lrama::Bitmap.from_array([]) + end + + while true do + changed = false + + @rules.each do |rule| + rule.rhs.each do |r| + if rule.lhs.first_set_bitmap | r.first_set_bitmap != rule.lhs.first_set_bitmap + changed = true + rule.lhs.first_set_bitmap = rule.lhs.first_set_bitmap | r.first_set_bitmap + end + + break unless r.nullable + end + end + + break unless changed + end + + nterms.each do |nterm| + nterm.first_set = Lrama::Bitmap.to_array(nterm.first_set_bitmap).map do |number| + find_symbol_by_number!(number) + end.to_set + end + end + + # @rbs () -> Array[RuleBuilder] + def setup_rules + @rule_builders.each do |builder| + builder.setup_rules + end + end + + # @rbs () -> Grammar::Symbol + def append_special_symbols + # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated + # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) + # term.number = -2 + # @empty_symbol = term + + # YYEOF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYEOF"), alias_name: "\"end of file\"", token_id: 0) + term.number = 0 + term.eof_symbol = true + @eof_symbol = term + + # YYerror + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYerror"), alias_name: "error") + term.number = 1 + term.error_symbol = true + @error_symbol = term + + # YYUNDEF + term = add_term(id: Lrama::Lexer::Token::Ident.new(s_value: "YYUNDEF"), alias_name: "\"invalid token\"") + term.number = 2 + term.undef_symbol = true + @undef_symbol = term + + # $accept + term = add_nterm(id: Lrama::Lexer::Token::Ident.new(s_value: "$accept")) + term.accept_symbol = true + @accept_symbol = term + end + + # @rbs () -> void + def resolve_inline_rules + while @rule_builders.any?(&:has_inline_rules?) do + @rule_builders = @rule_builders.flat_map do |builder| + if builder.has_inline_rules? + Inline::Resolver.new(builder).resolve + else + builder + end + end + end + end + + # @rbs () -> void + def normalize_rules + add_accept_rule + setup_rules + @rule_builders.each do |builder| + builder.rules.each do |rule| + add_nterm(id: rule._lhs, tag: rule.lhs_tag) + @rules << rule + end + end + + nterms.freeze + @rules.sort_by!(&:id).freeze + end + + # Add $accept rule to the top of rules + def add_accept_rule + if @start_nterm + start = @start_nterm #: Lrama::Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [start, @eof_symbol.id], token_code: nil, lineno: start.line) + else + rule_builder = @rule_builders.first #: RuleBuilder + lineno = rule_builder ? rule_builder.line : 0 + lhs = rule_builder.lhs #: Lexer::Token::Base + @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [lhs, @eof_symbol.id], token_code: nil, lineno: lineno) + end + end + + # Collect symbols from rules + # + # @rbs () -> void + def collect_symbols + @rules.flat_map(&:_rhs).each do |s| + case s + when Lrama::Lexer::Token::Char + add_term(id: s) + when Lrama::Lexer::Token::Base + # skip + else + raise "Unknown class: #{s}" + end + end + + terms.freeze + end + + # @rbs () -> void + def set_lhs_and_rhs + @rules.each do |rule| + rule.lhs = token_to_symbol(rule._lhs) if rule._lhs + + rule.rhs = rule._rhs.map do |t| + token_to_symbol(t) + end + end + end + + # Rule inherits precedence from the last term in RHS. + # + # https://www.gnu.org/software/bison/manual/html_node/How-Precedence.html + # + # @rbs () -> void + def fill_default_precedence + @rules.each do |rule| + # Explicitly specified precedence has the highest priority + next if rule.precedence_sym + + precedence_sym = nil + rule.rhs.each do |sym| + precedence_sym = sym if sym.term? + end + + rule.precedence_sym = precedence_sym + end + end + + # @rbs () -> Array[Grammar::Symbol] + def fill_symbols + fill_symbol_number + fill_nterm_type(@types) + fill_printer(@printers) + fill_destructor(@destructors) + fill_error_token(@error_tokens) + sort_by_number! + end + + # @rbs () -> Array[Rule] + def fill_sym_to_rules + @rules.each do |rule| + key = rule.lhs.number + @sym_to_rules[key] ||= [] + @sym_to_rules[key] << rule + end + end + + # @rbs () -> void + def validate_no_precedence_for_nterm! + errors = [] #: Array[String] + + nterms.each do |nterm| + next if nterm.precedence.nil? + + errors << "[BUG] Precedence #{nterm.name} (line: #{nterm.precedence.lineno}) is defined for nonterminal symbol (line: #{nterm.id.first_line}). Precedence can be defined for only terminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void + def validate_rule_lhs_is_nterm! + errors = [] #: Array[String] + + rules.each do |rule| + next if rule.lhs.nterm? + + errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is terminal symbol. It should be nonterminal symbol." + end + + return if errors.empty? + + raise errors.join("\n") + end + + # # @rbs () -> void + def validate_duplicated_precedence! + errors = [] #: Array[String] + seen = {} #: Hash[String, Precedence] + + precedences.each do |prec| + s_value = prec.s_value + if first = seen[s_value] + errors << "%#{prec.type} redeclaration for #{s_value} (line: #{prec.lineno}) previous declaration was %#{first.type} (line: #{first.lineno})" + else + seen[s_value] = prec + end + end + + return if errors.empty? + + raise errors.join("\n") + end + + # @rbs () -> void + def set_locations + @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? } + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/auxiliary.rb b/tool/lrama/lib/lrama/grammar/auxiliary.rb new file mode 100644 index 0000000000..76cfb74d4d --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/auxiliary.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # Grammar file information not used by States but by Output + class Auxiliary + attr_accessor :prologue_first_lineno #: Integer? + attr_accessor :prologue #: String? + attr_accessor :epilogue_first_lineno #: Integer? + attr_accessor :epilogue #: String? + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/binding.rb b/tool/lrama/lib/lrama/grammar/binding.rb new file mode 100644 index 0000000000..5940d153a9 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/binding.rb @@ -0,0 +1,79 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Binding + # @rbs @actual_args: Array[Lexer::Token::Base] + # @rbs @param_to_arg: Hash[String, Lexer::Token::Base] + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> void + def initialize(params, actual_args) + @actual_args = actual_args + @param_to_arg = build_param_to_arg(params, @actual_args) + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def resolve_symbol(sym) + return create_instantiate_rule(sym) if sym.is_a?(Lexer::Token::InstantiateRule) + find_arg_for_param(sym) + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def concatenated_args_str(token) + "#{token.rule_name}_#{format_args(token)}" + end + + private + + # @rbs (Lexer::Token::InstantiateRule sym) -> Lexer::Token::InstantiateRule + def create_instantiate_rule(sym) + Lrama::Lexer::Token::InstantiateRule.new( + s_value: sym.s_value, + location: sym.location, + args: resolve_args(sym.args), + lhs_tag: sym.lhs_tag + ) + end + + # @rbs (Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def resolve_args(args) + args.map { |arg| resolve_symbol(arg) } + end + + # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base + def find_arg_for_param(sym) + if (arg = @param_to_arg[sym.s_value]&.dup) + arg.alias_name = sym.alias_name + arg + else + sym + end + end + + # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> Hash[String, Lexer::Token::Base?] + def build_param_to_arg(params, actual_args) + params.zip(actual_args).map do |param, arg| + [param.s_value, arg] + end.to_h + end + + # @rbs (Lexer::Token::InstantiateRule token) -> String + def format_args(token) + token_to_args_s_values(token).join('_') + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Array[String] + def token_to_args_s_values(token) + token.args.flat_map do |arg| + resolved = resolve_symbol(arg) + if resolved.is_a?(Lexer::Token::InstantiateRule) + [resolved.s_value] + resolved.args.map(&:s_value) + else + [resolved.s_value] + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb new file mode 100644 index 0000000000..f1b860eeba --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -0,0 +1,68 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "code/destructor_code" +require_relative "code/initial_action_code" +require_relative "code/no_reference_code" +require_relative "code/printer_code" +require_relative "code/rule_action" + +module Lrama + class Grammar + class Code + # @rbs! + # + # # delegated + # def s_value: -> String + # def line: -> Integer + # def column: -> Integer + # def references: -> Array[Lrama::Grammar::Reference] + + extend Forwardable + + def_delegators "token_code", :s_value, :line, :column, :references + + attr_reader :type #: ::Symbol + attr_reader :token_code #: Lexer::Token::UserCode + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode) -> void + def initialize(type:, token_code:) + @type = type + @token_code = token_code + end + + # @rbs (Code other) -> bool + def ==(other) + self.class == other.class && + self.type == other.type && + self.token_code == other.token_code + end + + # $$, $n, @$, @n are translated to C code + # + # @rbs () -> String + def translated_code + t_code = s_value.dup + + references.reverse_each do |ref| + first_column = ref.first_column + last_column = ref.last_column + + str = reference_to_c(ref) + + t_code[first_column...last_column] = str + end + + return t_code + end + + private + + # @rbs (Lrama::Grammar::Reference ref) -> bot + def reference_to_c(ref) + raise NotImplementedError.new("#reference_to_c is not implemented") + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/destructor_code.rb b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb new file mode 100644 index 0000000000..d71b62e513 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/destructor_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class DestructorCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb new file mode 100644 index 0000000000..cb36041524 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/initial_action_code.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class InitialActionCode < Code + private + + # * ($$) yylval + # * (@$) yylloc + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + "yylval" + when ref.type == :at && ref.name == "$" # @$ + "yylloc" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in initial_action." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in initial_action." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in initial_action." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in initial_action." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb new file mode 100644 index 0000000000..1d39919979 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/no_reference_code.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class NoReferenceCode < Code + private + + # * ($$) error + # * (@$) error + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> bot + def reference_to_c(ref) + case + when ref.type == :dollar # $$, $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @$, @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:$, $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/printer_code.rb b/tool/lrama/lib/lrama/grammar/code/printer_code.rb new file mode 100644 index 0000000000..c6e25d5235 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/printer_code.rb @@ -0,0 +1,53 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class PrinterCode < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @tag: Lexer::Token::Tag + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, tag: Lexer::Token::Tag) -> void + def initialize(type:, token_code:, tag:) + super(type: type, token_code: token_code) + @tag = tag + end + + private + + # * ($$) *yyvaluep + # * (@$) *yylocationp + # * ($:$) error + # * ($1) error + # * (@1) error + # * ($:1) error + # + # @rbs (Reference ref) -> (String | bot) + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + member = @tag.member + "((*yyvaluep).#{member})" + when ref.type == :at && ref.name == "$" # @$ + "(*yylocationp)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:#{ref.value} can not be used in #{type}." + when ref.type == :dollar # $n + raise "$#{ref.value} can not be used in #{type}." + when ref.type == :at # @n + raise "@#{ref.value} can not be used in #{type}." + when ref.type == :index # $:n + raise "$:#{ref.value} can not be used in #{type}." + else + raise "Unexpected. #{self}, #{ref}" + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/code/rule_action.rb b/tool/lrama/lib/lrama/grammar/code/rule_action.rb new file mode 100644 index 0000000000..e71e93e5a5 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/code/rule_action.rb @@ -0,0 +1,109 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Code + class RuleAction < Code + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @rule: Rule + + # @rbs (type: ::Symbol, token_code: Lexer::Token::UserCode, rule: Rule) -> void + def initialize(type:, token_code:, rule:) + super(type: type, token_code: token_code) + @rule = rule + end + + private + + # * ($$) yyval + # * (@$) yyloc + # * ($:$) error + # * ($1) yyvsp[i] + # * (@1) yylsp[i] + # * ($:1) i - 1 + # + # + # Consider a rule like + # + # class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # + # For the semantic action of original rule: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 $2 $3 $4 $5 + # "Index for yyvsp" -4 -3 -2 -1 0 + # "$:n" $:1 $:2 $:3 $:4 $:5 + # "index of $:n" -5 -4 -3 -2 -1 + # + # + # For the first midrule action: + # + # "Rule" class: keyword_class { $1 } tSTRING { $2 + $3 } keyword_end { $class = $1 + $keyword_end } + # "Position in grammar" $1 + # "Index for yyvsp" 0 + # "$:n" $:1 + # + # @rbs (Reference ref) -> String + def reference_to_c(ref) + case + when ref.type == :dollar && ref.name == "$" # $$ + tag = ref.ex_tag || lhs.tag + raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag + "(yyval.#{tag.member})" + when ref.type == :at && ref.name == "$" # @$ + "(yyloc)" + when ref.type == :index && ref.name == "$" # $:$ + raise "$:$ is not supported" + when ref.type == :dollar # $n + i = -position_in_rhs + ref.index + tag = ref.ex_tag || rhs[ref.index - 1].tag + raise_tag_not_found_error(ref) unless tag + # @type var tag: Lexer::Token::Tag + "(yyvsp[#{i}].#{tag.member})" + when ref.type == :at # @n + i = -position_in_rhs + ref.index + "(yylsp[#{i}])" + when ref.type == :index # $:n + i = -position_in_rhs + ref.index + "(#{i} - 1)" + else + raise "Unexpected. #{self}, #{ref}" + end + end + + # @rbs () -> Integer + def position_in_rhs + # If rule is not derived rule, User Code is only action at + # the end of rule RHS. In such case, the action is located on + # `@rule.rhs.count`. + @rule.position_in_original_rule_rhs || @rule.rhs.count + end + + # If this is midrule action, RHS is an RHS of the original rule. + # + # @rbs () -> Array[Grammar::Symbol] + def rhs + (@rule.original_rule || @rule).rhs + end + + # Unlike `rhs`, LHS is always an LHS of the rule. + # + # @rbs () -> Grammar::Symbol + def lhs + @rule.lhs + end + + # @rbs (Reference ref) -> bot + def raise_tag_not_found_error(ref) + raise "Tag is not specified for '$#{ref.value}' in '#{@rule.display_name}'" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/counter.rb b/tool/lrama/lib/lrama/grammar/counter.rb new file mode 100644 index 0000000000..ced934309d --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/counter.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Counter + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @number: Integer + + # @rbs (Integer number) -> void + def initialize(number) + @number = number + end + + # @rbs () -> Integer + def increment + n = @number + @number += 1 + n + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/destructor.rb b/tool/lrama/lib/lrama/grammar/destructor.rb new file mode 100644 index 0000000000..0ce8611e77 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/destructor.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Destructor + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident|Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::DestructorCode.new(type: :destructor, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/error_token.rb b/tool/lrama/lib/lrama/grammar/error_token.rb new file mode 100644 index 0000000000..9d9ed54ae2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/error_token.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class ErrorToken + attr_reader :ident_or_tags #: Array[Lexer::Token::Ident | Lexer::Token::Tag] + attr_reader :token_code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], token_code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(ident_or_tags:, token_code:, lineno:) + @ident_or_tags = ident_or_tags + @token_code = token_code + @lineno = lineno + end + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::PrinterCode.new(type: :error_token, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/inline.rb b/tool/lrama/lib/lrama/grammar/inline.rb new file mode 100644 index 0000000000..c02ab6002b --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative 'inline/resolver' diff --git a/tool/lrama/lib/lrama/grammar/inline/resolver.rb b/tool/lrama/lib/lrama/grammar/inline/resolver.rb new file mode 100644 index 0000000000..aca689ccfb --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/inline/resolver.rb @@ -0,0 +1,80 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Inline + class Resolver + # @rbs (Lrama::Grammar::RuleBuilder rule_builder) -> void + def initialize(rule_builder) + @rule_builder = rule_builder + end + + # @rbs () -> Array[Lrama::Grammar::RuleBuilder] + def resolve + resolved_builders = [] #: Array[Lrama::Grammar::RuleBuilder] + @rule_builder.rhs.each_with_index do |token, i| + if (rule = @rule_builder.parameterized_resolver.find_inline(token)) + rule.rhs.each do |rhs| + builder = build_rule(rhs, token, i, rule) + resolved_builders << builder + end + break + end + end + resolved_builders + end + + private + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Lrama::Lexer::Token token, Integer index, Lrama::Grammar::Parameterized::Rule rule) -> Lrama::Grammar::RuleBuilder + def build_rule(rhs, token, index, rule) + builder = RuleBuilder.new( + @rule_builder.rule_counter, + @rule_builder.midrule_action_counter, + @rule_builder.parameterized_resolver, + lhs_tag: @rule_builder.lhs_tag + ) + resolve_rhs(builder, rhs, index, token, rule) + builder.lhs = @rule_builder.lhs + builder.line = @rule_builder.line + builder.precedence_sym = @rule_builder.precedence_sym + builder.user_code = replace_user_code(rhs, index) + builder + end + + # @rbs (Lrama::Grammar::RuleBuilder builder, Lrama::Grammar::Parameterized::Rhs rhs, Integer index, Lrama::Lexer::Token token, Lrama::Grammar::Parameterized::Rule rule) -> void + def resolve_rhs(builder, rhs, index, token, rule) + @rule_builder.rhs.each_with_index do |tok, i| + if i == index + rhs.symbols.each do |sym| + if token.is_a?(Lexer::Token::InstantiateRule) + bindings = Binding.new(rule.parameters, token.args) + builder.add_rhs(bindings.resolve_symbol(sym)) + else + builder.add_rhs(sym) + end + end + else + builder.add_rhs(tok) + end + end + end + + # @rbs (Lrama::Grammar::Parameterized::Rhs rhs, Integer index) -> Lrama::Lexer::Token::UserCode + def replace_user_code(rhs, index) + user_code = @rule_builder.user_code + return user_code if rhs.user_code.nil? || user_code.nil? + + code = user_code.s_value.gsub(/\$#{index + 1}/, rhs.user_code.s_value) + user_code.references.each do |ref| + next if ref.index.nil? || ref.index <= index # nil は $$ の場合 + code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (rhs.symbols.count - 1)}") + code = code.gsub(/@#{ref.index}/, "@#{ref.index + (rhs.symbols.count - 1)}") + end + Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized.rb b/tool/lrama/lib/lrama/grammar/parameterized.rb new file mode 100644 index 0000000000..48db3433f3 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative 'parameterized/resolver' +require_relative 'parameterized/rhs' +require_relative 'parameterized/rule' diff --git a/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb new file mode 100644 index 0000000000..558f308190 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/resolver.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Resolver + attr_accessor :rules #: Array[Rule] + attr_accessor :created_lhs_list #: Array[Lexer::Token::Base] + + # @rbs () -> void + def initialize + @rules = [] + @created_lhs_list = [] + end + + # @rbs (Rule rule) -> Array[Rule] + def add_rule(rule) + @rules << rule + end + + # @rbs (Lexer::Token::InstantiateRule token) -> Rule? + def find_rule(token) + select_rules(@rules, token).last + end + + # @rbs (Lexer::Token::Base token) -> Rule? + def find_inline(token) + @rules.reverse.find { |rule| rule.name == token.s_value && rule.inline? } + end + + # @rbs (String lhs_s_value) -> Lexer::Token::Base? + def created_lhs(lhs_s_value) + @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } + end + + # @rbs () -> Array[Rule] + def redefined_rules + @rules.select { |rule| @rules.count { |r| r.name == rule.name && r.required_parameters_count == rule.required_parameters_count } > 1 } + end + + private + + # @rbs (Array[Rule] rules, Lexer::Token::InstantiateRule token) -> Array[Rule] + def select_rules(rules, token) + rules = reject_inline_rules(rules) + rules = select_rules_by_name(rules, token.rule_name) + rules = rules.select { |rule| rule.required_parameters_count == token.args_count } + if rules.empty? + raise "Invalid number of arguments. `#{token.rule_name}`" + else + rules + end + end + + # @rbs (Array[Rule] rules) -> Array[Rule] + def reject_inline_rules(rules) + rules.reject(&:inline?) + end + + # @rbs (Array[Rule] rules, String rule_name) -> Array[Rule] + def select_rules_by_name(rules, rule_name) + rules = rules.select { |rule| rule.name == rule_name } + if rules.empty? + raise "Parameterized rule does not exist. `#{rule_name}`" + else + rules + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb new file mode 100644 index 0000000000..663de49100 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rhs.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rhs + attr_accessor :symbols #: Array[Lexer::Token::Base] + attr_accessor :user_code #: Lexer::Token::UserCode? + attr_accessor :precedence_sym #: Grammar::Symbol? + + # @rbs () -> void + def initialize + @symbols = [] + @user_code = nil + @precedence_sym = nil + end + + # @rbs (Grammar::Binding bindings) -> Lexer::Token::UserCode? + def resolve_user_code(bindings) + return unless user_code + + resolved = Lexer::Token::UserCode.new(s_value: user_code.s_value, location: user_code.location) + var_to_arg = {} #: Hash[String, String] + symbols.each do |sym| + resolved_sym = bindings.resolve_symbol(sym) + if resolved_sym != sym + var_to_arg[sym.s_value] = resolved_sym.s_value + end + end + + var_to_arg.each do |var, arg| + resolved.references.each do |ref| + if ref.name == var + ref.name = arg + end + end + end + + return resolved + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/parameterized/rule.rb b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb new file mode 100644 index 0000000000..7048be3cff --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/parameterized/rule.rb @@ -0,0 +1,36 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Parameterized + class Rule + attr_reader :name #: String + attr_reader :parameters #: Array[Lexer::Token::Base] + attr_reader :rhs #: Array[Rhs] + attr_reader :required_parameters_count #: Integer + attr_reader :tag #: Lexer::Token::Tag? + + # @rbs (String name, Array[Lexer::Token::Base] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void + def initialize(name, parameters, rhs, tag: nil, is_inline: false) + @name = name + @parameters = parameters + @rhs = rhs + @tag = tag + @is_inline = is_inline + @required_parameters_count = parameters.count + end + + # @rbs () -> String + def to_s + "#{@name}(#{@parameters.map(&:s_value).join(', ')})" + end + + # @rbs () -> bool + def inline? + @is_inline + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/percent_code.rb b/tool/lrama/lib/lrama/grammar/percent_code.rb new file mode 100644 index 0000000000..9afb903056 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/percent_code.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class PercentCode + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @name: String + # @code: String + + attr_reader :name #: String + attr_reader :code #: String + + # @rbs (String name, String code) -> void + def initialize(name, code) + @name = name + @code = code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/precedence.rb b/tool/lrama/lib/lrama/grammar/precedence.rb new file mode 100644 index 0000000000..b4c6403372 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/precedence.rb @@ -0,0 +1,55 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Precedence < Struct.new(:type, :symbol, :precedence, :s_value, :lineno, keyword_init: true) + include Comparable + # @rbs! + # type type_enum = :left | :right | :nonassoc | :precedence + # + # attr_accessor type: type_enum + # attr_accessor symbol: Grammar::Symbol + # attr_accessor precedence: Integer + # attr_accessor s_value: String + # attr_accessor lineno: Integer + # + # def initialize: (?type: type_enum, ?symbol: Grammar::Symbol, ?precedence: Integer, ?s_value: ::String, ?lineno: Integer) -> void + + attr_reader :used_by_lalr #: Array[State::ResolvedConflict] + attr_reader :used_by_ielr #: Array[State::ResolvedConflict] + + # @rbs (Precedence other) -> Integer + def <=>(other) + self.precedence <=> other.precedence + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_lalr(resolved_conflict) + @used_by_lalr ||= [] #: Array[State::ResolvedConflict] + @used_by_lalr << resolved_conflict + end + + # @rbs (State::ResolvedConflict resolved_conflict) -> void + def mark_used_by_ielr(resolved_conflict) + @used_by_ielr ||= [] #: Array[State::ResolvedConflict] + @used_by_ielr << resolved_conflict + end + + # @rbs () -> bool + def used_by? + used_by_lalr? || used_by_ielr? + end + + # @rbs () -> bool + def used_by_lalr? + !@used_by_lalr.nil? && !@used_by_lalr.empty? + end + + # @rbs () -> bool + def used_by_ielr? + !@used_by_ielr.nil? && !@used_by_ielr.empty? + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/printer.rb b/tool/lrama/lib/lrama/grammar/printer.rb new file mode 100644 index 0000000000..490fe701db --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/printer.rb @@ -0,0 +1,20 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Printer < Struct.new(:ident_or_tags, :token_code, :lineno, keyword_init: true) + # @rbs! + # attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag] + # attr_accessor token_code: Lexer::Token::UserCode + # attr_accessor lineno: Integer + # + # def initialize: (?ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag], ?token_code: Lexer::Token::UserCode, ?lineno: Integer) -> void + + # @rbs (Lexer::Token::Tag tag) -> String + def translated_code(tag) + Code::PrinterCode.new(type: :printer, token_code: token_code, tag: tag).translated_code + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/reference.rb b/tool/lrama/lib/lrama/grammar/reference.rb new file mode 100644 index 0000000000..7e3badfecc --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/reference.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # type: :dollar or :at + # name: String (e.g. $$, $foo, $expr.right) + # number: Integer (e.g. $1) + # index: Integer + # ex_tag: "$<tag>1" (Optional) + class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_column, :last_column, keyword_init: true) + # @rbs! + # attr_accessor type: ::Symbol + # attr_accessor name: String + # attr_accessor number: Integer + # attr_accessor index: Integer + # attr_accessor ex_tag: Lexer::Token::Base? + # attr_accessor first_column: Integer + # attr_accessor last_column: Integer + # + # def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token::Base?, first_column: Integer, last_column: Integer) -> void + + # @rbs () -> (String|Integer) + def value + name || number + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule.rb b/tool/lrama/lib/lrama/grammar/rule.rb new file mode 100644 index 0000000000..d00d6a8883 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule.rb @@ -0,0 +1,135 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # _rhs holds original RHS element. Use rhs to refer to Symbol. + class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true) + # @rbs! + # + # interface _DelegatedMethods + # def lhs: -> Grammar::Symbol + # def rhs: -> Array[Grammar::Symbol] + # end + # + # attr_accessor id: Integer + # attr_accessor _lhs: Lexer::Token::Base + # attr_accessor lhs: Grammar::Symbol + # attr_accessor lhs_tag: Lexer::Token::Tag? + # attr_accessor _rhs: Array[Lexer::Token::Base] + # attr_accessor rhs: Array[Grammar::Symbol] + # attr_accessor token_code: Lexer::Token::UserCode? + # attr_accessor position_in_original_rule_rhs: Integer + # attr_accessor nullable: bool + # attr_accessor precedence_sym: Grammar::Symbol? + # attr_accessor lineno: Integer? + # + # def initialize: ( + # ?id: Integer, ?_lhs: Lexer::Token::Base?, ?lhs: Lexer::Token::Base, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token::Base], ?rhs: Array[Grammar::Symbol], + # ?token_code: Lexer::Token::UserCode?, ?position_in_original_rule_rhs: Integer?, ?nullable: bool, + # ?precedence_sym: Grammar::Symbol?, ?lineno: Integer? + # ) -> void + + attr_accessor :original_rule #: Rule + + # @rbs (Rule other) -> bool + def ==(other) + self.class == other.class && + self.lhs == other.lhs && + self.lhs_tag == other.lhs_tag && + self.rhs == other.rhs && + self.token_code == other.token_code && + self.position_in_original_rule_rhs == other.position_in_original_rule_rhs && + self.nullable == other.nullable && + self.precedence_sym == other.precedence_sym && + self.lineno == other.lineno + end + + # @rbs () -> String + def display_name + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map {|r| r.id.s_value }.join(" ") + "#{l} -> #{r}" + end + + # @rbs () -> String + def display_name_without_action + l = lhs.id.s_value + r = empty_rule? ? "ε" : rhs.map do |r| + r.id.s_value if r.first_set.any? + end.compact.join(" ") + + "#{l} -> #{r}" + end + + # @rbs () -> (RailroadDiagrams::Skip | RailroadDiagrams::Sequence) + def to_diagrams + if rhs.empty? + RailroadDiagrams::Skip.new + else + RailroadDiagrams::Sequence.new(*rhs_to_diagram) + end + end + + # Used by #user_actions + # + # @rbs () -> String + def as_comment + l = lhs.id.s_value + r = empty_rule? ? "%empty" : rhs.map(&:display_name).join(" ") + + "#{l}: #{r}" + end + + # @rbs () -> String + def with_actions + "#{display_name} {#{token_code&.s_value}}" + end + + # opt_nl: ε <-- empty_rule + # | '\n' <-- not empty_rule + # + # @rbs () -> bool + def empty_rule? + rhs.empty? + end + + # @rbs () -> Precedence? + def precedence + precedence_sym&.precedence + end + + # @rbs () -> bool + def initial_rule? + id == 0 + end + + # @rbs () -> String? + def translated_code + return nil unless token_code + + Code::RuleAction.new(type: :rule_action, token_code: token_code, rule: self).translated_code + end + + # @rbs () -> bool + def contains_at_reference? + return false unless token_code + + token_code.references.any? {|r| r.type == :at } + end + + private + + # @rbs () -> Array[(RailroadDiagrams::Terminal | RailroadDiagrams::NonTerminal)] + def rhs_to_diagram + rhs.map do |r| + if r.term + RailroadDiagrams::Terminal.new(r.id.s_value) + else + RailroadDiagrams::NonTerminal.new(r.id.s_value) + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb new file mode 100644 index 0000000000..34fdca6c86 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -0,0 +1,270 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class RuleBuilder + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @position_in_original_rule_rhs: Integer? + # @skip_preprocess_references: bool + # @rules: Array[Rule] + # @rule_builders_for_parameterized: Array[RuleBuilder] + # @rule_builders_for_derived_rules: Array[RuleBuilder] + # @parameterized_rules: Array[Rule] + # @midrule_action_rules: Array[Rule] + # @replaced_rhs: Array[Lexer::Token::Base]? + + attr_accessor :lhs #: Lexer::Token::Base? + attr_accessor :line #: Integer? + attr_reader :rule_counter #: Counter + attr_reader :midrule_action_counter #: Counter + attr_reader :parameterized_resolver #: Grammar::Parameterized::Resolver + attr_reader :lhs_tag #: Lexer::Token::Tag? + attr_reader :rhs #: Array[Lexer::Token::Base] + attr_reader :user_code #: Lexer::Token::UserCode? + attr_reader :precedence_sym #: Grammar::Symbol? + + # @rbs (Counter rule_counter, Counter midrule_action_counter, Grammar::Parameterized::Resolver parameterized_resolver, ?Integer position_in_original_rule_rhs, ?lhs_tag: Lexer::Token::Tag?, ?skip_preprocess_references: bool) -> void + def initialize(rule_counter, midrule_action_counter, parameterized_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false) + @rule_counter = rule_counter + @midrule_action_counter = midrule_action_counter + @parameterized_resolver = parameterized_resolver + @position_in_original_rule_rhs = position_in_original_rule_rhs + @skip_preprocess_references = skip_preprocess_references + + @lhs = nil + @lhs_tag = lhs_tag + @rhs = [] + @user_code = nil + @precedence_sym = nil + @line = nil + @rules = [] + @rule_builders_for_parameterized = [] + @rule_builders_for_derived_rules = [] + @parameterized_rules = [] + @midrule_action_rules = [] + end + + # @rbs (Lexer::Token::Base rhs) -> void + def add_rhs(rhs) + @line ||= rhs.line + + flush_user_code + + @rhs << rhs + end + + # @rbs (Lexer::Token::UserCode? user_code) -> void + def user_code=(user_code) + @line ||= user_code&.line + + flush_user_code + + @user_code = user_code + end + + # @rbs (Grammar::Symbol? precedence_sym) -> void + def precedence_sym=(precedence_sym) + flush_user_code + + @precedence_sym = precedence_sym + end + + # @rbs () -> void + def complete_input + freeze_rhs + end + + # @rbs () -> void + def setup_rules + preprocess_references unless @skip_preprocess_references + process_rhs + resolve_inline_rules + build_rules + end + + # @rbs () -> Array[Grammar::Rule] + def rules + @parameterized_rules + @midrule_action_rules + @rules + end + + # @rbs () -> bool + def has_inline_rules? + rhs.any? { |token| @parameterized_resolver.find_inline(token) } + end + + private + + # @rbs () -> void + def freeze_rhs + @rhs.freeze + end + + # @rbs () -> void + def preprocess_references + numberize_references + end + + # @rbs () -> void + def build_rules + tokens = @replaced_rhs #: Array[Lexer::Token::Base] + return if tokens.any? { |t| @parameterized_resolver.find_inline(t) } + + rule = Rule.new( + id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code, + position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line + ) + @rules = [rule] + @parameterized_rules = @rule_builders_for_parameterized.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder| + rule_builder.rules + end.flatten + @midrule_action_rules.each do |r| + r.original_rule = rule + end + end + + # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on. + # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`. + # + # @rbs () -> void + def process_rhs + return if @replaced_rhs + + replaced_rhs = [] #: Array[Lexer::Token::Base] + + rhs.each_with_index do |token, i| + case token + when Lrama::Lexer::Token::Char + replaced_rhs << token + when Lrama::Lexer::Token::Ident + replaced_rhs << token + when Lrama::Lexer::Token::InstantiateRule + parameterized_rule = @parameterized_resolver.find_rule(token) + raise "Unexpected token. #{token}" unless parameterized_rule + + bindings = Binding.new(parameterized_rule.parameters, token.args) + lhs_s_value = bindings.concatenated_args_str(token) + if (created_lhs = @parameterized_resolver.created_lhs(lhs_s_value)) + replaced_rhs << created_lhs + else + lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) + replaced_rhs << lhs_token + @parameterized_resolver.created_lhs_list << lhs_token + parameterized_rule.rhs.each do |r| + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, lhs_tag: token.lhs_tag || parameterized_rule.tag) + rule_builder.lhs = lhs_token + r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) } + rule_builder.line = line + rule_builder.precedence_sym = r.precedence_sym + rule_builder.user_code = r.resolve_user_code(bindings) + rule_builder.complete_input + rule_builder.setup_rules + @rule_builders_for_parameterized << rule_builder + end + end + when Lrama::Lexer::Token::UserCode + prefix = token.referred ? "@" : "$@" + tag = token.tag || lhs_tag + new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) + replaced_rhs << new_token + + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, i, lhs_tag: tag, skip_preprocess_references: true) + rule_builder.lhs = new_token + rule_builder.user_code = token + rule_builder.complete_input + rule_builder.setup_rules + + @rule_builders_for_derived_rules << rule_builder + when Lrama::Lexer::Token::Empty + # Noop + else + raise "Unexpected token. #{token}" + end + end + + @replaced_rhs = replaced_rhs + end + + # @rbs () -> void + def resolve_inline_rules + while @rule_builders_for_parameterized.any?(&:has_inline_rules?) do + @rule_builders_for_parameterized = @rule_builders_for_parameterized.flat_map do |rule_builder| + if rule_builder.has_inline_rules? + inlined_builders = Inline::Resolver.new(rule_builder).resolve + inlined_builders.each { |builder| builder.setup_rules } + inlined_builders + else + rule_builder + end + end + end + end + + # @rbs () -> void + def numberize_references + # Bison n'th component is 1-origin + (rhs + [user_code]).compact.each.with_index(1) do |token, i| + next unless token.is_a?(Lrama::Lexer::Token::UserCode) + + token.references.each do |ref| + ref_name = ref.name + + if ref_name + if ref_name == '$' + ref.name = '$' + else + candidates = ([lhs] + rhs).each_with_index.select do |token, _i| + # @type var token: Lexer::Token::Base + token.referred_by?(ref_name) + end + + if candidates.size >= 2 + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is duplicated.") + end + + unless (referring_symbol = candidates.first) + token.invalid_ref(ref, "Referring symbol `#{ref_name}` is not found.") + end + + if referring_symbol[1] == 0 # Refers to LHS + ref.name = '$' + else + ref.number = referring_symbol[1] + end + end + end + + if ref.number + ref.index = ref.number + end + + # TODO: Need to check index of @ too? + next if ref.type == :at + + if ref.index + # TODO: Prohibit $0 even so Bison allows it? + # See: https://www.gnu.org/software/bison/manual/html_node/Actions.html + token.invalid_ref(ref, "Can not refer following component. #{ref.index} >= #{i}.") if ref.index >= i + rhs[ref.index - 1].referred = true + end + end + end + end + + # @rbs () -> void + def flush_user_code + if (c = @user_code) + @rhs << c + @user_code = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/stdlib.y b/tool/lrama/lib/lrama/grammar/stdlib.y new file mode 100644 index 0000000000..dd397c9e08 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/stdlib.y @@ -0,0 +1,142 @@ +/********************************************************************** + + stdlib.y + + This is lrama's standard library. It provides a number of + parameterized rule definitions, such as options and lists, + that should be useful in a number of situations. + +**********************************************************************/ + +%% + +// ------------------------------------------------------------------- +// Options + +/* + * program: option(X) + * + * => + * + * program: option_X + * option_X: %empty + * option_X: X + */ +%rule option(X) + : /* empty */ + | X + ; + + +/* + * program: ioption(X) + * + * => + * + * program: %empty + * program: X + */ +%rule %inline ioption(X) + : /* empty */ + | X + ; + +// ------------------------------------------------------------------- +// Sequences + +/* + * program: preceded(opening, X) + * + * => + * + * program: preceded_opening_X + * preceded_opening_X: opening X + */ +%rule preceded(opening, X) + : opening X { $$ = $2; } + ; + +/* + * program: terminated(X, closing) + * + * => + * + * program: terminated_X_closing + * terminated_X_closing: X closing + */ +%rule terminated(X, closing) + : X closing { $$ = $1; } + ; + +/* + * program: delimited(opening, X, closing) + * + * => + * + * program: delimited_opening_X_closing + * delimited_opening_X_closing: opening X closing + */ +%rule delimited(opening, X, closing) + : opening X closing { $$ = $2; } + ; + +// ------------------------------------------------------------------- +// Lists + +/* + * program: list(X) + * + * => + * + * program: list_X + * list_X: %empty + * list_X: list_X X + */ +%rule list(X) + : /* empty */ + | list(X) X + ; + +/* + * program: nonempty_list(X) + * + * => + * + * program: nonempty_list_X + * nonempty_list_X: X + * nonempty_list_X: nonempty_list_X X + */ +%rule nonempty_list(X) + : X + | nonempty_list(X) X + ; + +/* + * program: separated_nonempty_list(separator, X) + * + * => + * + * program: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separated_nonempty_list_separator_X separator X + */ +%rule separated_nonempty_list(separator, X) + : X + | separated_nonempty_list(separator, X) separator X + ; + +/* + * program: separated_list(separator, X) + * + * => + * + * program: separated_list_separator_X + * separated_list_separator_X: option_separated_nonempty_list_separator_X + * option_separated_nonempty_list_separator_X: %empty + * option_separated_nonempty_list_separator_X: separated_nonempty_list_separator_X + * separated_nonempty_list_separator_X: X + * separated_nonempty_list_separator_X: separator separated_nonempty_list_separator_X X + */ +%rule separated_list(separator, X) + : option(separated_nonempty_list(separator, X)) + ; diff --git a/tool/lrama/lib/lrama/grammar/symbol.rb b/tool/lrama/lib/lrama/grammar/symbol.rb new file mode 100644 index 0000000000..07aee0c0a2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbol.rb @@ -0,0 +1,149 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# Symbol is both of nterm and term +# `number` is both for nterm and term +# `token_id` is tokentype for term, internal sequence number for nterm +# + +module Lrama + class Grammar + class Symbol + attr_accessor :id #: Lexer::Token::Base + attr_accessor :alias_name #: String? + attr_reader :number #: Integer + attr_accessor :number_bitmap #: Bitmap::bitmap + attr_accessor :tag #: Lexer::Token::Tag? + attr_accessor :token_id #: Integer + attr_accessor :nullable #: bool + attr_accessor :precedence #: Precedence? + attr_accessor :printer #: Printer? + attr_accessor :destructor #: Destructor? + attr_accessor :error_token #: ErrorToken + attr_accessor :first_set #: Set[Grammar::Symbol] + attr_accessor :first_set_bitmap #: Bitmap::bitmap + attr_reader :term #: bool + attr_writer :eof_symbol #: bool + attr_writer :error_symbol #: bool + attr_writer :undef_symbol #: bool + attr_writer :accept_symbol #: bool + + # @rbs (id: Lexer::Token::Base, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token::Tag?, + # ?token_id: Integer?, ?nullable: bool?, ?precedence: Precedence?, ?printer: Printer?) -> void + def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil) + @id = id + @alias_name = alias_name + @number = number + @tag = tag + @term = term + @token_id = token_id + @nullable = nullable + @precedence = precedence + @printer = printer + @destructor = destructor + end + + # @rbs (Integer) -> void + def number=(number) + @number = number + @number_bitmap = Bitmap::from_integer(number) + end + + # @rbs () -> bool + def term? + term + end + + # @rbs () -> bool + def nterm? + !term + end + + # @rbs () -> bool + def eof_symbol? + !!@eof_symbol + end + + # @rbs () -> bool + def error_symbol? + !!@error_symbol + end + + # @rbs () -> bool + def undef_symbol? + !!@undef_symbol + end + + # @rbs () -> bool + def accept_symbol? + !!@accept_symbol + end + + # @rbs () -> bool + def midrule? + return false if term? + + name.include?("$") || name.include?("@") + end + + # @rbs () -> String + def name + id.s_value + end + + # @rbs () -> String + def display_name + alias_name || name + end + + # name for yysymbol_kind_t + # + # See: b4_symbol_kind_base + # @type var name: String + # @rbs () -> String + def enum_name + case + when accept_symbol? + res = "YYACCEPT" + when eof_symbol? + res = "YYEOF" + when term? && id.is_a?(Lrama::Lexer::Token::Char) + res = number.to_s + display_name + when term? && id.is_a?(Lrama::Lexer::Token::Ident) + res = name + when midrule? + res = number.to_s + name + when nterm? + res = name + else + raise "Unexpected #{self}" + end + + "YYSYMBOL_" + res.gsub(/\W+/, "_") + end + + # comment for yysymbol_kind_t + # + # @rbs () -> String? + def comment + case + when accept_symbol? + # YYSYMBOL_YYACCEPT + name + when eof_symbol? + # YYEOF + alias_name + when (term? && 0 < token_id && token_id < 128) + # YYSYMBOL_3_backslash_, YYSYMBOL_14_ + display_name + when midrule? + # YYSYMBOL_21_1 + name + else + # YYSYMBOL_keyword_class, YYSYMBOL_strings_1 + display_name + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/symbols.rb b/tool/lrama/lib/lrama/grammar/symbols.rb new file mode 100644 index 0000000000..337241d1b2 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative "symbols/resolver" diff --git a/tool/lrama/lib/lrama/grammar/symbols/resolver.rb b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb new file mode 100644 index 0000000000..085a835d28 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/symbols/resolver.rb @@ -0,0 +1,362 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Symbols + class Resolver + # @rbs! + # + # interface _DelegatedMethods + # def symbols: () -> Array[Grammar::Symbol] + # def nterms: () -> Array[Grammar::Symbol] + # def terms: () -> Array[Grammar::Symbol] + # def add_nterm: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + # def add_term: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + # def find_symbol_by_number!: (Integer number) -> Grammar::Symbol + # def find_symbol_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol + # def token_to_symbol: (Lexer::Token::Base token) -> Grammar::Symbol + # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol + # def fill_nterm_type: (Array[Grammar::Type] types) -> void + # def fill_symbol_number: () -> void + # def fill_printer: (Array[Grammar::Printer] printers) -> void + # def fill_destructor: (Array[Destructor] destructors) -> (Destructor | bot) + # def fill_error_token: (Array[Grammar::ErrorToken] error_tokens) -> void + # def sort_by_number!: () -> Array[Grammar::Symbol] + # end + # + # @symbols: Array[Grammar::Symbol]? + # @number: Integer + # @used_numbers: Hash[Integer, bool] + + attr_reader :terms #: Array[Grammar::Symbol] + attr_reader :nterms #: Array[Grammar::Symbol] + + # @rbs () -> void + def initialize + @terms = [] + @nterms = [] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols + @symbols ||= (@terms + @nterms) + end + + # @rbs () -> Array[Grammar::Symbol] + def sort_by_number! + symbols.sort_by!(&:number) + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol + def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) + if token_id && (sym = find_symbol_by_token_id(token_id)) + if replace + sym.id = id + sym.alias_name = alias_name + sym.tag = tag + end + + return sym + end + + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + term = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: true, token_id: token_id, nullable: false + ) + @terms << term + term + end + + # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol + def add_nterm(id:, alias_name: nil, tag: nil) + if (sym = find_symbol_by_id(id)) + return sym + end + + @symbols = nil + nterm = Symbol.new( + id: id, alias_name: alias_name, number: nil, tag: tag, + term: false, token_id: nil, nullable: nil, + ) + @nterms << nterm + nterm + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_term_by_s_value(s_value) + terms.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol? + def find_symbol_by_s_value(s_value) + symbols.find { |s| s.id.s_value == s_value } + end + + # @rbs (::String s_value) -> Grammar::Symbol + def find_symbol_by_s_value!(s_value) + find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`") + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol? + def find_symbol_by_id(id) + symbols.find do |s| + s.id == id || s.alias_name == id.s_value + end + end + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_symbol_by_id!(id) + find_symbol_by_id(id) || (raise "Symbol not found. #{id}") + end + + # @rbs (Integer token_id) -> Grammar::Symbol? + def find_symbol_by_token_id(token_id) + symbols.find {|s| s.token_id == token_id } + end + + # @rbs (Integer number) -> Grammar::Symbol + def find_symbol_by_number!(number) + sym = symbols[number] + + raise "Symbol not found. number: `#{number}`" unless sym + raise "[BUG] Symbol number mismatch. #{number}, #{sym}" if sym.number != number + + sym + end + + # @rbs () -> void + def fill_symbol_number + # YYEMPTY = -2 + # YYEOF = 0 + # YYerror = 1 + # YYUNDEF = 2 + @number = 3 + fill_terms_number + fill_nterms_number + end + + # @rbs (Array[Grammar::Type] types) -> void + def fill_nterm_type(types) + types.each do |type| + nterm = find_nterm_by_id!(type.id) + nterm.tag = type.tag + end + end + + # @rbs (Array[Grammar::Printer] printers) -> void + def fill_printer(printers) + symbols.each do |sym| + printers.each do |printer| + printer.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.printer = printer if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.printer = printer if sym.tag == ident_or_tag + else + raise "Unknown token type. #{printer}" + end + end + end + end + end + + # @rbs (Array[Destructor] destructors) -> (Array[Grammar::Symbol] | bot) + def fill_destructor(destructors) + symbols.each do |sym| + destructors.each do |destructor| + destructor.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.destructor = destructor if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.destructor = destructor if sym.tag == ident_or_tag + else + raise "Unknown token type. #{destructor}" + end + end + end + end + end + + # @rbs (Array[Grammar::ErrorToken] error_tokens) -> void + def fill_error_token(error_tokens) + symbols.each do |sym| + error_tokens.each do |token| + token.ident_or_tags.each do |ident_or_tag| + case ident_or_tag + when Lrama::Lexer::Token::Ident + sym.error_token = token if sym.id == ident_or_tag + when Lrama::Lexer::Token::Tag + sym.error_token = token if sym.tag == ident_or_tag + else + raise "Unknown token type. #{token}" + end + end + end + end + end + + # @rbs (Lexer::Token::Base token) -> Grammar::Symbol + def token_to_symbol(token) + case token + when Lrama::Lexer::Token::Base + find_symbol_by_id!(token) + else + raise "Unknown class: #{token}" + end + end + + # @rbs () -> void + def validate! + validate_number_uniqueness! + validate_alias_name_uniqueness! + validate_symbols! + end + + private + + # @rbs (Lexer::Token::Base id) -> Grammar::Symbol + def find_nterm_by_id!(id) + @nterms.find do |s| + s.id == id + end || (raise "Symbol not found. #{id}") + end + + # @rbs () -> void + def fill_terms_number + # Character literal in grammar file has + # token id corresponding to ASCII code by default, + # so start token_id from 256. + token_id = 256 + + @terms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + # If id is Token::Char, it uses ASCII code + if sym.token_id.nil? + if sym.id.is_a?(Lrama::Lexer::Token::Char) + # Ignore ' on the both sides + case sym.id.s_value[1..-2] + when "\\b" + sym.token_id = 8 + when "\\f" + sym.token_id = 12 + when "\\n" + sym.token_id = 10 + when "\\r" + sym.token_id = 13 + when "\\t" + sym.token_id = 9 + when "\\v" + sym.token_id = 11 + when "\"" + sym.token_id = 34 + when "'" + sym.token_id = 39 + when "\\\\" + sym.token_id = 92 + when /\A\\(\d+)\z/ + unless (id = Integer($1, 8)).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + when /\A(.)\z/ + unless (id = $1&.bytes&.first).nil? + sym.token_id = id + else + raise "Unknown Char s_value #{sym}" + end + else + raise "Unknown Char s_value #{sym}" + end + else + sym.token_id = token_id + token_id += 1 + end + end + end + end + + # @rbs () -> void + def fill_nterms_number + token_id = 0 + + @nterms.each do |sym| + while used_numbers[@number] do + @number += 1 + end + + if sym.number.nil? + sym.number = @number + used_numbers[@number] = true + @number += 1 + end + + if sym.token_id.nil? + sym.token_id = token_id + token_id += 1 + end + end + end + + # @rbs () -> Hash[Integer, bool] + def used_numbers + return @used_numbers if defined?(@used_numbers) + + @used_numbers = {} + symbols.map(&:number).each do |n| + @used_numbers[n] = true + end + @used_numbers + end + + # @rbs () -> void + def validate_number_uniqueness! + invalid = symbols.group_by(&:number).select do |number, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol number is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_alias_name_uniqueness! + invalid = symbols.select(&:alias_name).group_by(&:alias_name).select do |alias_name, syms| + syms.count > 1 + end + + return if invalid.empty? + + raise "Symbol alias name is duplicated. #{invalid}" + end + + # @rbs () -> void + def validate_symbols! + symbols.each { |sym| sym.id.validate } + errors = symbols.map { |sym| sym.id.errors }.flatten.compact + return if errors.empty? + + raise errors.join("\n") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/type.rb b/tool/lrama/lib/lrama/grammar/type.rb new file mode 100644 index 0000000000..c631769447 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/type.rb @@ -0,0 +1,32 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Type + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @id: Lexer::Token::Base + # @tag: Lexer::Token::Tag + + attr_reader :id #: Lexer::Token::Base + attr_reader :tag #: Lexer::Token::Tag + + # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> void + def initialize(id:, tag:) + @id = id + @tag = tag + end + + # @rbs (Grammar::Type other) -> bool + def ==(other) + self.class == other.class && + self.id == other.id && + self.tag == other.tag + end + end + end +end diff --git a/tool/lrama/lib/lrama/grammar/union.rb b/tool/lrama/lib/lrama/grammar/union.rb new file mode 100644 index 0000000000..774cc66fc6 --- /dev/null +++ b/tool/lrama/lib/lrama/grammar/union.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + class Union + attr_reader :code #: Grammar::Code::NoReferenceCode + attr_reader :lineno #: Integer + + # @rbs (code: Grammar::Code::NoReferenceCode, lineno: Integer) -> void + def initialize(code:, lineno:) + @code = code + @lineno = lineno + end + + # @rbs () -> String + def braces_less_code + # Braces is already removed by lexer + code.s_value + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb new file mode 100644 index 0000000000..ce98b505a7 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer.rb @@ -0,0 +1,219 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "strscan" + +require_relative "lexer/grammar_file" +require_relative "lexer/location" +require_relative "lexer/token" + +module Lrama + class Lexer + # @rbs! + # + # type token = lexer_token | c_token + # + # type lexer_token = [String, Token::Token] | + # [::Symbol, Token::Tag] | + # [::Symbol, Token::Char] | + # [::Symbol, Token::Str] | + # [::Symbol, Token::Int] | + # [::Symbol, Token::Ident] + # + # type c_token = [:C_DECLARATION, Token::UserCode] + + attr_reader :head_line #: Integer + attr_reader :head_column #: Integer + attr_reader :line #: Integer + attr_accessor :status #: :initial | :c_declaration + attr_accessor :end_symbol #: String? + + SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze #: Array[String] + PERCENT_TOKENS = %w( + %union + %token + %type + %nterm + %left + %right + %nonassoc + %expect + %define + %require + %printer + %destructor + %lex-param + %parse-param + %initial-action + %precedence + %prec + %error-token + %before-reduce + %after-reduce + %after-shift-error-token + %after-shift + %after-pop-stack + %empty + %code + %rule + %no-stdlib + %inline + %locations + %categories + %start + ).freeze #: Array[String] + + # @rbs (GrammarFile grammar_file) -> void + def initialize(grammar_file) + @grammar_file = grammar_file + @scanner = StringScanner.new(grammar_file.text) + @head_column = @head = @scanner.pos + @head_line = @line = 1 + @status = :initial + @end_symbol = nil + end + + # @rbs () -> token? + def next_token + case @status + when :initial + lex_token + when :c_declaration + lex_c_code + end + end + + # @rbs () -> Integer + def column + @scanner.pos - @head + end + + # @rbs () -> Location + def location + Location.new( + grammar_file: @grammar_file, + first_line: @head_line, first_column: @head_column, + last_line: line, last_column: column + ) + end + + # @rbs () -> lexer_token? + def lex_token + until @scanner.eos? do + case + when @scanner.scan(/\n/) + newline + when @scanner.scan(/\s+/) + @scanner.matched.count("\n").times { newline } + when @scanner.scan(/\/\*/) + lex_comment + when @scanner.scan(/\/\/.*(?<newline>\n)?/) + newline if @scanner[:newline] + else + break + end + end + + reset_first_position + + case + when @scanner.eos? + return + when @scanner.scan(/#{SYMBOLS.join('|')}/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/[\?\+\*]/) + return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/<\w+>/) + return [:TAG, Lrama::Lexer::Token::Tag.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'.'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) + return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] + when @scanner.scan(/".*?"/) + return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] + when @scanner.scan(/\d+/) + return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)] + when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) + token = Lrama::Lexer::Token::Ident.new(s_value: @scanner.matched, location: location) + type = + if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/) + :IDENT_COLON + else + :IDENTIFIER + end + return [type, token] + else + raise ParseError, location.generate_error_message("Unexpected token") # steep:ignore UnknownConstant + end + end + + # @rbs () -> c_token + def lex_c_code + nested = 0 + code = +'' + reset_first_position + + until @scanner.eos? do + case + when @scanner.scan(/{/) + code << @scanner.matched + nested += 1 + when @scanner.scan(/}/) + if nested == 0 && @end_symbol == '}' + @scanner.unscan + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + else + code << @scanner.matched + nested -= 1 + end + when @scanner.check(/#{@end_symbol}/) + return [:C_DECLARATION, Lrama::Lexer::Token::UserCode.new(s_value: code, location: location)] + when @scanner.scan(/\n/) + code << @scanner.matched + newline + when @scanner.scan(/".*?"/) + code << %Q(#{@scanner.matched}) + @line += @scanner.matched.count("\n") + when @scanner.scan(/'.*?'/) + code << %Q(#{@scanner.matched}) + when @scanner.scan(/[^\"'\{\}\n]+/) + code << @scanner.matched + when @scanner.scan(/#{Regexp.escape(@end_symbol)}/) # steep:ignore + code << @scanner.matched + else + code << @scanner.getch + end + end + raise ParseError, location.generate_error_message("Unexpected code: #{code}") # steep:ignore UnknownConstant + end + + private + + # @rbs () -> void + def lex_comment + until @scanner.eos? do + case + when @scanner.scan_until(/[\s\S]*?\*\//) + @scanner.matched.count("\n").times { newline } + return + when @scanner.scan_until(/\n/) + newline + end + end + end + + # @rbs () -> void + def reset_first_position + @head_line = line + @head_column = column + end + + # @rbs () -> void + def newline + @line += 1 + @head = @scanner.pos + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/grammar_file.rb b/tool/lrama/lib/lrama/lexer/grammar_file.rb new file mode 100644 index 0000000000..37e82ff18d --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/grammar_file.rb @@ -0,0 +1,40 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + class GrammarFile + class Text < String + # @rbs () -> String + def inspect + length <= 50 ? super : "#{self[0..47]}...".inspect + end + end + + attr_reader :path #: String + attr_reader :text #: String + + # @rbs (String path, String text) -> void + def initialize(path, text) + @path = path + @text = Text.new(text).freeze + end + + # @rbs () -> String + def inspect + "<#{self.class}: @path=#{path}, @text=#{text.inspect}>" + end + + # @rbs (GrammarFile other) -> bool + def ==(other) + self.class == other.class && + self.path == other.path + end + + # @rbs () -> Array[String] + def lines + @lines ||= text.split("\n") + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/location.rb b/tool/lrama/lib/lrama/lexer/location.rb new file mode 100644 index 0000000000..4465576d53 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/location.rb @@ -0,0 +1,132 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + class Location + attr_reader :grammar_file #: GrammarFile + attr_reader :first_line #: Integer + attr_reader :first_column #: Integer + attr_reader :last_line #: Integer + attr_reader :last_column #: Integer + + # @rbs (grammar_file: GrammarFile, first_line: Integer, first_column: Integer, last_line: Integer, last_column: Integer) -> void + def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:) + @grammar_file = grammar_file + @first_line = first_line + @first_column = first_column + @last_line = last_line + @last_column = last_column + end + + # @rbs (Location other) -> bool + def ==(other) + self.class == other.class && + self.grammar_file == other.grammar_file && + self.first_line == other.first_line && + self.first_column == other.first_column && + self.last_line == other.last_line && + self.last_column == other.last_column + end + + # @rbs (Integer left, Integer right) -> Location + def partial_location(left, right) + offset = -first_column + new_first_line = -1 + new_first_column = -1 + new_last_line = -1 + new_last_column = -1 + + _text.each.with_index do |line, index| + new_offset = offset + line.length + 1 + + if offset <= left && left <= new_offset + new_first_line = first_line + index + new_first_column = left - offset + end + + if offset <= right && right <= new_offset + new_last_line = first_line + index + new_last_column = right - offset + end + + offset = new_offset + end + + Location.new( + grammar_file: grammar_file, + first_line: new_first_line, first_column: new_first_column, + last_line: new_last_line, last_column: new_last_column + ) + end + + # @rbs () -> String + def to_s + "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})" + end + + # @rbs (String error_message) -> String + def generate_error_message(error_message) + <<~ERROR.chomp + #{path}:#{first_line}:#{first_column}: #{error_message} + #{error_with_carets} + ERROR + end + + # @rbs () -> String + def error_with_carets + <<~TEXT + #{formatted_first_lineno} | #{text} + #{line_number_padding} | #{carets_line} + TEXT + end + + private + + # @rbs () -> String + def path + grammar_file.path + end + + # @rbs () -> String + def carets_line + leading_whitespace + highlight_marker + end + + # @rbs () -> String + def leading_whitespace + (text[0...first_column] or raise "Invalid first_column: #{first_column}") + .gsub(/[^\t]/, ' ') + end + + # @rbs () -> String + def highlight_marker + length = last_column - first_column + '^' + '~' * [0, length - 1].max + end + + # @rbs () -> String + def formatted_first_lineno + first_line.to_s.rjust(4) + end + + # @rbs () -> String + def line_number_padding + ' ' * formatted_first_lineno.length + end + + # @rbs () -> String + def text + @text ||= _text.join("\n") + end + + # @rbs () -> Array[String] + def _text + @_text ||=begin + range = (first_line - 1)...last_line + grammar_file.lines[range] or raise "#{range} is invalid" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb new file mode 100644 index 0000000000..37f77aa069 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -0,0 +1,20 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'token/base' +require_relative 'token/char' +require_relative 'token/empty' +require_relative 'token/ident' +require_relative 'token/instantiate_rule' +require_relative 'token/int' +require_relative 'token/str' +require_relative 'token/tag' +require_relative 'token/token' +require_relative 'token/user_code' + +module Lrama + class Lexer + module Token + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/base.rb b/tool/lrama/lib/lrama/lexer/token/base.rb new file mode 100644 index 0000000000..3df93bbc73 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/base.rb @@ -0,0 +1,73 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Base + attr_reader :s_value #: String + attr_reader :location #: Location + attr_accessor :alias_name #: String + attr_accessor :referred #: bool + attr_reader :errors #: Array[String] + + # @rbs (s_value: String, ?alias_name: String, ?location: Location) -> void + def initialize(s_value:, alias_name: nil, location: nil) + s_value.freeze + @s_value = s_value + @alias_name = alias_name + @location = location + @errors = [] + end + + # @rbs () -> String + def to_s + "value: `#{s_value}`, location: #{location}" + end + + # @rbs (String string) -> bool + def referred_by?(string) + [self.s_value, self.alias_name].compact.include?(string) + end + + # @rbs (Lexer::Token::Base other) -> bool + def ==(other) + self.class == other.class && self.s_value == other.s_value + end + + # @rbs () -> Integer + def first_line + location.first_line + end + alias :line :first_line + + # @rbs () -> Integer + def first_column + location.first_column + end + alias :column :first_column + + # @rbs () -> Integer + def last_line + location.last_line + end + + # @rbs () -> Integer + def last_column + location.last_column + end + + # @rbs (Lrama::Grammar::Reference ref, String message) -> bot + def invalid_ref(ref, message) + location = self.location.partial_location(ref.first_column, ref.last_column) + raise location.generate_error_message(message) + end + + # @rbs () -> bool + def validate + true + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/char.rb b/tool/lrama/lib/lrama/lexer/token/char.rb new file mode 100644 index 0000000000..f4ef7c9fbc --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/char.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Char < Base + # @rbs () -> void + def validate + validate_ascii_code_range + end + + private + + # @rbs () -> void + def validate_ascii_code_range + unless s_value.ascii_only? + errors << "Invalid character: `#{s_value}`. Only ASCII characters are allowed." + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/empty.rb b/tool/lrama/lib/lrama/lexer/token/empty.rb new file mode 100644 index 0000000000..375e256493 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/empty.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Empty < Base + def initialize(location: nil) + super(s_value: '%empty', location: location) + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/ident.rb b/tool/lrama/lib/lrama/lexer/token/ident.rb new file mode 100644 index 0000000000..4880be9073 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/ident.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Ident < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb new file mode 100644 index 0000000000..7051ba75a4 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/instantiate_rule.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class InstantiateRule < Base + attr_reader :args #: Array[Lexer::Token::Base] + attr_reader :lhs_tag #: Lexer::Token::Tag? + + # @rbs (s_value: String, ?alias_name: String, ?location: Location, ?args: Array[Lexer::Token::Base], ?lhs_tag: Lexer::Token::Tag?) -> void + def initialize(s_value:, alias_name: nil, location: nil, args: [], lhs_tag: nil) + super s_value: s_value, alias_name: alias_name, location: location + @args = args + @lhs_tag = lhs_tag + end + + # @rbs () -> String + def rule_name + s_value + end + + # @rbs () -> Integer + def args_count + args.count + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/int.rb b/tool/lrama/lib/lrama/lexer/token/int.rb new file mode 100644 index 0000000000..7daf48d4d3 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/int.rb @@ -0,0 +1,14 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Int < Base + # @rbs! + # def initialize: (s_value: Integer, ?alias_name: String, ?location: Location) -> void + # def s_value: () -> Integer + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/str.rb b/tool/lrama/lib/lrama/lexer/token/str.rb new file mode 100644 index 0000000000..cf9de6cf0f --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/str.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Str < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/tag.rb b/tool/lrama/lib/lrama/lexer/token/tag.rb new file mode 100644 index 0000000000..68c6268219 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/tag.rb @@ -0,0 +1,16 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Tag < Base + # @rbs () -> String + def member + # Omit "<>" + s_value[1..-2] or raise "Unexpected Tag format (#{s_value})" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/token.rb b/tool/lrama/lib/lrama/lexer/token/token.rb new file mode 100644 index 0000000000..935797efc6 --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/token.rb @@ -0,0 +1,11 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + class Token < Base + end + end + end +end diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb new file mode 100644 index 0000000000..166f04954a --- /dev/null +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -0,0 +1,109 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "strscan" + +module Lrama + class Lexer + module Token + class UserCode < Base + attr_accessor :tag #: Lexer::Token::Tag + + # @rbs () -> Array[Lrama::Grammar::Reference] + def references + @references ||= _references + end + + private + + # @rbs () -> Array[Lrama::Grammar::Reference] + def _references + scanner = StringScanner.new(s_value) + references = [] #: Array[Grammar::Reference] + + until scanner.eos? do + case + when reference = scan_reference(scanner) + references << reference + when scanner.scan(/\/\*/) + scanner.scan_until(/\*\//) + else + scanner.getch + end + end + + references + end + + # @rbs (StringScanner scanner) -> Lrama::Grammar::Reference? + def scan_reference(scanner) + start = scanner.pos + if scanner.scan(/ + # $ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + \$(<[a-zA-Z0-9_]+>)?(?: + (\$) # $$, $<long>$ + | (\d+) # $1, $2, $<long>1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $foo, $expr, $<long>program (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + ) + | + # @ references + # It need to wrap an identifier with brackets to use ".-" for identifiers + @(?: + (\$) # @$ + | (\d+) # @1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # @foo, @expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # @[expr.right], @[expr-right] (named reference with brackets) + ) + | + # $: references + \$: + (?: + (\$) # $:$ + | (\d+) # $:1 + | ([a-zA-Z_][a-zA-Z0-9_]*) # $:foo, $:expr (named reference without brackets) + | \[([a-zA-Z_.][-a-zA-Z0-9_.]*)\] # $:[expr.right], $:[expr-right] (named reference with brackets) + ) + /x) + case + # $ references + when scanner[2] # $$, $<long>$ + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: "$", ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[3] # $1, $2, $<long>1 + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, number: Integer(scanner[3]), index: Integer(scanner[3]), ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[4] # $foo, $expr, $<long>program (named reference without brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[4], ex_tag: tag, first_column: start, last_column: scanner.pos) + when scanner[5] # $[expr.right], $[expr-right], $<long>[expr.right] (named reference with brackets) + tag = scanner[1] ? Lrama::Lexer::Token::Tag.new(s_value: scanner[1]) : nil + return Lrama::Grammar::Reference.new(type: :dollar, name: scanner[5], ex_tag: tag, first_column: start, last_column: scanner.pos) + + # @ references + when scanner[6] # @$ + return Lrama::Grammar::Reference.new(type: :at, name: "$", first_column: start, last_column: scanner.pos) + when scanner[7] # @1 + return Lrama::Grammar::Reference.new(type: :at, number: Integer(scanner[7]), index: Integer(scanner[7]), first_column: start, last_column: scanner.pos) + when scanner[8] # @foo, @expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[8], first_column: start, last_column: scanner.pos) + when scanner[9] # @[expr.right], @[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :at, name: scanner[9], first_column: start, last_column: scanner.pos) + + # $: references + when scanner[10] # $:$ + return Lrama::Grammar::Reference.new(type: :index, name: "$", first_column: start, last_column: scanner.pos) + when scanner[11] # $:1 + return Lrama::Grammar::Reference.new(type: :index, number: Integer(scanner[11]), index: Integer(scanner[11]), first_column: start, last_column: scanner.pos) + when scanner[12] # $:foo, $:expr (named reference without brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[12], first_column: start, last_column: scanner.pos) + when scanner[13] # $:[expr.right], $:[expr-right] (named reference with brackets) + return Lrama::Grammar::Reference.new(type: :index, name: scanner[13], first_column: start, last_column: scanner.pos) + end + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/logger.rb b/tool/lrama/lib/lrama/logger.rb new file mode 100644 index 0000000000..291eea5296 --- /dev/null +++ b/tool/lrama/lib/lrama/logger.rb @@ -0,0 +1,31 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Logger + # @rbs (IO out) -> void + def initialize(out = STDERR) + @out = out + end + + # @rbs () -> void + def line_break + @out << "\n" + end + + # @rbs (String message) -> void + def trace(message) + @out << message << "\n" + end + + # @rbs (String message) -> void + def warn(message) + @out << 'warning: ' << message << "\n" + end + + # @rbs (String message) -> void + def error(message) + @out << 'error: ' << message << "\n" + end + end +end diff --git a/tool/lrama/lib/lrama/option_parser.rb b/tool/lrama/lib/lrama/option_parser.rb new file mode 100644 index 0000000000..5a15d59c7b --- /dev/null +++ b/tool/lrama/lib/lrama/option_parser.rb @@ -0,0 +1,223 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require 'optparse' + +module Lrama + # Handle option parsing for the command line interface. + class OptionParser + # @rbs! + # @options: Lrama::Options + # @trace: Array[String] + # @report: Array[String] + # @profile: Array[String] + + # @rbs (Array[String]) -> Lrama::Options + def self.parse(argv) + new.parse(argv) + end + + # @rbs () -> void + def initialize + @options = Options.new + @trace = [] + @report = [] + @profile = [] + end + + # @rbs (Array[String]) -> Lrama::Options + def parse(argv) + parse_by_option_parser(argv) + + @options.trace_opts = validate_trace(@trace) + @options.report_opts = validate_report(@report) + @options.profile_opts = validate_profile(@profile) + @options.grammar_file = argv.shift + + unless @options.grammar_file + abort "File should be specified\n" + end + + if @options.grammar_file == '-' + @options.grammar_file = argv.shift or abort "File name for STDIN should be specified\n" + else + @options.y = File.open(@options.grammar_file, 'r') + end + + if !@report.empty? && @options.report_file.nil? && @options.grammar_file + @options.report_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".output" + end + + if !@options.header_file && @options.header + case + when @options.outfile + @options.header_file = File.dirname(@options.outfile) + "/" + File.basename(@options.outfile, ".*") + ".h" + when @options.grammar_file + @options.header_file = File.dirname(@options.grammar_file) + "/" + File.basename(@options.grammar_file, ".*") + ".h" + end + end + + @options + end + + private + + # @rbs (Array[String]) -> void + def parse_by_option_parser(argv) + ::OptionParser.new do |o| + o.banner = <<~BANNER + Lrama is LALR (1) parser generator written by Ruby. + + Usage: lrama [options] FILE + BANNER + o.separator '' + o.separator 'STDIN mode:' + o.separator 'lrama [options] - FILE read grammar from STDIN' + o.separator '' + o.separator 'Tuning the Parser:' + o.on('-S', '--skeleton=FILE', 'specify the skeleton to use') {|v| @options.skeleton = v } + o.on('-t', '--debug', 'display debugging outputs of internal parser') {|v| @options.debug = true } + o.separator " same as '-Dparse.trace'" + o.on('--locations', 'enable location support') {|v| @options.locations = true } + o.on('-D', '--define=NAME[=VALUE]', Array, "similar to '%define NAME VALUE'") do |v| + @options.define = v.each_with_object({}) do |item, hash| # steep:ignore UnannotatedEmptyCollection + key, value = item.split('=', 2) + hash[key] = value + end + end + o.separator '' + o.separator 'Output:' + o.on('-H', '--header=[FILE]', 'also produce a header file named FILE') {|v| @options.header = true; @options.header_file = v } + o.on('-d', 'also produce a header file') { @options.header = true } + o.on('-r', '--report=REPORTS', Array, 'also produce details on the automaton') {|v| @report = v } + o.on_tail '' + o.on_tail 'REPORTS is a list of comma-separated words that can include:' + o.on_tail ' states describe the states' + o.on_tail ' itemsets complete the core item sets with their closure' + o.on_tail ' lookaheads explicitly associate lookahead tokens to items' + o.on_tail ' solved describe shift/reduce conflicts solving' + o.on_tail ' counterexamples, cex generate conflict counterexamples' + o.on_tail ' rules list unused rules' + o.on_tail ' terms list unused terminals' + o.on_tail ' verbose report detailed internal state and analysis results' + o.on_tail ' all include all the above reports' + o.on_tail ' none disable all reports' + o.on('--report-file=FILE', 'also produce details on the automaton output to a file named FILE') {|v| @options.report_file = v } + o.on('-o', '--output=FILE', 'leave output to FILE') {|v| @options.outfile = v } + o.on('--trace=TRACES', Array, 'also output trace logs at runtime') {|v| @trace = v } + o.on_tail '' + o.on_tail 'TRACES is a list of comma-separated words that can include:' + o.on_tail ' automaton display states' + o.on_tail ' closure display states' + o.on_tail ' rules display grammar rules' + o.on_tail ' only-explicit-rules display only explicit grammar rules' + o.on_tail ' actions display grammar rules with actions' + o.on_tail ' time display generation time' + o.on_tail ' all include all the above traces' + o.on_tail ' none disable all traces' + o.on('--diagram=[FILE]', 'generate a diagram of the rules') do |v| + @options.diagram = true + @options.diagram_file = v if v + end + o.on('--profile=PROFILES', Array, 'profiles parser generation parts') {|v| @profile = v } + o.on_tail '' + o.on_tail 'PROFILES is a list of comma-separated words that can include:' + o.on_tail ' call-stack use sampling call-stack profiler (stackprof gem)' + o.on_tail ' memory use memory profiler (memory_profiler gem)' + o.on('-v', '--verbose', "same as '--report=state'") {|_v| @report << 'states' } + o.separator '' + o.separator 'Diagnostics:' + o.on('-W', '--warnings', 'report the warnings') {|v| @options.warnings = true } + o.separator '' + o.separator 'Error Recovery:' + o.on('-e', 'enable error recovery') {|v| @options.error_recovery = true } + o.separator '' + o.separator 'Other options:' + o.on('-V', '--version', "output version information and exit") {|v| puts "lrama #{Lrama::VERSION}"; exit 0 } + o.on('-h', '--help', "display this help and exit") {|v| puts o; exit 0 } + o.on_tail + o.parse!(argv) + end + end + + ALIASED_REPORTS = { cex: :counterexamples }.freeze #: Hash[Symbol, Symbol] + VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze #: Array[Symbol] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_report(report) + h = { grammar: true } + return h if report.empty? + return {} if report == ['none'] + if report == ['all'] + VALID_REPORTS.each { |r| h[r] = true } + return h + end + + report.each do |r| + aliased = aliased_report_option(r) + if VALID_REPORTS.include?(aliased) + h[aliased] = true + else + raise "Invalid report option \"#{r}\"." + end + end + + return h + end + + # @rbs (String) -> Symbol + def aliased_report_option(opt) + (ALIASED_REPORTS[opt.to_sym] || opt).to_sym + end + + VALID_TRACES = %w[ + locations scan parse automaton bitsets closure + grammar rules only-explicit-rules actions resource + sets muscles tools m4-early m4 skeleton time ielr cex + ].freeze #: Array[String] + NOT_SUPPORTED_TRACES = %w[ + locations scan parse bitsets grammar resource + sets muscles tools m4-early m4 skeleton ielr cex + ].freeze #: Array[String] + SUPPORTED_TRACES = VALID_TRACES - NOT_SUPPORTED_TRACES #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_trace(trace) + h = {} #: Hash[Symbol, bool] + return h if trace.empty? || trace == ['none'] + all_traces = SUPPORTED_TRACES - %w[only-explicit-rules] + if trace == ['all'] + all_traces.each { |t| h[t.gsub(/-/, '_').to_sym] = true } + return h + end + + trace.each do |t| + if SUPPORTED_TRACES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid trace option \"#{t}\".\nValid options are [#{SUPPORTED_TRACES.join(", ")}]." + end + end + + return h + end + + VALID_PROFILES = %w[call-stack memory].freeze #: Array[String] + + # @rbs (Array[String]) -> Hash[Symbol, bool] + def validate_profile(profile) + h = {} #: Hash[Symbol, bool] + return h if profile.empty? + + profile.each do |t| + if VALID_PROFILES.include?(t) + h[t.gsub(/-/, '_').to_sym] = true + else + raise "Invalid profile option \"#{t}\".\nValid options are [#{VALID_PROFILES.join(", ")}]." + end + end + + return h + end + end +end diff --git a/tool/lrama/lib/lrama/options.rb b/tool/lrama/lib/lrama/options.rb new file mode 100644 index 0000000000..87aec62448 --- /dev/null +++ b/tool/lrama/lib/lrama/options.rb @@ -0,0 +1,46 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Command line options. + class Options + attr_accessor :skeleton #: String + attr_accessor :locations #: bool + attr_accessor :header #: bool + attr_accessor :header_file #: String? + attr_accessor :report_file #: String? + attr_accessor :outfile #: String + attr_accessor :error_recovery #: bool + attr_accessor :grammar_file #: String + attr_accessor :trace_opts #: Hash[Symbol, bool]? + attr_accessor :report_opts #: Hash[Symbol, bool]? + attr_accessor :warnings #: bool + attr_accessor :y #: IO + attr_accessor :debug #: bool + attr_accessor :define #: Hash[String, String] + attr_accessor :diagram #: bool + attr_accessor :diagram_file #: String + attr_accessor :profile_opts #: Hash[Symbol, bool]? + + # @rbs () -> void + def initialize + @skeleton = "bison/yacc.c" + @locations = false + @define = {} + @header = false + @header_file = nil + @report_file = nil + @outfile = "y.tab.c" + @error_recovery = false + @grammar_file = '' + @trace_opts = nil + @report_opts = nil + @warnings = false + @y = STDIN + @debug = false + @diagram = false + @diagram_file = "diagram.html" + @profile_opts = nil + end + end +end diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb new file mode 100644 index 0000000000..d527be8bd4 --- /dev/null +++ b/tool/lrama/lib/lrama/output.rb @@ -0,0 +1,452 @@ +# frozen_string_literal: true + +require "forwardable" +require_relative "tracer/duration" + +module Lrama + class Output + extend Forwardable + include Tracer::Duration + + attr_reader :grammar_file_path, :context, :grammar, :error_recovery, :include_header + + def_delegators "@context", :yyfinal, :yylast, :yyntokens, :yynnts, :yynrules, :yynstates, + :yymaxutok, :yypact_ninf, :yytable_ninf + + def_delegators "@grammar", :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol + + def initialize( + out:, output_file_path:, template_name:, grammar_file_path:, + context:, grammar:, header_out: nil, header_file_path: nil, error_recovery: false + ) + @out = out + @output_file_path = output_file_path + @template_name = template_name + @grammar_file_path = grammar_file_path + @header_out = header_out + @header_file_path = header_file_path + @context = context + @grammar = grammar + @error_recovery = error_recovery + @include_header = header_file_path ? header_file_path.sub("./", "") : nil + end + + if ERB.instance_method(:initialize).parameters.last.first == :key + def self.erb(input) + ERB.new(input, trim_mode: '-') + end + else + def self.erb(input) + ERB.new(input, nil, '-') + end + end + + def render_partial(file) + ERB.render(partial_file(file), context: @context, output: self) + end + + def render + report_duration(:render) do + tmp = eval_template(template_file, @output_file_path) + @out << tmp + + if @header_file_path + tmp = eval_template(header_template_file, @header_file_path) + + if @header_out + @header_out << tmp + else + File.write(@header_file_path, tmp) + end + end + end + end + + # A part of b4_token_enums + def token_enums + @context.yytokentype.map do |s_value, token_id, display_name| + s = sprintf("%s = %d%s", s_value, token_id, token_id == yymaxutok ? "" : ",") + + if display_name + sprintf(" %-30s /* %s */\n", s, display_name) + else + sprintf(" %s\n", s) + end + end.join + end + + # b4_symbol_enum + def symbol_enum + last_sym_number = @context.yysymbol_kind_t.last[1] + @context.yysymbol_kind_t.map do |s_value, sym_number, display_name| + s = sprintf("%s = %d%s", s_value, sym_number, (sym_number == last_sym_number) ? "" : ",") + + if display_name + sprintf(" %-40s /* %s */\n", s, display_name) + else + sprintf(" %s\n", s) + end + end.join + end + + def yytranslate + int_array_to_string(@context.yytranslate) + end + + def yytranslate_inverted + int_array_to_string(@context.yytranslate_inverted) + end + + def yyrline + int_array_to_string(@context.yyrline) + end + + def yytname + string_array_to_string(@context.yytname) + " YY_NULLPTR" + end + + # b4_int_type_for + def int_type_for(ary) + min = ary.min + max = ary.max + + case + when (-127 <= min && min <= 127) && (-127 <= max && max <= 127) + "yytype_int8" + when (0 <= min && min <= 255) && (0 <= max && max <= 255) + "yytype_uint8" + when (-32767 <= min && min <= 32767) && (-32767 <= max && max <= 32767) + "yytype_int16" + when (0 <= min && min <= 65535) && (0 <= max && max <= 65535) + "yytype_uint16" + else + "int" + end + end + + def symbol_actions_for_printer + @grammar.symbols.map do |sym| + next unless sym.printer + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.printer.lineno} "#{@grammar_file_path}" + {#{sym.printer.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + def symbol_actions_for_destructor + @grammar.symbols.map do |sym| + next unless sym.destructor + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.destructor.lineno} "#{@grammar_file_path}" + {#{sym.destructor.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + # b4_user_initial_action + def user_initial_action(comment = "") + return "" unless @grammar.initial_action + + <<-STR + #{comment} +#line #{@grammar.initial_action.line} "#{@grammar_file_path}" + {#{@grammar.initial_action.translated_code}} + STR + end + + def after_shift_function(comment = "") + return "" unless @grammar.after_shift + + <<-STR + #{comment} +#line #{@grammar.after_shift.line} "#{@grammar_file_path}" + {#{@grammar.after_shift.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def before_reduce_function(comment = "") + return "" unless @grammar.before_reduce + + <<-STR + #{comment} +#line #{@grammar.before_reduce.line} "#{@grammar_file_path}" + {#{@grammar.before_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_reduce_function(comment = "") + return "" unless @grammar.after_reduce + + <<-STR + #{comment} +#line #{@grammar.after_reduce.line} "#{@grammar_file_path}" + {#{@grammar.after_reduce.s_value}(yylen#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def after_shift_error_token_function(comment = "") + return "" unless @grammar.after_shift_error_token + + <<-STR + #{comment} +#line #{@grammar.after_shift_error_token.line} "#{@grammar_file_path}" + {#{@grammar.after_shift_error_token.s_value}(#{parse_param_name});} +#line [@oline@] [@ofile@] + STR + end + + def after_pop_stack_function(len, comment = "") + return "" unless @grammar.after_pop_stack + + <<-STR + #{comment} +#line #{@grammar.after_pop_stack.line} "#{@grammar_file_path}" + {#{@grammar.after_pop_stack.s_value}(#{len}#{user_args});} +#line [@oline@] [@ofile@] + STR + end + + def symbol_actions_for_error_token + @grammar.symbols.map do |sym| + next unless sym.error_token + + <<-STR + case #{sym.enum_name}: /* #{sym.comment} */ +#line #{sym.error_token.lineno} "#{@grammar_file_path}" + {#{sym.error_token.translated_code(sym.tag)}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + end + + # b4_user_actions + def user_actions + action = @context.states.rules.map do |rule| + next unless rule.token_code + + code = rule.token_code + spaces = " " * (code.column - 1) + + <<-STR + case #{rule.id + 1}: /* #{rule.as_comment} */ +#line #{code.line} "#{@grammar_file_path}" +#{spaces}{#{rule.translated_code}} +#line [@oline@] [@ofile@] + break; + + STR + end.join + + action + <<-STR + +#line [@oline@] [@ofile@] + STR + end + + def omit_blanks(param) + param.strip + end + + # b4_parse_param + def parse_param + if @grammar.parse_param + omit_blanks(@grammar.parse_param) + else + "" + end + end + + def lex_param + if @grammar.lex_param + omit_blanks(@grammar.lex_param) + else + "" + end + end + + # b4_user_formals + def user_formals + if @grammar.parse_param + ", #{parse_param}" + else + "" + end + end + + # b4_user_args + def user_args + if @grammar.parse_param + ", #{parse_param_name}" + else + "" + end + end + + def extract_param_name(param) + param[/\b([a-zA-Z0-9_]+)(?=\s*\z)/] + end + + def parse_param_name + if @grammar.parse_param + extract_param_name(parse_param) + else + "" + end + end + + def lex_param_name + if @grammar.lex_param + extract_param_name(lex_param) + else + "" + end + end + + # b4_parse_param_use + def parse_param_use(val, loc) + str = <<-STR.dup + YY_USE (#{val}); + YY_USE (#{loc}); + STR + + if @grammar.parse_param + str << " YY_USE (#{parse_param_name});" + end + + str + end + + # b4_yylex_formals + def yylex_formals + ary = ["&yylval"] + ary << "&yylloc" if @grammar.locations + + if @grammar.lex_param + ary << lex_param_name + end + + "(#{ary.join(', ')})" + end + + # b4_table_value_equals + def table_value_equals(table, value, literal, symbol) + if literal < table.min || table.max < literal + "0" + else + "((#{value}) == #{symbol})" + end + end + + # b4_yyerror_args + def yyerror_args + ary = ["&yylloc"] + + if @grammar.parse_param + ary << parse_param_name + end + + "#{ary.join(', ')}" + end + + def template_basename + File.basename(template_file) + end + + def aux + @grammar.aux + end + + def int_array_to_string(ary) + last = ary.count - 1 + + ary.each_with_index.each_slice(10).map do |slice| + " " + slice.map { |e, i| sprintf("%6d%s", e, (i == last) ? "" : ",") }.join + end.join("\n") + end + + def spec_mapped_header_file + @header_file_path + end + + def b4_cpp_guard__b4_spec_mapped_header_file + if @header_file_path + "YY_YY_" + @header_file_path.gsub(/[^a-zA-Z_0-9]+/, "_").upcase + "_INCLUDED" + else + "" + end + end + + # b4_percent_code_get + def percent_code(name) + @grammar.percent_codes.select do |percent_code| + percent_code.name == name + end.map do |percent_code| + percent_code.code + end.join + end + + private + + def eval_template(file, path) + tmp = ERB.render(file, context: @context, output: self) + replace_special_variables(tmp, path) + end + + def template_file + File.join(template_dir, @template_name) + end + + def header_template_file + File.join(template_dir, "bison/yacc.h") + end + + def partial_file(file) + File.join(template_dir, file) + end + + def template_dir + File.expand_path('../../template', __dir__) + end + + def string_array_to_string(ary) + result = "" + tmp = " " + + ary.each do |s| + replaced = s.gsub('\\', '\\\\\\\\').gsub('"', '\\"') + if (tmp + replaced + " \"\",").length > 75 + result = "#{result}#{tmp}\n" + tmp = " \"#{replaced}\"," + else + tmp = "#{tmp} \"#{replaced}\"," + end + end + + result + tmp + end + + def replace_special_variables(str, ofile) + str.each_line.with_index(1).map do |line, i| + line.gsub!("[@oline@]", (i + 1).to_s) + line.gsub!("[@ofile@]", "\"#{ofile}\"") + line + end.join + end + end +end diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb new file mode 100644 index 0000000000..20c3ad347f --- /dev/null +++ b/tool/lrama/lib/lrama/parser.rb @@ -0,0 +1,2275 @@ +# frozen_string_literal: true +# +# DO NOT MODIFY!!!! +# This file is automatically generated by Racc 1.8.1 +# from Racc grammar file "parser.y". +# + +###### racc/parser.rb begin +unless $".find {|p| p.end_with?('/racc/parser.rb')} +$".push "#{__dir__}/racc/parser.rb" +self.class.module_eval(<<'...end racc/parser.rb/module_eval...', 'racc/parser.rb', 1) +#-- +# Copyright (c) 1999-2006 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the same terms of ruby. +# +# As a special exception, when this code is copied by Racc +# into a Racc output file, you may use that output file +# without restriction. +#++ + +unless $".find {|p| p.end_with?('/racc/info.rb')} +$".push "#{__dir__}/racc/info.rb" + +module Racc + VERSION = '1.8.1' + Version = VERSION + Copyright = 'Copyright (c) 1999-2006 Minero Aoki' +end + +end + + +module Racc + class ParseError < StandardError; end +end +unless defined?(::ParseError) + ParseError = Racc::ParseError # :nodoc: +end + +# Racc is an LALR(1) parser generator. +# It is written in Ruby itself, and generates Ruby programs. +# +# == Command-line Reference +# +# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] +# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] +# [-v] [--verbose] +# [-O<var>filename</var>] [--log-file=<var>filename</var>] +# [-g] [--debug] +# [-E] [--embedded] +# [-l] [--no-line-convert] +# [-c] [--line-convert-all] +# [-a] [--no-omit-actions] +# [-C] [--check-only] +# [-S] [--output-status] +# [--version] [--copyright] [--help] <var>grammarfile</var> +# +# [+grammarfile+] +# Racc grammar file. Any extension is permitted. +# [-o+outfile+, --output-file=+outfile+] +# A filename for output. default is <+filename+>.tab.rb +# [-O+filename+, --log-file=+filename+] +# Place logging output in file +filename+. +# Default log file name is <+filename+>.output. +# [-e+rubypath+, --executable=+rubypath+] +# output executable file(mode 755). where +path+ is the Ruby interpreter. +# [-v, --verbose] +# verbose mode. create +filename+.output file, like yacc's y.output file. +# [-g, --debug] +# add debug code to parser class. To display debugging information, +# use this '-g' option and set @yydebug true in parser class. +# [-E, --embedded] +# Output parser which doesn't need runtime files (racc/parser.rb). +# [-F, --frozen] +# Output parser which declares frozen_string_literals: true +# [-C, --check-only] +# Check syntax of racc grammar file and quit. +# [-S, --output-status] +# Print messages time to time while compiling. +# [-l, --no-line-convert] +# turns off line number converting. +# [-c, --line-convert-all] +# Convert line number of actions, inner, header and footer. +# [-a, --no-omit-actions] +# Call all actions, even if an action is empty. +# [--version] +# print Racc version and quit. +# [--copyright] +# Print copyright and quit. +# [--help] +# Print usage and quit. +# +# == Generating Parser Using Racc +# +# To compile Racc grammar file, simply type: +# +# $ racc parse.y +# +# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. +# +# == Writing A Racc Grammar File +# +# If you want your own parser, you have to write a grammar file. +# A grammar file contains the name of your parser class, grammar for the parser, +# user code, and anything else. +# When writing a grammar file, yacc's knowledge is helpful. +# If you have not used yacc before, Racc is not too difficult. +# +# Here's an example Racc grammar file. +# +# class Calcparser +# rule +# target: exp { print val[0] } +# +# exp: exp '+' exp +# | exp '*' exp +# | '(' exp ')' +# | NUMBER +# end +# +# Racc grammar files resemble yacc files. +# But (of course), this is Ruby code. +# yacc's $$ is the 'result', $0, $1... is +# an array called 'val', and $-1, $-2... is an array called '_values'. +# +# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for +# more information on grammar files. +# +# == Parser +# +# Then you must prepare the parse entry method. There are two types of +# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse +# +# Racc::Parser#do_parse is simple. +# +# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). +# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. +# EOF is [false, false]. +# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. +# If you want to change this, see the grammar reference. +# +# Racc::Parser#yyparse is little complicated, but useful. +# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. +# +# For example, <code>yyparse(obj, :scan)</code> causes +# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. +# +# == Debugging +# +# When debugging, "-v" or/and the "-g" option is helpful. +# +# "-v" creates verbose log file (.output). +# "-g" creates a "Verbose Parser". +# Verbose Parser prints the internal status when parsing. +# But it's _not_ automatic. +# You must use -g option and set +@yydebug+ to +true+ in order to get output. +# -g option only creates the verbose parser. +# +# === Racc reported syntax error. +# +# Isn't there too many "end"? +# grammar of racc file is changed in v0.10. +# +# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. +# +# === Racc reported "XXXX conflicts". +# +# Try "racc -v xxxx.y". +# It causes producing racc's internal log file, xxxx.output. +# +# === Generated parsers does not work correctly +# +# Try "racc -g xxxx.y". +# This command let racc generate "debugging parser". +# Then set @yydebug=true in your parser. +# It produces a working log of your parser. +# +# == Re-distributing Racc runtime +# +# A parser, which is created by Racc, requires the Racc runtime module; +# racc/parser.rb. +# +# Ruby 1.8.x comes with Racc runtime module, +# you need NOT distribute Racc runtime files. +# +# If you want to include the Racc runtime module with your parser. +# This can be done by using '-E' option: +# +# $ racc -E -omyparser.rb myparser.y +# +# This command creates myparser.rb which `includes' Racc runtime. +# Only you must do is to distribute your parser file (myparser.rb). +# +# Note: parser.rb is ruby license, but your parser is not. +# Your own parser is completely yours. +module Racc + + unless defined?(Racc_No_Extensions) + Racc_No_Extensions = false # :nodoc: + end + + class Parser + + Racc_Runtime_Version = ::Racc::VERSION + Racc_Runtime_Core_Version_R = ::Racc::VERSION + + begin + if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' + require 'jruby' + require 'racc/cparse-jruby.jar' + com.headius.racc.Cparse.new.load(JRuby.runtime, false) + else + require 'racc/cparse' + end + + unless new.respond_to?(:_racc_do_parse_c, true) + raise LoadError, 'old cparse.so' + end + if Racc_No_Extensions + raise LoadError, 'selecting ruby version of racc runtime core' + end + + Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: + Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: + Racc_Runtime_Type = 'c' # :nodoc: + rescue LoadError + Racc_Main_Parsing_Routine = :_racc_do_parse_rb + Racc_YY_Parse_Method = :_racc_yyparse_rb + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R + Racc_Runtime_Type = 'ruby' + end + + def Parser.racc_runtime_type # :nodoc: + Racc_Runtime_Type + end + + def _racc_setup + @yydebug = false unless self.class::Racc_debug_parser + @yydebug = false unless defined?(@yydebug) + if @yydebug + @racc_debug_out = $stderr unless defined?(@racc_debug_out) + @racc_debug_out ||= $stderr + end + arg = self.class::Racc_arg + arg[13] = true if arg.size < 14 + arg + end + + def _racc_init_sysvars + @racc_state = [0] + @racc_tstack = [] + @racc_vstack = [] + + @racc_t = nil + @racc_val = nil + + @racc_read_next = true + + @racc_user_yyerror = false + @racc_error_status = 0 + end + + # The entry point of the parser. This method is used with #next_token. + # If Racc wants to get token (and its value), calls next_token. + # + # Example: + # def parse + # @q = [[1,1], + # [2,2], + # [3,3], + # [false, '$']] + # do_parse + # end + # + # def next_token + # @q.shift + # end + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def do_parse + #{Racc_Main_Parsing_Routine}(_racc_setup(), false) + end + RUBY + + # The method to fetch next token. + # If you use #do_parse method, you must implement #next_token. + # + # The format of return value is [TOKEN_SYMBOL, VALUE]. + # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT + # for 'IDENT'. ";" (String) for ';'. + # + # The final symbol (End of file) must be false. + def next_token + raise NotImplementedError, "#{self.class}\#next_token is not defined" + end + + def _racc_do_parse_rb(arg, in_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + tok = act = i = nil + + catch(:racc_end_parse) { + while true + if i = action_pointer[@racc_state[-1]] + if @racc_read_next + if @racc_t != 0 # not EOF + tok, @racc_val = next_token() + unless tok # EOF + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + racc_read_token(@racc_t, tok, @racc_val) if @yydebug + @racc_read_next = false + end + end + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + else + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + } + end + + # Another entry point for the parser. + # If you use this method, you must implement RECEIVER#METHOD_ID method. + # + # RECEIVER#METHOD_ID is a method to get next token. + # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def yyparse(recv, mid) + #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) + end + RUBY + + def _racc_yyparse_rb(recv, mid, arg, c_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + + catch(:racc_end_parse) { + until i = action_pointer[@racc_state[-1]] + while act = _racc_evalact(action_default[@racc_state[-1]], arg) + ; + end + end + recv.__send__(mid) do |tok, val| + unless tok + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + @racc_val = val + @racc_read_next = false + + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + + while !(i = action_pointer[@racc_state[-1]]) || + ! @racc_read_next || + @racc_t == 0 # $ + unless i and i += @racc_t and + i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + end + } + end + + ### + ### common + ### + + def _racc_evalact(act, arg) + action_table, action_check, _, action_pointer, + _, _, _, _, + _, _, _, shift_n, + reduce_n, * = arg + nerr = 0 # tmp + + if act > 0 and act < shift_n + # + # shift + # + if @racc_error_status > 0 + @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF + end + @racc_vstack.push @racc_val + @racc_state.push act + @racc_read_next = true + if @yydebug + @racc_tstack.push @racc_t + racc_shift @racc_t, @racc_tstack, @racc_vstack + end + + elsif act < 0 and act > -reduce_n + # + # reduce + # + code = catch(:racc_jump) { + @racc_state.push _racc_do_reduce(arg, act) + false + } + if code + case code + when 1 # yyerror + @racc_user_yyerror = true # user_yyerror + return -reduce_n + when 2 # yyaccept + return shift_n + else + raise '[Racc Bug] unknown jump code' + end + end + + elsif act == shift_n + # + # accept + # + racc_accept if @yydebug + throw :racc_end_parse, @racc_vstack[0] + + elsif act == -reduce_n + # + # error + # + case @racc_error_status + when 0 + unless arg[21] # user_yyerror + nerr += 1 + on_error @racc_t, @racc_val, @racc_vstack + end + when 3 + if @racc_t == 0 # is $ + # We're at EOF, and another error occurred immediately after + # attempting auto-recovery + throw :racc_end_parse, nil + end + @racc_read_next = true + end + @racc_user_yyerror = false + @racc_error_status = 3 + while true + if i = action_pointer[@racc_state[-1]] + i += 1 # error token + if i >= 0 and + (act = action_table[i]) and + action_check[i] == @racc_state[-1] + break + end + end + throw :racc_end_parse, nil if @racc_state.size <= 1 + @racc_state.pop + @racc_vstack.pop + if @yydebug + @racc_tstack.pop + racc_e_pop @racc_state, @racc_tstack, @racc_vstack + end + end + return act + + else + raise "[Racc Bug] unknown action #{act.inspect}" + end + + racc_next_state(@racc_state[-1], @racc_state) if @yydebug + + nil + end + + def _racc_do_reduce(arg, act) + _, _, _, _, + goto_table, goto_check, goto_default, goto_pointer, + nt_base, reduce_table, _, _, + _, use_result, * = arg + + state = @racc_state + vstack = @racc_vstack + tstack = @racc_tstack + + i = act * -3 + len = reduce_table[i] + reduce_to = reduce_table[i+1] + method_id = reduce_table[i+2] + void_array = [] + + tmp_t = tstack[-len, len] if @yydebug + tmp_v = vstack[-len, len] + tstack[-len, len] = void_array if @yydebug + vstack[-len, len] = void_array + state[-len, len] = void_array + + # tstack must be updated AFTER method call + if use_result + vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) + else + vstack.push __send__(method_id, tmp_v, vstack) + end + tstack.push reduce_to + + racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug + + k1 = reduce_to - nt_base + if i = goto_pointer[k1] + i += state[-1] + if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 + return curstate + end + end + goto_default[k1] + end + + # This method is called when a parse error is found. + # + # ERROR_TOKEN_ID is an internal ID of token which caused error. + # You can get string representation of this ID by calling + # #token_to_str. + # + # ERROR_VALUE is a value of error token. + # + # value_stack is a stack of symbol values. + # DO NOT MODIFY this object. + # + # This method raises ParseError by default. + # + # If this method returns, parsers enter "error recovering mode". + def on_error(t, val, vstack) + raise ParseError, sprintf("parse error on value %s (%s)", + val.inspect, token_to_str(t) || '?') + end + + # Enter error recovering mode. + # This method does not call #on_error. + def yyerror + throw :racc_jump, 1 + end + + # Exit parser. + # Return value is +Symbol_Value_Stack[0]+. + def yyaccept + throw :racc_jump, 2 + end + + # Leave error recovering mode. + def yyerrok + @racc_error_status = 0 + end + + # For debugging output + def racc_read_token(t, tok, val) + @racc_debug_out.print 'read ' + @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' + @racc_debug_out.puts val.inspect + @racc_debug_out.puts + end + + def racc_shift(tok, tstack, vstack) + @racc_debug_out.puts "shift #{racc_token2str tok}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_reduce(toks, sim, tstack, vstack) + out = @racc_debug_out + out.print 'reduce ' + if toks.empty? + out.print ' <none>' + else + toks.each {|t| out.print ' ', racc_token2str(t) } + end + out.puts " --> #{racc_token2str(sim)}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_accept + @racc_debug_out.puts 'accept' + @racc_debug_out.puts + end + + def racc_e_pop(state, tstack, vstack) + @racc_debug_out.puts 'error recovering mode: pop token' + racc_print_states state + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_next_state(curstate, state) + @racc_debug_out.puts "goto #{curstate}" + racc_print_states state + @racc_debug_out.puts + end + + def racc_print_stacks(t, v) + out = @racc_debug_out + out.print ' [' + t.each_index do |i| + out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' + end + out.puts ' ]' + end + + def racc_print_states(s) + out = @racc_debug_out + out.print ' [' + s.each {|st| out.print ' ', st } + out.puts ' ]' + end + + def racc_token2str(tok) + self.class::Racc_token_to_s_table[tok] or + raise "[Racc Bug] can't convert token #{tok} to string" + end + + # Convert internal ID of token symbol to the string. + def token_to_str(t) + self.class::Racc_token_to_s_table[t] + end + + end + +end + +...end racc/parser.rb/module_eval... +end +###### racc/parser.rb end +module Lrama + class Parser < Racc::Parser + +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 504) + +include Lrama::Tracer::Duration + +def initialize(text, path, debug = false, locations = false, define = {}) + @path = path + @grammar_file = Lrama::Lexer::GrammarFile.new(path, text) + @yydebug = debug || define.key?('parse.trace') + @rule_counter = Lrama::Grammar::Counter.new(0) + @midrule_action_counter = Lrama::Grammar::Counter.new(1) + @locations = locations + @define = define +end + +def parse + message = "parse '#{File.basename(@path)}'" + report_duration(message) do + @lexer = Lrama::Lexer.new(@grammar_file) + @grammar = Lrama::Grammar.new(@rule_counter, @locations, @define) + @precedence_number = 0 + reset_precs + do_parse + @grammar + end +end + +def next_token + @lexer.next_token +end + +def on_error(error_token_id, error_value, value_stack) + case error_value + when Lrama::Lexer::Token::Int + location = error_value.location + value = "#{error_value.s_value}" + when Lrama::Lexer::Token::Token + location = error_value.location + value = "\"#{error_value.s_value}\"" + when Lrama::Lexer::Token::Base + location = error_value.location + value = "'#{error_value.s_value}'" + else + location = @lexer.location + value = error_value.inspect + end + + error_message = "parse error on value #{value} (#{token_to_str(error_token_id) || '?'})" + + raise_parse_error(error_message, location) +end + +def on_action_error(error_message, error_value) + if error_value.is_a?(Lrama::Lexer::Token::Base) + location = error_value.location + else + location = @lexer.location + end + + raise_parse_error(error_message, location) +end + +private + +def reset_precs + @opening_prec_seen = false + @trailing_prec_seen = false + @code_after_prec = false +end + +def prec_seen? + @opening_prec_seen || @trailing_prec_seen +end + +def begin_c_declaration(end_symbol) + @lexer.status = :c_declaration + @lexer.end_symbol = end_symbol +end + +def end_c_declaration + @lexer.status = :initial + @lexer.end_symbol = nil +end + +def raise_parse_error(error_message, location) + raise ParseError, location.generate_error_message(error_message) +end +...end parser.y/module_eval... +##### State transition tables begin ### + +racc_action_table = [ + 98, 98, 99, 99, 87, 53, 53, 52, 178, 110, + 110, 97, 53, 53, 184, 178, 110, 110, 53, 181, + 184, 162, 110, 6, 163, 181, 181, 53, 53, 52, + 52, 181, 79, 79, 53, 53, 52, 52, 43, 79, + 79, 53, 4, 52, 5, 110, 88, 94, 182, 125, + 126, 163, 100, 100, 180, 193, 194, 195, 137, 185, + 188, 180, 4, 44, 5, 185, 188, 94, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 46, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 47, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 47, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 50, + 57, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 12, 13, 57, + 60, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 53, 53, 52, + 52, 110, 105, 53, 53, 52, 52, 110, 105, 53, + 53, 52, 52, 110, 105, 53, 53, 52, 52, 110, + 105, 53, 53, 52, 52, 110, 110, 53, 53, 52, + 209, 110, 110, 53, 53, 209, 52, 110, 110, 53, + 53, 209, 52, 110, 193, 194, 195, 137, 216, 222, + 229, 217, 217, 217, 53, 53, 52, 52, 193, 194, + 195, 57, 57, 57, 57, 66, 67, 68, 69, 70, + 72, 72, 72, 86, 89, 47, 57, 57, 113, 117, + 117, 79, 123, 124, 131, 47, 133, 137, 139, 143, + 149, 150, 151, 152, 133, 155, 156, 157, 110, 166, + 149, 169, 172, 173, 72, 175, 176, 183, 189, 166, + 196, 137, 200, 202, 137, 166, 211, 166, 137, 72, + 176, 218, 176, 72, 72, 227, 137, 72 ] + +racc_action_check = [ + 51, 97, 51, 97, 41, 75, 165, 75, 165, 75, + 165, 51, 171, 190, 171, 190, 171, 190, 201, 165, + 201, 148, 201, 1, 148, 171, 190, 36, 37, 36, + 37, 201, 36, 37, 38, 39, 38, 39, 5, 38, + 39, 117, 0, 117, 0, 117, 41, 46, 168, 88, + 88, 168, 51, 97, 165, 177, 177, 177, 177, 171, + 171, 190, 2, 6, 2, 201, 201, 90, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 9, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 10, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 11, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 3, 3, 12, + 14, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 15, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 8, 8, 16, + 17, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 18, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 63, 13, 63, + 13, 63, 63, 64, 73, 64, 73, 64, 64, 65, + 78, 65, 78, 65, 65, 106, 79, 106, 79, 106, + 106, 118, 180, 118, 180, 118, 180, 188, 196, 188, + 196, 188, 196, 202, 217, 202, 217, 202, 217, 218, + 113, 218, 113, 218, 186, 186, 186, 186, 208, 213, + 226, 208, 213, 226, 114, 123, 114, 123, 210, 210, + 210, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 40, 42, 47, 55, 60, 71, 74, + 76, 80, 81, 87, 91, 92, 93, 94, 102, 116, + 124, 125, 126, 127, 133, 136, 137, 138, 144, 150, + 151, 153, 156, 158, 162, 163, 164, 170, 174, 176, + 178, 179, 182, 184, 187, 189, 199, 200, 204, 205, + 207, 209, 212, 214, 216, 221, 222, 228 ] + +racc_action_pointer = [ + 32, 23, 52, 93, nil, 31, 63, nil, 123, 68, + 74, 84, 103, 165, 94, 111, 123, 135, 141, nil, + nil, nil, nil, nil, 215, 216, 217, 218, 230, 231, + 232, 233, 234, 232, 233, 234, 24, 25, 31, 32, + 238, -1, 242, nil, nil, nil, 43, 232, nil, nil, + nil, -5, nil, nil, nil, 230, nil, nil, nil, nil, + 231, nil, nil, 164, 170, 176, nil, nil, nil, nil, + nil, 240, nil, 171, 241, 2, 242, nil, 177, 183, + 243, 244, nil, nil, nil, nil, nil, 209, 45, nil, + 63, 245, 242, 243, 202, nil, nil, -4, nil, nil, + nil, nil, 256, nil, nil, nil, 182, nil, nil, nil, + nil, nil, nil, 207, 221, nil, 253, 38, 188, nil, + nil, nil, nil, 222, 255, 215, 218, 252, nil, nil, + nil, nil, nil, 251, nil, nil, 219, 261, 250, nil, + nil, nil, nil, nil, 261, nil, nil, nil, -24, nil, + 219, 265, nil, 269, nil, nil, 216, nil, 256, nil, + nil, nil, 266, 270, 227, 3, nil, nil, 3, nil, + 228, 9, nil, nil, 232, nil, 229, 3, 236, 226, + 189, nil, 236, nil, 239, nil, 162, 229, 194, 235, + 10, nil, nil, nil, nil, nil, 195, nil, nil, 284, + 237, 15, 200, nil, 233, 281, nil, 241, 173, 247, + 176, nil, 243, 174, 285, nil, 286, 201, 206, nil, + nil, 278, 241, nil, nil, nil, 175, nil, 289, nil, + nil ] + +racc_action_default = [ + -1, -136, -1, -3, -10, -136, -136, -2, -3, -136, + -14, -14, -136, -136, -136, -136, -136, -136, -136, -28, + -29, -34, -35, -36, -136, -136, -136, -136, -136, -136, + -136, -136, -136, -54, -54, -54, -136, -136, -136, -136, + -136, -136, -136, -13, 231, -4, -136, -14, -16, -17, + -20, -131, -100, -101, -130, -18, -23, -89, -24, -25, + -136, -27, -37, -136, -136, -136, -41, -42, -43, -44, + -45, -46, -55, -136, -47, -136, -48, -49, -92, -136, + -95, -97, -98, -50, -51, -52, -53, -136, -136, -11, + -5, -7, -14, -136, -72, -15, -21, -131, -132, -133, + -134, -19, -136, -26, -30, -31, -32, -38, -87, -88, + -135, -39, -40, -136, -56, -58, -60, -136, -83, -85, + -93, -94, -96, -136, -136, -136, -136, -136, -6, -8, + -9, -128, -104, -102, -105, -73, -136, -136, -136, -90, + -33, -59, -57, -61, -80, -86, -84, -99, -136, -66, + -70, -136, -12, -136, -103, -109, -136, -22, -136, -62, + -81, -82, -54, -136, -64, -68, -71, -74, -136, -129, + -106, -107, -127, -91, -136, -67, -70, -72, -100, -72, + -136, -124, -136, -109, -100, -110, -72, -72, -136, -70, + -69, -75, -76, -116, -117, -118, -136, -78, -79, -136, + -70, -108, -136, -111, -72, -54, -115, -63, -136, -100, + -119, -125, -65, -136, -54, -114, -54, -136, -136, -120, + -121, -136, -72, -112, -77, -122, -136, -126, -54, -123, + -113 ] + +racc_goto_table = [ + 73, 118, 136, 54, 48, 49, 164, 96, 91, 120, + 121, 93, 187, 148, 107, 111, 112, 119, 134, 171, + 56, 58, 59, 3, 61, 7, 78, 78, 78, 78, + 62, 63, 64, 65, 115, 74, 76, 192, 1, 129, + 168, 95, 187, 118, 118, 207, 204, 201, 77, 83, + 84, 85, 128, 138, 147, 93, 212, 140, 154, 145, + 146, 101, 130, 116, 42, 127, 103, 208, 78, 78, + 219, 9, 51, 213, 141, 142, 45, 71, 159, 144, + 190, 160, 161, 102, 158, 191, 132, 197, 122, 226, + 170, 177, 220, 199, 203, 205, 221, 186, 153, nil, + nil, nil, nil, 116, 116, nil, 198, nil, nil, nil, + nil, nil, 214, 78, 206, nil, 177, nil, nil, nil, + nil, nil, 210, nil, nil, nil, nil, 186, 210, 174, + 228, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 225, 210, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 215, nil, nil, nil, nil, nil, nil, nil, + nil, 223, nil, 224, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 230 ] + +racc_goto_check = [ + 29, 22, 42, 31, 14, 14, 35, 16, 8, 48, + 48, 13, 40, 34, 24, 24, 24, 45, 52, 54, + 18, 18, 18, 6, 17, 6, 31, 31, 31, 31, + 17, 17, 17, 17, 30, 26, 26, 38, 1, 5, + 34, 14, 40, 22, 22, 35, 38, 54, 27, 27, + 27, 27, 8, 16, 48, 13, 35, 24, 52, 45, + 45, 18, 9, 31, 10, 11, 17, 39, 31, 31, + 38, 7, 15, 39, 30, 30, 7, 25, 32, 33, + 36, 43, 44, 46, 47, 42, 14, 42, 50, 39, + 53, 22, 55, 56, 42, 42, 57, 22, 58, nil, + nil, nil, nil, 31, 31, nil, 22, nil, nil, nil, + nil, nil, 42, 31, 22, nil, 22, nil, nil, nil, + nil, nil, 22, nil, nil, nil, nil, 22, 22, 29, + 42, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 22, 22, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, 29, nil, nil, nil, nil, nil, nil, nil, + nil, 29, nil, 29, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 29 ] + +racc_goto_pointer = [ + nil, 38, nil, nil, nil, -52, 23, 68, -38, -29, + 60, -24, nil, -35, -6, 59, -44, 6, 6, nil, + nil, nil, -74, nil, -49, 44, 1, 12, nil, -33, + -39, -10, -66, -37, -111, -144, -96, nil, -140, -129, + -159, nil, -92, -63, -62, -58, 26, -55, -69, nil, + 8, nil, -75, -65, -136, -118, -88, -115, -33 ] + +racc_goto_default = [ + nil, nil, 2, 8, 90, nil, nil, nil, nil, nil, + nil, nil, 10, 11, nil, nil, nil, 55, nil, 21, + 22, 23, 104, 106, nil, nil, nil, nil, 114, 75, + nil, 108, nil, nil, nil, nil, 165, 135, nil, nil, + 179, 167, nil, 109, nil, nil, nil, nil, 81, 80, + 82, 92, nil, nil, nil, nil, nil, nil, nil ] + +racc_reduce_table = [ + 0, 0, :racc_error, + 0, 64, :_reduce_1, + 2, 64, :_reduce_2, + 0, 65, :_reduce_3, + 2, 65, :_reduce_4, + 1, 66, :_reduce_5, + 2, 66, :_reduce_6, + 0, 67, :_reduce_none, + 1, 67, :_reduce_none, + 5, 59, :_reduce_none, + 0, 68, :_reduce_10, + 0, 69, :_reduce_11, + 5, 60, :_reduce_12, + 2, 60, :_reduce_13, + 0, 72, :_reduce_14, + 2, 72, :_reduce_15, + 2, 61, :_reduce_none, + 2, 61, :_reduce_none, + 1, 76, :_reduce_18, + 2, 76, :_reduce_19, + 2, 70, :_reduce_20, + 3, 70, :_reduce_21, + 5, 70, :_reduce_22, + 2, 70, :_reduce_none, + 2, 70, :_reduce_24, + 2, 70, :_reduce_25, + 3, 70, :_reduce_26, + 2, 70, :_reduce_27, + 1, 70, :_reduce_28, + 1, 70, :_reduce_29, + 1, 81, :_reduce_30, + 1, 81, :_reduce_31, + 1, 82, :_reduce_32, + 2, 82, :_reduce_33, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 1, 71, :_reduce_none, + 2, 71, :_reduce_37, + 3, 71, :_reduce_38, + 3, 71, :_reduce_39, + 3, 71, :_reduce_40, + 2, 71, :_reduce_41, + 2, 71, :_reduce_42, + 2, 71, :_reduce_43, + 2, 71, :_reduce_44, + 2, 71, :_reduce_45, + 2, 77, :_reduce_none, + 2, 77, :_reduce_47, + 2, 77, :_reduce_48, + 2, 77, :_reduce_49, + 2, 77, :_reduce_50, + 2, 77, :_reduce_51, + 2, 77, :_reduce_52, + 2, 77, :_reduce_53, + 0, 87, :_reduce_none, + 1, 87, :_reduce_none, + 1, 88, :_reduce_56, + 2, 88, :_reduce_57, + 2, 83, :_reduce_58, + 3, 83, :_reduce_59, + 0, 91, :_reduce_none, + 1, 91, :_reduce_none, + 3, 86, :_reduce_62, + 8, 78, :_reduce_63, + 5, 79, :_reduce_64, + 8, 79, :_reduce_65, + 1, 92, :_reduce_66, + 3, 92, :_reduce_67, + 1, 93, :_reduce_68, + 3, 93, :_reduce_69, + 0, 99, :_reduce_none, + 1, 99, :_reduce_none, + 0, 100, :_reduce_none, + 1, 100, :_reduce_none, + 1, 94, :_reduce_74, + 3, 94, :_reduce_75, + 3, 94, :_reduce_76, + 6, 94, :_reduce_77, + 3, 94, :_reduce_78, + 3, 94, :_reduce_79, + 0, 102, :_reduce_none, + 1, 102, :_reduce_none, + 1, 90, :_reduce_82, + 1, 103, :_reduce_83, + 2, 103, :_reduce_84, + 2, 84, :_reduce_85, + 3, 84, :_reduce_86, + 1, 80, :_reduce_none, + 1, 80, :_reduce_none, + 0, 104, :_reduce_89, + 0, 105, :_reduce_90, + 5, 75, :_reduce_91, + 1, 106, :_reduce_92, + 2, 106, :_reduce_93, + 2, 107, :_reduce_94, + 1, 108, :_reduce_95, + 2, 108, :_reduce_96, + 1, 85, :_reduce_97, + 1, 85, :_reduce_98, + 3, 85, :_reduce_99, + 1, 89, :_reduce_none, + 1, 89, :_reduce_none, + 1, 110, :_reduce_102, + 2, 110, :_reduce_103, + 2, 62, :_reduce_none, + 2, 62, :_reduce_none, + 4, 109, :_reduce_106, + 1, 111, :_reduce_107, + 3, 111, :_reduce_108, + 0, 112, :_reduce_109, + 2, 112, :_reduce_110, + 3, 112, :_reduce_111, + 5, 112, :_reduce_112, + 7, 112, :_reduce_113, + 4, 112, :_reduce_114, + 3, 112, :_reduce_115, + 1, 96, :_reduce_116, + 1, 96, :_reduce_117, + 1, 96, :_reduce_118, + 0, 113, :_reduce_none, + 1, 113, :_reduce_none, + 2, 97, :_reduce_121, + 3, 97, :_reduce_122, + 4, 97, :_reduce_123, + 0, 114, :_reduce_124, + 0, 115, :_reduce_125, + 5, 98, :_reduce_126, + 3, 95, :_reduce_127, + 0, 116, :_reduce_128, + 3, 63, :_reduce_129, + 1, 73, :_reduce_none, + 0, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 74, :_reduce_none, + 1, 101, :_reduce_135 ] + +racc_reduce_n = 136 + +racc_shift_n = 231 + +racc_token_table = { + false => 0, + :error => 1, + :C_DECLARATION => 2, + :CHARACTER => 3, + :IDENT_COLON => 4, + :IDENTIFIER => 5, + :INTEGER => 6, + :STRING => 7, + :TAG => 8, + "%%" => 9, + "%{" => 10, + "%}" => 11, + "%require" => 12, + ";" => 13, + "%expect" => 14, + "%define" => 15, + "{" => 16, + "}" => 17, + "%param" => 18, + "%lex-param" => 19, + "%parse-param" => 20, + "%code" => 21, + "%initial-action" => 22, + "%no-stdlib" => 23, + "%locations" => 24, + "%union" => 25, + "%destructor" => 26, + "%printer" => 27, + "%error-token" => 28, + "%after-shift" => 29, + "%before-reduce" => 30, + "%after-reduce" => 31, + "%after-shift-error-token" => 32, + "%after-pop-stack" => 33, + "-temp-group" => 34, + "%token" => 35, + "%type" => 36, + "%nterm" => 37, + "%left" => 38, + "%right" => 39, + "%precedence" => 40, + "%nonassoc" => 41, + "%start" => 42, + "%rule" => 43, + "(" => 44, + ")" => 45, + ":" => 46, + "%inline" => 47, + "," => 48, + "|" => 49, + "%empty" => 50, + "%prec" => 51, + "?" => 52, + "+" => 53, + "*" => 54, + "[" => 55, + "]" => 56, + "{...}" => 57 } + +racc_nt_base = 58 + +racc_use_result_var = true + +Racc_arg = [ + racc_action_table, + racc_action_check, + racc_action_default, + racc_action_pointer, + racc_goto_table, + racc_goto_check, + racc_goto_default, + racc_goto_pointer, + racc_nt_base, + racc_reduce_table, + racc_token_table, + racc_shift_n, + racc_reduce_n, + racc_use_result_var ] +Ractor.make_shareable(Racc_arg) if defined?(Ractor) + +Racc_token_to_s_table = [ + "$end", + "error", + "C_DECLARATION", + "CHARACTER", + "IDENT_COLON", + "IDENTIFIER", + "INTEGER", + "STRING", + "TAG", + "\"%%\"", + "\"%{\"", + "\"%}\"", + "\"%require\"", + "\";\"", + "\"%expect\"", + "\"%define\"", + "\"{\"", + "\"}\"", + "\"%param\"", + "\"%lex-param\"", + "\"%parse-param\"", + "\"%code\"", + "\"%initial-action\"", + "\"%no-stdlib\"", + "\"%locations\"", + "\"%union\"", + "\"%destructor\"", + "\"%printer\"", + "\"%error-token\"", + "\"%after-shift\"", + "\"%before-reduce\"", + "\"%after-reduce\"", + "\"%after-shift-error-token\"", + "\"%after-pop-stack\"", + "\"-temp-group\"", + "\"%token\"", + "\"%type\"", + "\"%nterm\"", + "\"%left\"", + "\"%right\"", + "\"%precedence\"", + "\"%nonassoc\"", + "\"%start\"", + "\"%rule\"", + "\"(\"", + "\")\"", + "\":\"", + "\"%inline\"", + "\",\"", + "\"|\"", + "\"%empty\"", + "\"%prec\"", + "\"?\"", + "\"+\"", + "\"*\"", + "\"[\"", + "\"]\"", + "\"{...}\"", + "$start", + "input", + "prologue_declaration", + "bison_declaration", + "rules_or_grammar_declaration", + "epilogue_declaration", + "\"-many@prologue_declaration\"", + "\"-many@bison_declaration\"", + "\"-many1@rules_or_grammar_declaration\"", + "\"-option@epilogue_declaration\"", + "@1", + "@2", + "parser_option", + "grammar_declaration", + "\"-many@;\"", + "variable", + "value", + "param", + "\"-many1@param\"", + "symbol_declaration", + "rule_declaration", + "inline_declaration", + "symbol", + "\"-group@symbol|TAG\"", + "\"-many1@-group@symbol|TAG\"", + "token_declarations", + "symbol_declarations", + "token_declarations_for_precedence", + "token_declaration", + "\"-option@TAG\"", + "\"-many1@token_declaration\"", + "id", + "alias", + "\"-option@INTEGER\"", + "rule_args", + "rule_rhs_list", + "rule_rhs", + "named_ref", + "parameterized_suffix", + "parameterized_args", + "action", + "\"-option@%empty\"", + "\"-option@named_ref\"", + "string_as_id", + "\"-option@string_as_id\"", + "\"-many1@symbol\"", + "@3", + "@4", + "\"-many1@id\"", + "\"-group@TAG-\\\"-many1@id\\\"\"", + "\"-many1@-group@TAG-\\\"-many1@id\\\"\"", + "rules", + "\"-many1@;\"", + "rhs_list", + "rhs", + "\"-option@parameterized_suffix\"", + "@5", + "@6", + "@7" ] +Ractor.make_shareable(Racc_token_to_s_table) if defined?(Ractor) + +Racc_debug_parser = true + +##### State transition tables end ##### + +# reduce 0 omitted + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_1(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_2(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_3(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_4(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_5(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 11) + def _reduce_6(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 7 omitted + +# reduce 8 omitted + +# reduce 9 omitted + +module_eval(<<'.,.,', 'parser.y', 13) + def _reduce_10(val, _values, result) + begin_c_declaration("%}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 17) + def _reduce_11(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 21) + def _reduce_12(val, _values, result) + @grammar.prologue_first_lineno = val[0].first_line + @grammar.prologue = val[2].s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 26) + def _reduce_13(val, _values, result) + @grammar.required = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_14(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 34) + def _reduce_15(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 16 omitted + +# reduce 17 omitted + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_18(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 77) + def _reduce_19(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 36) + def _reduce_20(val, _values, result) + @grammar.expect = val[1].s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 40) + def _reduce_21(val, _values, result) + @grammar.define[val[1].s_value] = val[2]&.s_value + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 44) + def _reduce_22(val, _values, result) + @grammar.define[val[1].s_value] = val[3]&.s_value + + result + end +.,., + +# reduce 23 omitted + +module_eval(<<'.,.,', 'parser.y', 49) + def _reduce_24(val, _values, result) + val[1].each {|token| + @grammar.lex_param = Grammar::Code::NoReferenceCode.new(type: :lex_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 55) + def _reduce_25(val, _values, result) + val[1].each {|token| + @grammar.parse_param = Grammar::Code::NoReferenceCode.new(type: :parse_param, token_code: token).token_code.s_value + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 61) + def _reduce_26(val, _values, result) + @grammar.add_percent_code(id: val[1], code: val[2]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 65) + def _reduce_27(val, _values, result) + @grammar.initial_action = Grammar::Code::InitialActionCode.new(type: :initial_action, token_code: val[1]) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 69) + def _reduce_28(val, _values, result) + @grammar.no_stdlib = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 73) + def _reduce_29(val, _values, result) + @grammar.locations = true + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_30(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_31(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_32(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 133) + def _reduce_33(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 34 omitted + +# reduce 35 omitted + +# reduce 36 omitted + +module_eval(<<'.,.,', 'parser.y', 82) + def _reduce_37(val, _values, result) + @grammar.set_union( + Grammar::Code::NoReferenceCode.new(type: :union, token_code: val[1]), + val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 89) + def _reduce_38(val, _values, result) + @grammar.add_destructor( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 97) + def _reduce_39(val, _values, result) + @grammar.add_printer( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 105) + def _reduce_40(val, _values, result) + @grammar.add_error_token( + ident_or_tags: val[2].flatten, + token_code: val[1], + lineno: val[1].line + ) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 113) + def _reduce_41(val, _values, result) + @grammar.after_shift = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 117) + def _reduce_42(val, _values, result) + @grammar.before_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 121) + def _reduce_43(val, _values, result) + @grammar.after_reduce = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 125) + def _reduce_44(val, _values, result) + @grammar.after_shift_error_token = val[1] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 129) + def _reduce_45(val, _values, result) + @grammar.after_pop_stack = val[1] + + result + end +.,., + +# reduce 46 omitted + +module_eval(<<'.,.,', 'parser.y', 136) + def _reduce_47(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + @grammar.add_type(id: id, tag: hash[:tag]) + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 144) + def _reduce_48(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + if @grammar.find_term_by_s_value(id.s_value) + on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id) + else + @grammar.add_type(id: id, tag: hash[:tag]) + end + } + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 156) + def _reduce_49(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_left(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 166) + def _reduce_50(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_right(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 176) + def _reduce_51(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_precedence(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 186) + def _reduce_52(val, _values, result) + val[1].each {|hash| + hash[:tokens].each {|id| + sym = @grammar.add_term(id: id, tag: hash[:tag]) + @grammar.add_nonassoc(sym, @precedence_number, id.s_value, id.first_line) + } + } + @precedence_number += 1 + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 196) + def _reduce_53(val, _values, result) + @grammar.set_start_nterm(val[1]) + + result + end +.,., + +# reduce 54 omitted + +# reduce 55 omitted + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_56(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 214) + def _reduce_57(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 202) + def _reduce_58(val, _values, result) + val[1].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[0], replace: true) + } + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 208) + def _reduce_59(val, _values, result) + val[2].each {|token_declaration| + @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1]&.s_value, tag: val[1], replace: true) + } + + result + end +.,., + +# reduce 60 omitted + +# reduce 61 omitted + +module_eval(<<'.,.,', 'parser.y', 213) + def _reduce_62(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 218) + def _reduce_63(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[1].s_value, val[3], val[7], tag: val[5]) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 225) + def _reduce_64(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, [], val[4], is_inline: true) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 230) + def _reduce_65(val, _values, result) + rule = Grammar::Parameterized::Rule.new(val[2].s_value, val[4], val[7], is_inline: true) + @grammar.add_parameterized_rule(rule) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 235) + def _reduce_66(val, _values, result) + result = [val[0]] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 236) + def _reduce_67(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 241) + def _reduce_68(val, _values, result) + builder = val[0] + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 246) + def _reduce_69(val, _values, result) + builder = val[2] + result = val[0].append(builder) + + result + end +.,., + +# reduce 70 omitted + +# reduce 71 omitted + +# reduce 72 omitted + +# reduce 73 omitted + +module_eval(<<'.,.,', 'parser.y', 253) + def _reduce_74(val, _values, result) + reset_precs + result = Grammar::Parameterized::Rhs.new + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 258) + def _reduce_75(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.symbols << token + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 267) + def _reduce_76(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], location: @lexer.location, args: [val[1]]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 274) + def _reduce_77(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + builder = val[0] + builder.symbols << Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[3], lhs_tag: val[5]) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 281) + def _reduce_78(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 289) + def _reduce_79(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +# reduce 80 omitted + +# reduce 81 omitted + +module_eval(<<'.,.,', 'parser.y', 301) + def _reduce_82(val, _values, result) + result = val[0].s_value if val[0] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_83(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 315) + def _reduce_84(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 306) + def _reduce_85(val, _values, result) + result = if val[0] + [{tag: val[0], tokens: val[1]}] + else + [{tag: nil, tokens: val[1]}] + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 312) + def _reduce_86(val, _values, result) + result = val[0].append({tag: val[1], tokens: val[2]}) + result + end +.,., + +# reduce 87 omitted + +# reduce 88 omitted + +module_eval(<<'.,.,', 'parser.y', 321) + def _reduce_89(val, _values, result) + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 325) + def _reduce_90(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 329) + def _reduce_91(val, _values, result) + result = val[2] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_92(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_93(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_94(val, _values, result) + result = val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_95(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 338) + def _reduce_96(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 333) + def _reduce_97(val, _values, result) + result = [{tag: nil, tokens: val[0]}] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 334) + def _reduce_98(val, _values, result) + result = val[0].map {|tag, ids| {tag: tag, tokens: ids} } + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 335) + def _reduce_99(val, _values, result) + result = [{tag: nil, tokens: val[0]}, {tag: val[1], tokens: val[2]}] + result + end +.,., + +# reduce 100 omitted + +# reduce 101 omitted + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_102(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 346) + def _reduce_103(val, _values, result) + result = val[1] ? val[1].unshift(val[0]) : val + result + end +.,., + +# reduce 104 omitted + +# reduce 105 omitted + +module_eval(<<'.,.,', 'parser.y', 348) + def _reduce_106(val, _values, result) + lhs = val[0] + lhs.alias_name = val[1] + val[3].each do |builder| + builder.lhs = lhs + builder.complete_input + @grammar.add_rule_builder(builder) + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 360) + def _reduce_107(val, _values, result) + if val[0].rhs.count > 1 + empties = val[0].rhs.select { |sym| sym.is_a?(Lrama::Lexer::Token::Empty) } + empties.each do |empty| + on_action_error("%empty on non-empty rule", empty) + end + end + builder = val[0] + if !builder.line + builder.line = @lexer.line - 1 + end + result = [builder] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 374) + def _reduce_108(val, _values, result) + builder = val[2] + if !builder.line + builder.line = @lexer.line - 1 + end + result = val[0].append(builder) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 384) + def _reduce_109(val, _values, result) + reset_precs + result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 389) + def _reduce_110(val, _values, result) + builder = val[0] + builder.add_rhs(Lrama::Lexer::Token::Empty.new(location: @lexer.location)) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 395) + def _reduce_111(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = val[1] + token.alias_name = val[2] + builder = val[0] + builder.add_rhs(token) + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 404) + def _reduce_112(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 413) + def _reduce_113(val, _values, result) + on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen + token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) + builder = val[0] + builder.add_rhs(token) + builder.line = val[1].first_line + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 422) + def _reduce_114(val, _values, result) + user_code = val[1] + user_code.alias_name = val[2] + user_code.tag = val[3] + builder = val[0] + builder.user_code = user_code + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 431) + def _reduce_115(val, _values, result) + on_action_error("multiple %prec in a rule", val[0]) if prec_seen? + sym = @grammar.find_symbol_by_id!(val[2]) + if val[0].rhs.empty? + @opening_prec_seen = true + else + @trailing_prec_seen = true + end + builder = val[0] + builder.precedence_sym = sym + result = builder + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 444) + def _reduce_116(val, _values, result) + result = "option" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 445) + def _reduce_117(val, _values, result) + result = "nonempty_list" + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 446) + def _reduce_118(val, _values, result) + result = "list" + result + end +.,., + +# reduce 119 omitted + +# reduce 120 omitted + +module_eval(<<'.,.,', 'parser.y', 451) + def _reduce_121(val, _values, result) + result = if val[1] + [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] + else + [val[0]] + end + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 457) + def _reduce_122(val, _values, result) + result = val[0].append(val[2]) + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 458) + def _reduce_123(val, _values, result) + result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 463) + def _reduce_124(val, _values, result) + if prec_seen? + on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec + @code_after_prec = true + end + begin_c_declaration("}") + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 471) + def _reduce_125(val, _values, result) + end_c_declaration + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 475) + def _reduce_126(val, _values, result) + result = val[2] + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 478) + def _reduce_127(val, _values, result) + result = val[1].s_value + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 483) + def _reduce_128(val, _values, result) + begin_c_declaration('\Z') + + result + end +.,., + +module_eval(<<'.,.,', 'parser.y', 487) + def _reduce_129(val, _values, result) + end_c_declaration + @grammar.epilogue_first_lineno = val[0].first_line + 1 + @grammar.epilogue = val[2].s_value + + result + end +.,., + +# reduce 130 omitted + +# reduce 131 omitted + +# reduce 132 omitted + +# reduce 133 omitted + +# reduce 134 omitted + +module_eval(<<'.,.,', 'parser.y', 499) + def _reduce_135(val, _values, result) + result = Lrama::Lexer::Token::Ident.new(s_value: val[0].s_value) + result + end +.,., + +def _reduce_none(val, _values, result) + val[0] +end + + end # class Parser +end # module Lrama diff --git a/tool/lrama/lib/lrama/reporter.rb b/tool/lrama/lib/lrama/reporter.rb new file mode 100644 index 0000000000..ed25cc7f8f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'reporter/conflicts' +require_relative 'reporter/grammar' +require_relative 'reporter/precedences' +require_relative 'reporter/profile' +require_relative 'reporter/rules' +require_relative 'reporter/states' +require_relative 'reporter/terms' + +module Lrama + class Reporter + include Lrama::Tracer::Duration + + # @rbs (**bool options) -> void + def initialize(**options) + @options = options + @rules = Rules.new(**options) + @terms = Terms.new(**options) + @conflicts = Conflicts.new + @precedences = Precedences.new + @grammar = Grammar.new(**options) + @states = States.new(**options) + end + + # @rbs (File io, Lrama::States states) -> void + def report(io, states) + report_duration(:report) do + report_duration(:report_rules) { @rules.report(io, states) } + report_duration(:report_terms) { @terms.report(io, states) } + report_duration(:report_conflicts) { @conflicts.report(io, states) } + report_duration(:report_precedences) { @precedences.report(io, states) } + report_duration(:report_grammar) { @grammar.report(io, states) } + report_duration(:report_states) { @states.report(io, states, ielr: states.ielr_defined?) } + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/conflicts.rb b/tool/lrama/lib/lrama/reporter/conflicts.rb new file mode 100644 index 0000000000..f4d8c604c9 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/conflicts.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Conflicts + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_conflicts(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_conflicts(io, states) + has_conflict = false + + states.states.each do |state| + messages = format_conflict_messages(state.conflicts) + + unless messages.empty? + has_conflict = true + io << "State #{state.id} conflicts: #{messages.join(', ')}\n" + end + end + + io << "\n\n" if has_conflict + end + + # @rbs (Array[(Lrama::State::ShiftReduceConflict | Lrama::State::ReduceReduceConflict)] conflicts) -> Array[String] + def format_conflict_messages(conflicts) + conflict_types = { + shift_reduce: "shift/reduce", + reduce_reduce: "reduce/reduce" + } + + conflict_types.keys.map do |type| + type_conflicts = conflicts.select { |c| c.type == type } + "#{type_conflicts.count} #{conflict_types[type]}" unless type_conflicts.empty? + end.compact + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/grammar.rb b/tool/lrama/lib/lrama/reporter/grammar.rb new file mode 100644 index 0000000000..dc3f3f6bfd --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/grammar.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Grammar + # @rbs (?grammar: bool, **bool _) -> void + def initialize(grammar: false, **_) + @grammar = grammar + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @grammar + + io << "Grammar\n" + last_lhs = nil + + states.rules.each do |rule| + if rule.empty_rule? + r = "ε" + else + r = rule.rhs.map(&:display_name).join(" ") + end + + if rule.lhs == last_lhs + io << sprintf("%5d %s| %s", rule.id, " " * rule.lhs.display_name.length, r) << "\n" + else + io << "\n" + io << sprintf("%5d %s: %s", rule.id, rule.lhs.display_name, r) << "\n" + end + + last_lhs = rule.lhs + end + io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/precedences.rb b/tool/lrama/lib/lrama/reporter/precedences.rb new file mode 100644 index 0000000000..73c0888700 --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/precedences.rb @@ -0,0 +1,54 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Precedences + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + report_precedences(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_precedences(io, states) + used_precedences = states.precedences.select(&:used_by?) + + return if used_precedences.empty? + + io << "Precedences\n\n" + + used_precedences.each do |precedence| + io << " precedence on #{precedence.symbol.display_name} is used to resolve conflict on\n" + + if precedence.used_by_lalr? + io << " LALR\n" + + precedence.used_by_lalr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + + if precedence.used_by_ielr? + io << " IELR\n" + + precedence.used_by_ielr.uniq.sort_by do |resolved_conflict| + resolved_conflict.state.id + end.each do |resolved_conflict| + io << " state #{resolved_conflict.state.id}. #{resolved_conflict.report_precedences_message}\n" + end + + io << "\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile.rb b/tool/lrama/lib/lrama/reporter/profile.rb new file mode 100644 index 0000000000..b569b94d4f --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative 'profile/call_stack' +require_relative 'profile/memory' diff --git a/tool/lrama/lib/lrama/reporter/profile/call_stack.rb b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb new file mode 100644 index 0000000000..8a4d44b61c --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/call_stack.rb @@ -0,0 +1,45 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module CallStack + # See "Call-stack Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_stackprof + ex = nil #: Exception? + path = 'tmp/stackprof-cpu-myapp.dump' + + StackProf.run(mode: :cpu, raw: true, out: path) do + yield + rescue Exception => e + ex = e + end + + STDERR.puts("Call-stack Profiling result is generated on #{path}") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_stackprof + require "stackprof" + true + rescue LoadError + warn "stackprof is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/profile/memory.rb b/tool/lrama/lib/lrama/reporter/profile/memory.rb new file mode 100644 index 0000000000..a019581fdf --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/profile/memory.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + module Profile + module Memory + # See "Memory Profiling Lrama" in README.md for how to use. + # + # @rbs enabled: bool + # @rbs &: -> void + # @rbs return: StackProf::result | void + def self.report(enabled) + if enabled && require_memory_profiler + ex = nil #: Exception? + + report = MemoryProfiler.report do # steep:ignore UnknownConstant + yield + rescue Exception => e + ex = e + end + + report.pretty_print(to_file: "tmp/memory_profiler.txt") + + if ex + raise ex + end + else + yield + end + end + + # @rbs return: bool + def self.require_memory_profiler + require "memory_profiler" + true + rescue LoadError + warn "memory_profiler is not installed. Please run `bundle install`." + false + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/rules.rb b/tool/lrama/lib/lrama/reporter/rules.rb new file mode 100644 index 0000000000..3e8bf19a0a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/rules.rb @@ -0,0 +1,43 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Rules + # @rbs (?rules: bool, **bool _) -> void + def initialize(rules: false, **_) + @rules = rules + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @rules + + used_rules = states.rules.flat_map(&:rhs) + + unless used_rules.empty? + io << "Rule Usage Frequency\n\n" + frequency_counts = used_rules.each_with_object(Hash.new(0)) { |rule, counts| counts[rule] += 1 } + + frequency_counts + .select { |rule,| !rule.midrule? } + .sort_by { |rule, count| [-count, rule.name] } + .each_with_index { |(rule, count), i| io << sprintf("%5d %s (%d times)", i, rule.name, count) << "\n" } + io << "\n\n" + end + + unused_rules = states.rules.map(&:lhs).select do |rule| + !used_rules.include?(rule) && rule.token_id != 0 + end + + unless unused_rules.empty? + io << "#{unused_rules.count} Unused Rules\n\n" + unused_rules.each_with_index do |rule, index| + io << sprintf("%5d %s", index, rule.display_name) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/states.rb b/tool/lrama/lib/lrama/reporter/states.rb new file mode 100644 index 0000000000..d152d0511a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/states.rb @@ -0,0 +1,387 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class States + # @rbs (?itemsets: bool, ?lookaheads: bool, ?solved: bool, ?counterexamples: bool, ?verbose: bool, **bool _) -> void + def initialize(itemsets: false, lookaheads: false, solved: false, counterexamples: false, verbose: false, **_) + @itemsets = itemsets + @lookaheads = lookaheads + @solved = solved + @counterexamples = counterexamples + @verbose = verbose + end + + # @rbs (IO io, Lrama::States states, ielr: bool) -> void + def report(io, states, ielr: false) + cex = Counterexamples.new(states) if @counterexamples + + states.compute_la_sources_for_conflicted_states + report_split_states(io, states.states) if ielr + + states.states.each do |state| + report_state_header(io, state) + report_items(io, state) + report_conflicts(io, state) + report_shifts(io, state) + report_nonassoc_errors(io, state) + report_reduces(io, state) + report_nterm_transitions(io, state) + report_conflict_resolutions(io, state) if @solved + report_counterexamples(io, state, cex) if @counterexamples && state.has_conflicts? # @type var cex: Lrama::Counterexamples + report_verbose_info(io, state, states) if @verbose + # End of Report State + io << "\n" + end + end + + private + + # @rbs (IO io, Array[Lrama::State] states) -> void + def report_split_states(io, states) + ss = states.select(&:split_state?) + + return if ss.empty? + + io << "Split States\n\n" + + ss.each do |state| + io << " State #{state.id} is split from state #{state.lalr_isocore.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_state_header(io, state) + io << "State #{state.id}\n\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_items(io, state) + last_lhs = nil + list = @itemsets ? state.items : state.kernels + + list.sort_by {|i| [i.rule_id, i.position] }.each do |item| + r = item.empty_rule? ? "ε •" : item.rhs.map(&:display_name).insert(item.position, "•").join(" ") + + l = if item.lhs == last_lhs + " " * item.lhs.id.s_value.length + "|" + else + item.lhs.id.s_value + ":" + end + + la = "" + if @lookaheads && item.end_of_rule? + reduce = state.find_reduce_by_item!(item) + look_ahead = reduce.selected_look_ahead + unless look_ahead.empty? + la = " [#{look_ahead.compact.map(&:display_name).join(", ")}]" + end + end + + last_lhs = item.lhs + io << sprintf("%5i %s %s%s", item.rule_id, l, r, la) << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflicts(io, state) + return if state.conflicts.empty? + + state.conflicts.each do |conflict| + syms = conflict.symbols.map { |sym| sym.display_name } + io << " Conflict on #{syms.join(", ")}. " + + case conflict.type + when :shift_reduce + # @type var conflict: Lrama::State::ShiftReduceConflict + io << "shift/reduce(#{conflict.reduce.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + when :reduce_reduce + # @type var conflict: Lrama::State::ReduceReduceConflict + io << "reduce(#{conflict.reduce1.item.rule.lhs.display_name})/reduce(#{conflict.reduce2.item.rule.lhs.display_name})\n" + + conflict.symbols.each do |token| + conflict.reduce1.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + + conflict.reduce2.look_ahead_sources[token].each do |goto| # steep:ignore NoMethod + io << " #{token.display_name} comes from state #{goto.from_state.id} goto by #{goto.next_sym.display_name}\n" + end + end + else + raise "Unknown conflict type #{conflict.type}" + end + + io << "\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_shifts(io, state) + shifts = state.term_transitions.reject(&:not_selected) + + return if shifts.empty? + + next_syms = shifts.map(&:next_sym) + max_len = next_syms.map(&:display_name).map(&:length).max + shifts.each do |shift| + io << " #{shift.next_sym.display_name.ljust(max_len)} shift, and go to state #{shift.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nonassoc_errors(io, state) + error_symbols = state.resolved_conflicts.select { |resolved| resolved.which == :error }.map { |error| error.symbol.display_name } + + return if error_symbols.empty? + + max_len = error_symbols.map(&:length).max + error_symbols.each do |name| + io << " #{name.ljust(max_len)} error (nonassociative)\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_reduces(io, state) + reduce_pairs = [] #: Array[[Lrama::Grammar::Symbol, Lrama::State::Action::Reduce]] + + state.non_default_reduces.each do |reduce| + reduce.look_ahead&.each do |term| + reduce_pairs << [term, reduce] + end + end + + return if reduce_pairs.empty? && !state.default_reduction_rule + + max_len = [ + reduce_pairs.map(&:first).map(&:display_name).map(&:length).max || 0, + state.default_reduction_rule ? "$default".length : 0 + ].max + + reduce_pairs.sort_by { |term, _| term.number }.each do |term, reduce| + rule = reduce.item.rule + io << " #{term.display_name.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.display_name})\n" + end + + if (r = state.default_reduction_rule) + s = "$default".ljust(max_len) + + if r.initial_rule? + io << " #{s} accept\n" + else + io << " #{s} reduce using rule #{r.id} (#{r.lhs.display_name})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_nterm_transitions(io, state) + return if state.nterm_transitions.empty? + + goto_transitions = state.nterm_transitions.sort_by do |goto| + goto.next_sym.number + end + + max_len = goto_transitions.map(&:next_sym).map do |nterm| + nterm.id.s_value.length + end.max + goto_transitions.each do |goto| + io << " #{goto.next_sym.id.s_value.ljust(max_len)} go to state #{goto.to_state.id}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state) -> void + def report_conflict_resolutions(io, state) + return if state.resolved_conflicts.empty? + + state.resolved_conflicts.each do |resolved| + io << " #{resolved.report_message}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::Counterexamples cex) -> void + def report_counterexamples(io, state, cex) + examples = cex.compute(state) + + examples.each do |example| + is_shift_reduce = example.type == :shift_reduce + label0 = is_shift_reduce ? "shift/reduce" : "reduce/reduce" + label1 = is_shift_reduce ? "Shift derivation" : "First Reduce derivation" + label2 = is_shift_reduce ? "Reduce derivation" : "Second Reduce derivation" + + io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" + io << " #{label1}\n" + + example.derivations1.render_strings_for_report.each do |str| + io << " #{str}\n" + end + + io << " #{label2}\n" + + example.derivations2.render_strings_for_report.each do |str| + io << " #{str}\n" + end + end + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_verbose_info(io, state, states) + report_direct_read_sets(io, state, states) + report_reads_relation(io, state, states) + report_read_sets(io, state, states) + report_includes_relation(io, state, states) + report_lookback_relation(io, state, states) + report_follow_sets(io, state, states) + report_look_ahead_sets(io, state, states) + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_direct_read_sets(io, state, states) + io << " [Direct Read sets]\n" + direct_read_sets = states.direct_read_sets + + state.nterm_transitions.each do |goto| + terms = direct_read_sets[goto] + next unless terms && !terms.empty? + + str = terms.map { |sym| sym.id.s_value }.join(", ") + io << " read #{goto.next_sym.id.s_value} shift #{str}\n" + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_reads_relation(io, state, states) + io << " [Reads Relation]\n" + + state.nterm_transitions.each do |goto| + goto2 = states.reads_relation[goto] + next unless goto2 + + goto2.each do |goto2| + io << " (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_read_sets(io, state, states) + io << " [Read sets]\n" + read_sets = states.read_sets + + state.nterm_transitions.each do |goto| + terms = read_sets[goto] + next unless terms && !terms.empty? + + terms.each do |sym| + io << " #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_includes_relation(io, state, states) + io << " [Includes Relation]\n" + + state.nterm_transitions.each do |goto| + gotos = states.includes_relation[goto] + next unless gotos + + gotos.each do |goto2| + io << " (State #{state.id}, #{goto.next_sym.id.s_value}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_lookback_relation(io, state, states) + io << " [Lookback Relation]\n" + + states.rules.each do |rule| + gotos = states.lookback_relation.dig(state.id, rule.id) + next unless gotos + + gotos.each do |goto2| + io << " (Rule: #{rule.display_name}) -> (State #{goto2.from_state.id}, #{goto2.next_sym.id.s_value})\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_follow_sets(io, state, states) + io << " [Follow sets]\n" + follow_sets = states.follow_sets + + state.nterm_transitions.each do |goto| + terms = follow_sets[goto] + next unless terms + + terms.each do |sym| + io << " #{goto.next_sym.id.s_value} -> #{sym.id.s_value}\n" + end + end + + io << "\n" + end + + # @rbs (IO io, Lrama::State state, Lrama::States states) -> void + def report_look_ahead_sets(io, state, states) + io << " [Look-Ahead Sets]\n" + look_ahead_rules = [] #: Array[[Lrama::Grammar::Rule, Array[Lrama::Grammar::Symbol]]] + + states.rules.each do |rule| + syms = states.la.dig(state.id, rule.id) + next unless syms + + look_ahead_rules << [rule, syms] + end + + return if look_ahead_rules.empty? + + max_len = look_ahead_rules.flat_map { |_, syms| syms.map { |s| s.id.s_value.length } }.max + + look_ahead_rules.each do |rule, syms| + syms.each do |sym| + io << " #{sym.id.s_value.ljust(max_len)} reduce using rule #{rule.id} (#{rule.lhs.id.s_value})\n" + end + end + + io << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/reporter/terms.rb b/tool/lrama/lib/lrama/reporter/terms.rb new file mode 100644 index 0000000000..f72d8b1a1a --- /dev/null +++ b/tool/lrama/lib/lrama/reporter/terms.rb @@ -0,0 +1,44 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Terms + # @rbs (?terms: bool, **bool _) -> void + def initialize(terms: false, **_) + @terms = terms + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @terms + + look_aheads = states.states.each do |state| + state.reduces.flat_map do |reduce| + reduce.look_ahead unless reduce.look_ahead.nil? + end + end + + next_terms = states.states.flat_map do |state| + state.term_transitions.map {|shift| shift.next_sym } + end + + unused_symbols = states.terms.reject do |term| + (look_aheads + next_terms).include?(term) + end + + io << states.terms.count << " Terms\n\n" + + io << states.nterms.count << " Non-Terminals\n\n" + + unless unused_symbols.empty? + io << "#{unused_symbols.count} Unused Terms\n\n" + unused_symbols.each_with_index do |term, index| + io << sprintf("%5d %s", index, term.id.s_value) << "\n" + end + io << "\n\n" + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb new file mode 100644 index 0000000000..50912e094e --- /dev/null +++ b/tool/lrama/lib/lrama/state.rb @@ -0,0 +1,534 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "state/action" +require_relative "state/inadequacy_annotation" +require_relative "state/item" +require_relative "state/reduce_reduce_conflict" +require_relative "state/resolved_conflict" +require_relative "state/shift_reduce_conflict" + +module Lrama + class State + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type conflict = State::ShiftReduceConflict | State::ReduceReduceConflict + # type transition = Action::Shift | Action::Goto + # type lookahead_set = Hash[Item, Array[Grammar::Symbol]] + # + # @id: Integer + # @accessing_symbol: Grammar::Symbol + # @kernels: Array[Item] + # @items: Array[Item] + # @items_to_state: Hash[Array[Item], State] + # @conflicts: Array[conflict] + # @resolved_conflicts: Array[ResolvedConflict] + # @default_reduction_rule: Grammar::Rule? + # @closure: Array[Item] + # @nterm_transitions: Array[Action::Goto] + # @term_transitions: Array[Action::Shift] + # @transitions: Array[transition] + # @internal_dependencies: Hash[Action::Goto, Array[Action::Goto]] + # @successor_dependencies: Hash[Action::Goto, Array[Action::Goto]] + + attr_reader :id #: Integer + attr_reader :accessing_symbol #: Grammar::Symbol + attr_reader :kernels #: Array[Item] + attr_reader :conflicts #: Array[conflict] + attr_reader :resolved_conflicts #: Array[ResolvedConflict] + attr_reader :default_reduction_rule #: Grammar::Rule? + attr_reader :closure #: Array[Item] + attr_reader :items #: Array[Item] + attr_reader :annotation_list #: Array[InadequacyAnnotation] + attr_reader :predecessors #: Array[State] + attr_reader :items_to_state #: Hash[Array[Item], State] + attr_reader :lane_items #: Hash[State, Array[[Item, Item]]] + + attr_accessor :_transitions #: Array[[Grammar::Symbol, Array[Item]]] + attr_accessor :reduces #: Array[Action::Reduce] + attr_accessor :ielr_isocores #: Array[State] + attr_accessor :lalr_isocore #: State + attr_accessor :lookaheads_recomputed #: bool + attr_accessor :follow_kernel_items #: Hash[Action::Goto, Hash[Item, bool]] + attr_accessor :always_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor :goto_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + + # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void + def initialize(id, accessing_symbol, kernels) + @id = id + @accessing_symbol = accessing_symbol + @kernels = kernels.freeze + @items = @kernels + # Manage relationships between items to state + # to resolve next state + @items_to_state = {} + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + @predecessors = [] + @lalr_isocore = self + @ielr_isocores = [self] + @internal_dependencies = {} + @successor_dependencies = {} + @annotation_list = [] + @lookaheads_recomputed = false + @follow_kernel_items = {} + @always_follows = {} + @goto_follows = {} + @lhs_contributions = {} + @lane_items = {} + end + + # @rbs (State other) -> bool + def ==(other) + self.id == other.id + end + + # @rbs (Array[Item] closure) -> void + def closure=(closure) + @closure = closure + @items = @kernels + @closure + end + + # @rbs () -> Array[Action::Reduce] + def non_default_reduces + reduces.reject do |reduce| + reduce.rule == @default_reduction_rule + end + end + + # @rbs () -> void + def compute_transitions_and_reduces + _transitions = {} + @_lane_items ||= {} + reduces = [] + items.each do |item| + # TODO: Consider what should be pushed + if item.end_of_rule? + reduces << Action::Reduce.new(item) + else + key = item.next_sym + _transitions[key] ||= [] + @_lane_items[key] ||= [] + next_item = item.new_by_next_position + _transitions[key] << next_item + @_lane_items[key] << [item, next_item] + end + end + + # It seems Bison 3.8.2 iterates transitions order by symbol number + transitions = _transitions.sort_by do |next_sym, to_items| + next_sym.number + end + + self._transitions = transitions.freeze + self.reduces = reduces.freeze + end + + # @rbs (Grammar::Symbol next_sym, State next_state) -> void + def set_lane_items(next_sym, next_state) + @lane_items[next_state] = @_lane_items[next_sym] + end + + # @rbs (Array[Item] items, State next_state) -> void + def set_items_to_state(items, next_state) + @items_to_state[items] = next_state + end + + # @rbs (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void + def set_look_ahead(rule, look_ahead) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead = look_ahead + end + + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void + def set_look_ahead_sources(rule, sources) + reduce = reduces.find do |r| + r.rule == rule + end + + reduce.look_ahead_sources = sources + end + + # @rbs () -> Array[Action::Goto] + def nterm_transitions # steep:ignore + @nterm_transitions ||= transitions.select {|transition| transition.is_a?(Action::Goto) } + end + + # @rbs () -> Array[Action::Shift] + def term_transitions # steep:ignore + @term_transitions ||= transitions.select {|transition| transition.is_a?(Action::Shift) } + end + + # @rbs () -> Array[transition] + def transitions + @transitions ||= _transitions.map do |next_sym, to_items| + if next_sym.term? + Action::Shift.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + else + Action::Goto.new(self, next_sym, to_items.flatten, @items_to_state[to_items]) + end + end + end + + # @rbs (transition transition, State next_state) -> void + def update_transition(transition, next_state) + set_items_to_state(transition.to_items, next_state) + next_state.append_predecessor(self) + update_transitions_caches(transition) + end + + # @rbs () -> void + def update_transitions_caches(transition) + new_transition = + if transition.next_sym.term? + Action::Shift.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + else + Action::Goto.new(self, transition.next_sym, transition.to_items, @items_to_state[transition.to_items]) + end + + @transitions.delete(transition) + @transitions << new_transition + @nterm_transitions = nil + @term_transitions = nil + + @follow_kernel_items[new_transition] = @follow_kernel_items.delete(transition) + @always_follows[new_transition] = @always_follows.delete(transition) + end + + # @rbs () -> Array[Action::Shift] + def selected_term_transitions + term_transitions.reject do |shift| + shift.not_selected + end + end + + # Move to next state by sym + # + # @rbs (Grammar::Symbol sym) -> State + def transition(sym) + result = nil + + if sym.term? + result = term_transitions.find {|shift| shift.next_sym == sym }.to_state + else + result = nterm_transitions.find {|goto| goto.next_sym == sym }.to_state + end + + raise "Can not transit by #{sym} #{self}" if result.nil? + + result + end + + # @rbs (Item item) -> Action::Reduce + def find_reduce_by_item!(item) + reduces.find do |r| + r.item == item + end || (raise "reduce is not found. #{item}") + end + + # @rbs (Grammar::Rule default_reduction_rule) -> void + def default_reduction_rule=(default_reduction_rule) + @default_reduction_rule = default_reduction_rule + + reduces.each do |r| + if r.rule == default_reduction_rule + r.default_reduction = true + end + end + end + + # @rbs () -> bool + def has_conflicts? + !@conflicts.empty? + end + + # @rbs () -> Array[conflict] + def sr_conflicts + @conflicts.select do |conflict| + conflict.type == :shift_reduce + end + end + + # @rbs () -> Array[conflict] + def rr_conflicts + @conflicts.select do |conflict| + conflict.type == :reduce_reduce + end + end + + # Clear information related to conflicts. + # IELR computation re-calculates conflicts and default reduction of states + # after LALR computation. + # Call this method before IELR computation to avoid duplicated conflicts information + # is stored. + # + # @rbs () -> void + def clear_conflicts + @conflicts = [] + @resolved_conflicts = [] + @default_reduction_rule = nil + + term_transitions.each(&:clear_conflicts) + reduces.each(&:clear_conflicts) + end + + # @rbs () -> bool + def split_state? + @lalr_isocore != self + end + + # Definition 3.40 (propagate_lookaheads) + # + # @rbs (State next_state) -> lookahead_set + def propagate_lookaheads(next_state) + next_state.kernels.map {|next_kernel| + lookahead_sets = + if next_kernel.position > 1 + kernel = kernels.find {|k| k.predecessor_item_of?(next_kernel) } + item_lookahead_set[kernel] + else + goto_follow_set(next_kernel.lhs) + end + + [next_kernel, lookahead_sets & next_state.lookahead_set_filters[next_kernel]] + }.to_h + end + + # Definition 3.43 (is_compatible) + # + # @rbs (lookahead_set filtered_lookahead) -> bool + def is_compatible?(filtered_lookahead) + !lookaheads_recomputed || + @lalr_isocore.annotation_list.all? {|annotation| + a = annotation.dominant_contribution(item_lookahead_set) + b = annotation.dominant_contribution(filtered_lookahead) + a.nil? || b.nil? || a == b + } + end + + # Definition 3.38 (lookahead_set_filters) + # + # @rbs () -> lookahead_set + def lookahead_set_filters + @lookahead_set_filters ||= kernels.map {|kernel| + [kernel, @lalr_isocore.annotation_list.select {|annotation| annotation.contributed?(kernel) }.map(&:token)] + }.to_h + end + + # Definition 3.27 (inadequacy_lists) + # + # @rbs () -> Hash[Grammar::Symbol, Array[Action::Shift | Action::Reduce]] + def inadequacy_list + return @inadequacy_list if @inadequacy_list + + inadequacy_list = {} + + term_transitions.each do |shift| + inadequacy_list[shift.next_sym] ||= [] + inadequacy_list[shift.next_sym] << shift.dup + end + reduces.each do |reduce| + next if reduce.look_ahead.nil? + + reduce.look_ahead.each do |token| + inadequacy_list[token] ||= [] + inadequacy_list[token] << reduce.dup + end + end + + @inadequacy_list = inadequacy_list.select {|token, actions| actions.size > 1 } + end + + # Definition 3.30 (annotate_manifestation) + # + # @rbs () -> void + def annotate_manifestation + inadequacy_list.each {|token, actions| + contribution_matrix = actions.map {|action| + if action.is_a?(Action::Shift) + [action, nil] + else + [action, action.rule.empty_rule? ? lhs_contributions(action.rule.lhs, token) : kernels.map {|k| [k, k.rule == action.item.rule && k.end_of_rule?] }.to_h] + end + }.to_h + @annotation_list << InadequacyAnnotation.new(self, token, actions, contribution_matrix) + } + end + + # Definition 3.32 (annotate_predecessor) + # + # @rbs (State predecessor) -> void + def annotate_predecessor(predecessor) + propagating_list = annotation_list.map {|annotation| + contribution_matrix = annotation.contribution_matrix.map {|action, contributions| + if contributions.nil? + [action, nil] + elsif first_kernels.any? {|kernel| contributions[kernel] && predecessor.lhs_contributions(kernel.lhs, annotation.token).empty? } + [action, nil] + else + cs = predecessor.lane_items[self].map {|pred_kernel, kernel| + c = contributions[kernel] && ( + (kernel.position > 1 && predecessor.item_lookahead_set[pred_kernel].include?(annotation.token)) || + (kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, annotation.token)[pred_kernel]) + ) + [pred_kernel, c] + }.to_h + [action, cs] + end + }.to_h + + # Observation 3.33 (Simple Split-Stable Dominance) + # + # If all of contributions in the contribution_matrix are + # always contribution or never contribution, we can stop annotate propagations + # to the predecessor state. + next nil if contribution_matrix.all? {|_, contributions| contributions.nil? || contributions.all? {|_, contributed| !contributed } } + + InadequacyAnnotation.new(annotation.state, annotation.token, annotation.actions, contribution_matrix) + }.compact + predecessor.append_annotation_list(propagating_list) + end + + # @rbs () -> Array[Item] + def first_kernels + @first_kernels ||= kernels.select {|kernel| kernel.position == 1 } + end + + # @rbs (Array[InadequacyAnnotation] propagating_list) -> void + def append_annotation_list(propagating_list) + annotation_list.each do |annotation| + merging_list = propagating_list.select {|a| a.state == annotation.state && a.token == annotation.token && a.actions == annotation.actions } + annotation.merge_matrix(merging_list.map(&:contribution_matrix)) + propagating_list -= merging_list + end + + @annotation_list += propagating_list + end + + # Definition 3.31 (compute_lhs_contributions) + # + # @rbs (Grammar::Symbol sym, Grammar::Symbol token) -> (nil | Hash[Item, bool]) + def lhs_contributions(sym, token) + return @lhs_contributions[sym][token] unless @lhs_contributions.dig(sym, token).nil? + + transition = nterm_transitions.find {|goto| goto.next_sym == sym } + @lhs_contributions[sym] ||= {} + @lhs_contributions[sym][token] = + if always_follows[transition].include?(token) + {} + else + kernels.map {|kernel| [kernel, follow_kernel_items[transition][kernel] && item_lookahead_set[kernel].include?(token)] }.to_h + end + end + + # Definition 3.26 (item_lookahead_sets) + # + # @rbs () -> lookahead_set + def item_lookahead_set + return @item_lookahead_set if @item_lookahead_set + + @item_lookahead_set = kernels.map {|k| [k, []] }.to_h + @item_lookahead_set = kernels.map {|kernel| + value = + if kernel.lhs.accept_symbol? + [] + elsif kernel.position > 1 + prev_items = predecessors_with_item(kernel) + prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms } + elsif kernel.position == 1 + prev_state = @predecessors.find {|p| p.transitions.any? {|transition| transition.next_sym == kernel.lhs } } + goto = prev_state.nterm_transitions.find {|goto| goto.next_sym == kernel.lhs } + prev_state.goto_follows[goto] + end + [kernel, value] + }.to_h + end + + # @rbs (lookahead_set k) -> void + def item_lookahead_set=(k) + @item_lookahead_set = k + end + + # @rbs (Item item) -> Array[[State, Item]] + def predecessors_with_item(item) + result = [] + @predecessors.each do |pre| + pre.items.each do |i| + result << [pre, i] if i.predecessor_item_of?(item) + end + end + result + end + + # @rbs (State prev_state) -> void + def append_predecessor(prev_state) + @predecessors << prev_state + @predecessors.uniq! + end + + # Definition 3.39 (compute_goto_follow_set) + # + # @rbs (Grammar::Symbol nterm_token) -> Array[Grammar::Symbol] + def goto_follow_set(nterm_token) + return [] if nterm_token.accept_symbol? + goto = @lalr_isocore.nterm_transitions.find {|g| g.next_sym == nterm_token } + + @kernels + .select {|kernel| @lalr_isocore.follow_kernel_items[goto][kernel] } + .map {|kernel| item_lookahead_set[kernel] } + .reduce(@lalr_isocore.always_follows[goto]) {|result, terms| result |= terms } + end + + # Definition 3.8 (Goto Follows Internal Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def internal_dependencies(goto) + return @internal_dependencies[goto] if @internal_dependencies[goto] + + syms = @items.select {|i| + i.next_sym == goto.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0 + }.map(&:lhs).uniq + @internal_dependencies[goto] = nterm_transitions.select {|goto2| syms.include?(goto2.next_sym) } + end + + # Definition 3.5 (Goto Follows Successor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def successor_dependencies(goto) + return @successor_dependencies[goto] if @successor_dependencies[goto] + + @successor_dependencies[goto] = goto.to_state.nterm_transitions.select {|next_goto| next_goto.next_sym.nullable } + end + + # Definition 3.9 (Goto Follows Predecessor Relation) + # + # @rbs (Action::Goto goto) -> Array[Action::Goto] + def predecessor_dependencies(goto) + state_items = [] + @kernels.select {|kernel| + kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) + }.each do |item| + queue = predecessors_with_item(item) + until queue.empty? + st, i = queue.pop + if i.position == 0 + state_items << [st, i] + else + st.predecessors_with_item(i).each {|v| queue << v } + end + end + end + + state_items.map {|state, item| + state.nterm_transitions.find {|goto2| goto2.next_sym == item.lhs } + } + end + end +end diff --git a/tool/lrama/lib/lrama/state/action.rb b/tool/lrama/lib/lrama/state/action.rb new file mode 100644 index 0000000000..791685fc23 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative "action/goto" +require_relative "action/reduce" +require_relative "action/shift" diff --git a/tool/lrama/lib/lrama/state/action/goto.rb b/tool/lrama/lib/lrama/state/action/goto.rb new file mode 100644 index 0000000000..4c2c82afdc --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/goto.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Goto + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/reduce.rb b/tool/lrama/lib/lrama/state/action/reduce.rb new file mode 100644 index 0000000000..9678ab0a98 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/reduce.rb @@ -0,0 +1,71 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Reduce + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @item: Item + # @look_ahead: Array[Grammar::Symbol]? + # @look_ahead_sources: Hash[Grammar::Symbol, Array[Action::Goto]]? + # @not_selected_symbols: Array[Grammar::Symbol] + + attr_reader :item #: Item + attr_reader :look_ahead #: Array[Grammar::Symbol]? + attr_reader :look_ahead_sources #: Hash[Grammar::Symbol, Array[Action::Goto]]? + attr_reader :not_selected_symbols #: Array[Grammar::Symbol] + + # https://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html + attr_accessor :default_reduction #: bool + + # @rbs (Item item) -> void + def initialize(item) + @item = item + @look_ahead = nil + @look_ahead_sources = nil + @not_selected_symbols = [] + end + + # @rbs () -> Grammar::Rule + def rule + @item.rule + end + + # @rbs (Array[Grammar::Symbol] look_ahead) -> Array[Grammar::Symbol] + def look_ahead=(look_ahead) + @look_ahead = look_ahead.freeze + end + + # @rbs (Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> Hash[Grammar::Symbol, Array[Action::Goto]] + def look_ahead_sources=(sources) + @look_ahead_sources = sources.freeze + end + + # @rbs (Grammar::Symbol sym) -> Array[Grammar::Symbol] + def add_not_selected_symbol(sym) + @not_selected_symbols << sym + end + + # @rbs () -> (::Array[Grammar::Symbol?]) + def selected_look_ahead + if look_ahead + look_ahead - @not_selected_symbols + else + [] + end + end + + # @rbs () -> void + def clear_conflicts + @not_selected_symbols = [] + @default_reduction = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/action/shift.rb b/tool/lrama/lib/lrama/state/action/shift.rb new file mode 100644 index 0000000000..52d9f8c4f0 --- /dev/null +++ b/tool/lrama/lib/lrama/state/action/shift.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class Action + class Shift + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @from_state: State + # @next_sym: Grammar::Symbol + # @to_items: Array[Item] + # @to_state: State + + attr_reader :from_state #: State + attr_reader :next_sym #: Grammar::Symbol + attr_reader :to_items #: Array[Item] + attr_reader :to_state #: State + attr_accessor :not_selected #: bool + + # @rbs (State from_state, Grammar::Symbol next_sym, Array[Item] to_items, State to_state) -> void + def initialize(from_state, next_sym, to_items, to_state) + @from_state = from_state + @next_sym = next_sym + @to_items = to_items + @to_state = to_state + end + + # @rbs () -> void + def clear_conflicts + @not_selected = nil + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/inadequacy_annotation.rb b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb new file mode 100644 index 0000000000..3654fa4607 --- /dev/null +++ b/tool/lrama/lib/lrama/state/inadequacy_annotation.rb @@ -0,0 +1,140 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class InadequacyAnnotation + # @rbs! + # type action = Action::Shift | Action::Reduce + + attr_accessor :state #: State + attr_accessor :token #: Grammar::Symbol + attr_accessor :actions #: Array[action] + attr_accessor :contribution_matrix #: Hash[action, Hash[Item, bool]] + + # @rbs (State state, Grammar::Symbol token, Array[action] actions, Hash[action, Hash[Item, bool]] contribution_matrix) -> void + def initialize(state, token, actions, contribution_matrix) + @state = state + @token = token + @actions = actions + @contribution_matrix = contribution_matrix + end + + # @rbs (Item item) -> bool + def contributed?(item) + @contribution_matrix.any? {|action, contributions| !contributions.nil? && contributions[item] } + end + + # @rbs (Array[Hash[action, Hash[Item, bool]]] another_matrixes) -> void + def merge_matrix(another_matrixes) + another_matrixes.each do |another_matrix| + @contribution_matrix.merge!(another_matrix) {|action, contributions, another_contributions| + next contributions if another_contributions.nil? + next another_contributions if contributions.nil? + + contributions.merge!(another_contributions) {|_, contributed, another_contributed| contributed || another_contributed } + } + end + end + + # Definition 3.42 (dominant_contribution) + # + # @rbs (State::lookahead_set lookaheads) -> Array[action]? + def dominant_contribution(lookaheads) + actions = @actions.select {|action| + contribution_matrix[action].nil? || contribution_matrix[action].any? {|item, contributed| contributed && lookaheads[item].include?(@token) } + } + return nil if actions.empty? + + resolve_conflict(actions) + end + + # @rbs (Array[action] actions) -> Array[action] + def resolve_conflict(actions) + # @type var shifts: Array[Action::Shift] + # @type var reduces: Array[Action::Reduce] + shifts = actions.select {|action| action.is_a?(Action::Shift)} + reduces = actions.select {|action| action.is_a?(Action::Reduce) } + + shifts.each do |shift| + reduces.each do |reduce| + sym = shift.next_sym + + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + actions.delete(shift) + next + when shift_prec > reduce_prec + # Shift is selected + actions.delete(reduce) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence&.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + next + when :right + # Shift is selected + actions.delete(reduce) + next + when :left + # Reduce is selected + actions.delete(shift) + next + when :nonassoc + # Can not resolve + # + # nonassoc creates "run-time" error, precedence creates "compile-time" error. + # Then omit both the shift and reduce. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + actions.delete(shift) + actions.delete(reduce) + else + raise "Unknown precedence type. #{sym}" + end + end + end + + actions + end + + # @rbs () -> String + def to_s + "State: #{@state.id}, Token: #{@token.id.s_value}, Actions: #{actions_to_s}, Contributions: #{contribution_matrix_to_s}" + end + + private + + # @rbs () -> String + def actions_to_s + '[' + @actions.map {|action| + if action.is_a?(Action::Shift) || action.is_a?(Action::Goto) + action.class.name + elsif action.is_a?(Action::Reduce) + "#{action.class.name}: (#{action.item})" + end + }.join(', ') + ']' + end + + # @rbs () -> String + def contribution_matrix_to_s + '[' + @contribution_matrix.map {|action, contributions| + "#{(action.is_a?(Action::Shift) || action.is_a?(Action::Goto)) ? action.class.name : "#{action.class.name}: (#{action.item})"}: " + contributions&.transform_keys(&:to_s).to_s + }.join(', ') + ']' + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/item.rb b/tool/lrama/lib/lrama/state/item.rb new file mode 100644 index 0000000000..3ecdd70b76 --- /dev/null +++ b/tool/lrama/lib/lrama/state/item.rb @@ -0,0 +1,120 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +# TODO: Validate position is not over rule rhs + +require "forwardable" + +module Lrama + class State + class Item < Struct.new(:rule, :position, keyword_init: true) + # @rbs! + # include Grammar::Rule::_DelegatedMethods + # + # attr_accessor rule: Grammar::Rule + # attr_accessor position: Integer + # + # def initialize: (?rule: Grammar::Rule, ?position: Integer) -> void + + extend Forwardable + + def_delegators "rule", :lhs, :rhs + + # Optimization for States#setup_state + # + # @rbs () -> Integer + def hash + [rule_id, position].hash + end + + # @rbs () -> Integer + def rule_id + rule.id + end + + # @rbs () -> bool + def empty_rule? + rule.empty_rule? + end + + # @rbs () -> Integer + def number_of_rest_symbols + @number_of_rest_symbols ||= rhs.count - position + end + + # @rbs () -> Grammar::Symbol + def next_sym + rhs[position] + end + + # @rbs () -> Grammar::Symbol + def next_next_sym + @next_next_sym ||= rhs[position + 1] + end + + # @rbs () -> Grammar::Symbol + def previous_sym + rhs[position - 1] + end + + # @rbs () -> bool + def end_of_rule? + rhs.count == position + end + + # @rbs () -> bool + def beginning_of_rule? + position == 0 + end + + # @rbs () -> bool + def start_item? + rule.initial_rule? && beginning_of_rule? + end + + # @rbs () -> State::Item + def new_by_next_position + Item.new(rule: rule, position: position + 1) + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_before_dot # steep:ignore + rhs[0...position] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_dot # steep:ignore + rhs[position..-1] + end + + # @rbs () -> Array[Grammar::Symbol] + def symbols_after_transition # steep:ignore + rhs[position+1..-1] + end + + # @rbs () -> ::String + def to_s + "#{lhs.id.s_value}: #{display_name}" + end + + # @rbs () -> ::String + def display_name + r = rhs.map(&:display_name).insert(position, "•").join(" ") + "#{r} (rule #{rule_id})" + end + + # Right after position + # + # @rbs () -> ::String + def display_rest + r = symbols_after_dot.map(&:display_name).join(" ") + ". #{r} (rule #{rule_id})" + end + + # @rbs (State::Item other_item) -> bool + def predecessor_item_of?(other_item) + rule == other_item.rule && position == other_item.position - 1 + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb new file mode 100644 index 0000000000..55ecad40bd --- /dev/null +++ b/tool/lrama/lib/lrama/state/reduce_reduce_conflict.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class ReduceReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :reduce1 #: State::Action::Reduce + attr_reader :reduce2 #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], reduce1: State::Action::Reduce, reduce2: State::Action::Reduce) -> void + def initialize(symbols:, reduce1:, reduce2:) + @symbols = symbols + @reduce1 = reduce1 + @reduce2 = reduce2 + end + + # @rbs () -> :reduce_reduce + def type + :reduce_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/resolved_conflict.rb b/tool/lrama/lib/lrama/state/resolved_conflict.rb new file mode 100644 index 0000000000..014533c233 --- /dev/null +++ b/tool/lrama/lib/lrama/state/resolved_conflict.rb @@ -0,0 +1,65 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + # * state: A state on which the conflct is resolved + # * symbol: A symbol under discussion + # * reduce: A reduce under discussion + # * which: For which a conflict is resolved. :shift, :reduce or :error (for nonassociative) + # * resolved_by_precedence: If the conflict is resolved by precedence definition or not + class ResolvedConflict + # @rbs! + # type which_enum = :reduce | :shift | :error + + attr_reader :state #: State + attr_reader :symbol #: Grammar::Symbol + attr_reader :reduce #: State::Action::Reduce + attr_reader :which #: which_enum + attr_reader :resolved_by_precedence #: bool + + # @rbs (state: State, symbol: Grammar::Symbol, reduce: State::Action::Reduce, which: which_enum, resolved_by_precedence: bool) -> void + def initialize(state:, symbol:, reduce:, which:, resolved_by_precedence:) + @state = state + @symbol = symbol + @reduce = reduce + @which = which + @resolved_by_precedence = resolved_by_precedence + end + + # @rbs () -> (::String | bot) + def report_message + "Conflict between rule #{reduce.rule.id} and token #{symbol.display_name} #{how_resolved}." + end + + # @rbs () -> (::String | bot) + def report_precedences_message + "Conflict between reduce by \"#{reduce.rule.display_name}\" and shift #{symbol.display_name} #{how_resolved}." + end + + private + + # @rbs () -> (::String | bot) + def how_resolved + s = symbol.display_name + r = reduce.rule.precedence_sym&.display_name + case + when which == :shift && resolved_by_precedence + msg = "resolved as #{which} (%right #{s})" + when which == :shift + msg = "resolved as #{which} (#{r} < #{s})" + when which == :reduce && resolved_by_precedence + msg = "resolved as #{which} (%left #{s})" + when which == :reduce + msg = "resolved as #{which} (#{s} < #{r})" + when which == :error + msg = "resolved as an #{which} (%nonassoc #{s})" + else + raise "Unknown direction. #{self}" + end + + msg + end + end + end +end diff --git a/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb new file mode 100644 index 0000000000..548f2de614 --- /dev/null +++ b/tool/lrama/lib/lrama/state/shift_reduce_conflict.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + class ShiftReduceConflict + attr_reader :symbols #: Array[Grammar::Symbol] + attr_reader :shift #: State::Action::Shift + attr_reader :reduce #: State::Action::Reduce + + # @rbs (symbols: Array[Grammar::Symbol], shift: State::Action::Shift, reduce: State::Action::Reduce) -> void + def initialize(symbols:, shift:, reduce:) + @symbols = symbols + @shift = shift + @reduce = reduce + end + + # @rbs () -> :shift_reduce + def type + :shift_reduce + end + end + end +end diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb new file mode 100644 index 0000000000..ddce627df4 --- /dev/null +++ b/tool/lrama/lib/lrama/states.rb @@ -0,0 +1,867 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "forwardable" +require_relative "tracer/duration" +require_relative "state/item" + +module Lrama + # States is passed to a template file + # + # "Efficient Computation of LALR(1) Look-Ahead Sets" + # https://dl.acm.org/doi/pdf/10.1145/69622.357187 + class States + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # type state_id = Integer + # type rule_id = Integer + # + # include Grammar::_DelegatedMethods + # + # @grammar: Grammar + # @tracer: Tracer + # @states: Array[State] + # @direct_read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @reads_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @read_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @includes_relation: Hash[State::Action::Goto, Array[State::Action::Goto]] + # @lookback_relation: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + # @follow_sets: Hash[State::Action::Goto, Bitmap::bitmap] + # @la: Hash[state_id, Hash[rule_id, Bitmap::bitmap]] + + extend Forwardable + include Lrama::Tracer::Duration + + def_delegators "@grammar", :symbols, :terms, :nterms, :rules, :precedences, + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined? + + attr_reader :states #: Array[State] + attr_reader :reads_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :includes_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] + attr_reader :lookback_relation #: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + + # @rbs (Grammar grammar, Tracer tracer) -> void + def initialize(grammar, tracer) + @grammar = grammar + @tracer = tracer + + @states = [] + + # `DR(p, A) = {t ∈ T | p -(A)-> r -(t)-> }` + # where p is state, A is nterm, t is term. + # + # `@direct_read_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @direct_read_sets = {} + + # Reads relation on nonterminal transitions (pair of state and nterm) + # `(p, A) reads (r, C) iff p -(A)-> r -(C)-> and C =>* ε` + # where p, r are state, A, C are nterm. + # + # `@reads_relation` is a hash whose + # key is goto, + # value is array of goto. + @reads_relation = {} + + # `Read(p, A) =s DR(p, A) ∪ ∪{Read(r, C) | (p, A) reads (r, C)}` + # + # `@read_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @read_sets = {} + + # `(p, A) includes (p', B) iff B -> βAγ, γ =>* ε, p' -(β)-> p` + # where p, p' are state, A, B are nterm, β, γ is sequence of symbol. + # + # `@includes_relation` is a hash whose + # key is goto, + # value is array of goto. + @includes_relation = {} + + # `(q, A -> ω) lookback (p, A) iff p -(ω)-> q` + # where p, q are state, A -> ω is rule, A is nterm, ω is sequence of symbol. + # + # `@lookback_relation` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is array of goto. + @lookback_relation = {} + + # `Follow(p, A) =s Read(p, A) ∪ ∪{Follow(p', B) | (p, A) includes (p', B)}` + # + # `@follow_sets` is a hash whose + # key is goto, + # value is bitmap of term. + @follow_sets = {} + + # `LA(q, A -> ω) = ∪{Follow(p, A) | (q, A -> ω) lookback (p, A)` + # + # `@la` is a two-stage hash whose + # first key is state_id, + # second key is rule_id, + # value is bitmap of term. + @la = {} + end + + # @rbs () -> void + def compute + report_duration(:compute_lr0_states) { compute_lr0_states } + + # Look Ahead Sets + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + + # Conflicts + report_duration(:compute_conflicts) { compute_conflicts(:lalr) } + + report_duration(:compute_default_reduction) { compute_default_reduction } + end + + # @rbs () -> void + def compute_ielr + # Preparation + report_duration(:clear_conflicts) { clear_conflicts } + # Phase 1 + report_duration(:compute_predecessors) { compute_predecessors } + report_duration(:compute_follow_kernel_items) { compute_follow_kernel_items } + report_duration(:compute_always_follows) { compute_always_follows } + report_duration(:compute_goto_follows) { compute_goto_follows } + # Phase 2 + report_duration(:compute_inadequacy_annotations) { compute_inadequacy_annotations } + # Phase 3 + report_duration(:split_states) { split_states } + # Phase 4 + report_duration(:clear_look_ahead_sets) { clear_look_ahead_sets } + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + # Phase 5 + report_duration(:compute_conflicts) { compute_conflicts(:ielr) } + report_duration(:compute_default_reduction) { compute_default_reduction } + end + + # @rbs () -> Integer + def states_count + @states.count + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def direct_read_sets + @_direct_read_sets ||= @direct_read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def read_sets + @_read_sets ||= @read_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[Grammar::Symbol]] + def follow_sets + @_follow_sets ||= @follow_sets.transform_values do |v| + bitmap_to_terms(v) + end + end + + # @rbs () -> Hash[state_id, Hash[rule_id, Array[Grammar::Symbol]]] + def la + @_la ||= @la.transform_values do |second_hash| + second_hash.transform_values do |v| + bitmap_to_terms(v) + end + end + end + + # @rbs () -> Integer + def sr_conflicts_count + @sr_conflicts_count ||= @states.flat_map(&:sr_conflicts).count + end + + # @rbs () -> Integer + def rr_conflicts_count + @rr_conflicts_count ||= @states.flat_map(&:rr_conflicts).count + end + + # @rbs (Logger logger) -> void + def validate!(logger) + validate_conflicts_within_threshold!(logger) + end + + def compute_la_sources_for_conflicted_states + reflexive = {} + @states.each do |state| + state.nterm_transitions.each do |goto| + reflexive[goto] = [goto] + end + end + + # compute_read_sets + read_sets = Digraph.new(nterm_transitions, @reads_relation, reflexive).compute + # compute_follow_sets + follow_sets = Digraph.new(nterm_transitions, @includes_relation, read_sets).compute + + @states.select(&:has_conflicts?).each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + sources = {} + + ary.each do |goto| + source = follow_sets[goto] + + next unless source + + source.each do |goto2| + tokens = direct_read_sets[goto2] + tokens.each do |token| + sources[token] ||= [] + sources[token] |= [goto2] + end + end + end + + state.set_look_ahead_sources(rule, sources) + end + end + end + + private + + # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] + def create_state(accessing_symbol, kernels, states_created) + # A item can appear in some states, + # so need to use `kernels` (not `kernels.first`) as a key. + # + # For example... + # + # %% + # program: '+' strings_1 + # | '-' strings_2 + # ; + # + # strings_1: string_1 + # ; + # + # strings_2: string_1 + # | string_2 + # ; + # + # string_1: string + # ; + # + # string_2: string '+' + # ; + # + # string: tSTRING + # ; + # %% + # + # For these grammar, there are 2 states + # + # State A + # string_1: string • + # + # State B + # string_1: string • + # string_2: string • '+' + # + return [states_created[kernels], false] if states_created[kernels] + + state = State.new(@states.count, accessing_symbol, kernels) + @states << state + states_created[kernels] = state + + return [state, true] + end + + # @rbs (State state) -> void + def setup_state(state) + # closure + closure = [] + queued = {} + items = state.kernels.dup + + items.each do |item| + queued[item.rule_id] = true if item.position == 0 + end + + while (item = items.shift) do + if (sym = item.next_sym) && sym.nterm? + @grammar.find_rules_by_symbol!(sym).each do |rule| + next if queued[rule.id] + i = State::Item.new(rule: rule, position: 0) + closure << i + items << i + queued[i.rule_id] = true + end + end + end + + state.closure = closure.sort_by {|i| i.rule.id } + + # Trace + @tracer.trace_closure(state) + + # shift & reduce + state.compute_transitions_and_reduces + end + + # @rbs (Array[State] states, State state) -> void + def enqueue_state(states, state) + # Trace + @tracer.trace_state_list_append(@states.count, state) + + states << state + end + + # @rbs () -> void + def compute_lr0_states + # State queue + states = [] + states_created = {} + + state, _ = create_state(symbols.first, [State::Item.new(rule: @grammar.rules.first, position: 0)], states_created) + enqueue_state(states, state) + + while (state = states.shift) do + # Trace + @tracer.trace_state(state) + + setup_state(state) + + # `State#transitions` can not be used here + # because `items_to_state` of the `state` is not set yet. + state._transitions.each do |next_sym, to_items| + new_state, created = create_state(next_sym, to_items, states_created) + state.set_items_to_state(to_items, new_state) + state.set_lane_items(next_sym, new_state) + enqueue_state(states, new_state) if created + end + end + end + + # @rbs () -> Array[State::Action::Goto] + def nterm_transitions + a = [] + + @states.each do |state| + state.nterm_transitions.each do |goto| + a << goto + end + end + + a + end + + # @rbs () -> void + def compute_look_ahead_sets + report_duration(:compute_direct_read_sets) { compute_direct_read_sets } + report_duration(:compute_reads_relation) { compute_reads_relation } + report_duration(:compute_read_sets) { compute_read_sets } + report_duration(:compute_includes_relation) { compute_includes_relation } + report_duration(:compute_lookback_relation) { compute_lookback_relation } + report_duration(:compute_follow_sets) { compute_follow_sets } + report_duration(:compute_la) { compute_la } + end + + # @rbs () -> void + def compute_direct_read_sets + @states.each do |state| + state.nterm_transitions.each do |goto| + ary = goto.to_state.term_transitions.map do |shift| + shift.next_sym.number + end + + @direct_read_sets[goto] = Bitmap.from_array(ary) + end + end + end + + # @rbs () -> void + def compute_reads_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + goto.to_state.nterm_transitions.each do |goto2| + nterm2 = goto2.next_sym + if nterm2.nullable + @reads_relation[goto] ||= [] + @reads_relation[goto] << goto2 + end + end + end + end + end + + # @rbs () -> void + def compute_read_sets + @read_sets = Digraph.new(nterm_transitions, @reads_relation, @direct_read_sets).compute + end + + # Execute transition of state by symbols + # then return final state. + # + # @rbs (State state, Array[Grammar::Symbol] symbols) -> State + def transition(state, symbols) + symbols.each do |sym| + state = state.transition(sym) + end + + state + end + + # @rbs () -> void + def compute_includes_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + nterm = goto.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + i = rule.rhs.count - 1 + + while (i > -1) do + sym = rule.rhs[i] + + break if sym.term? + state2 = transition(state, rule.rhs[0...i]) + # p' = state, B = nterm, p = state2, A = sym + key = state2.nterm_transitions.find do |goto2| + goto2.next_sym.token_id == sym.token_id + end || (raise "Goto by #{sym.name} on state #{state2.id} is not found") + # TODO: need to omit if state == state2 ? + @includes_relation[key] ||= [] + @includes_relation[key] << goto + break unless sym.nullable + i -= 1 + end + end + end + end + end + + # @rbs () -> void + def compute_lookback_relation + @states.each do |state| + state.nterm_transitions.each do |goto| + nterm = goto.next_sym + @grammar.find_rules_by_symbol!(nterm).each do |rule| + state2 = transition(state, rule.rhs) + # p = state, A = nterm, q = state2, A -> ω = rule + @lookback_relation[state2.id] ||= {} + @lookback_relation[state2.id][rule.id] ||= [] + @lookback_relation[state2.id][rule.id] << goto + end + end + end + end + + # @rbs () -> void + def compute_follow_sets + @follow_sets = Digraph.new(nterm_transitions, @includes_relation, @read_sets).compute + end + + # @rbs () -> void + def compute_la + @states.each do |state| + lookback_relation_on_state = @lookback_relation[state.id] + next unless lookback_relation_on_state + rules.each do |rule| + ary = lookback_relation_on_state[rule.id] + next unless ary + + ary.each do |goto| + # q = state, A -> ω = rule, p = state2, A = nterm + follows = @follow_sets[goto] + + next if follows == 0 + + @la[state.id] ||= {} + @la[state.id][rule.id] ||= 0 + look_ahead = @la[state.id][rule.id] | follows + @la[state.id][rule.id] |= look_ahead + + # No risk of conflict when + # * the state only has single reduce + # * the state only has nterm_transitions (GOTO) + next if state.reduces.count == 1 && state.term_transitions.count == 0 + + state.set_look_ahead(rule, bitmap_to_terms(look_ahead)) + end + end + end + end + + # @rbs (Bitmap::bitmap bit) -> Array[Grammar::Symbol] + def bitmap_to_terms(bit) + ary = Bitmap.to_array(bit) + ary.map do |i| + @grammar.find_symbol_by_number!(i) + end + end + + # @rbs () -> void + def compute_conflicts(lr_type) + compute_shift_reduce_conflicts(lr_type) + compute_reduce_reduce_conflicts + end + + # @rbs () -> void + def compute_shift_reduce_conflicts(lr_type) + states.each do |state| + state.term_transitions.each do |shift| + state.reduces.each do |reduce| + sym = shift.next_sym + + next unless reduce.look_ahead + next unless reduce.look_ahead.include?(sym) + + # Shift/Reduce conflict + shift_prec = sym.precedence + reduce_prec = reduce.item.rule.precedence + + # Can resolve only when both have prec + unless shift_prec && reduce_prec + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + end + + case + when shift_prec < reduce_prec + # Reduce is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when shift_prec > reduce_prec + # Shift is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + end + + # shift_prec == reduce_prec, then check associativity + case sym.precedence.type + when :precedence + # Can not resolve the conflict + # + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next + when :right + # Shift is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :shift, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when :left + # Reduce is selected + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :reduce, resolved_by_precedence: true) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + next + when :nonassoc + # The conflict is resolved + # + # %nonassoc creates "run-time" error by removing both shift and reduce from + # the state. This makes the state to get syntax error if the conflicted token appears. + # On the other hand, %precedence creates "compile-time" error by keeping both + # shift and reduce on the state. This makes the state to be conflicted on the token. + # + # https://www.gnu.org/software/bison/manual/html_node/Using-Precedence.html + resolved_conflict = State::ResolvedConflict.new(state: state, symbol: sym, reduce: reduce, which: :error, resolved_by_precedence: false) + state.resolved_conflicts << resolved_conflict + shift.not_selected = true + reduce.add_not_selected_symbol(sym) + mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + else + raise "Unknown precedence type. #{sym}" + end + end + end + end + end + + # @rbs (Grammar::Precedence shift_prec, Grammar::Precedence reduce_prec, State::ResolvedConflict resolved_conflict) -> void + def mark_precedences_used(lr_type, shift_prec, reduce_prec, resolved_conflict) + case lr_type + when :lalr + shift_prec.mark_used_by_lalr(resolved_conflict) + reduce_prec.mark_used_by_lalr(resolved_conflict) + when :ielr + shift_prec.mark_used_by_ielr(resolved_conflict) + reduce_prec.mark_used_by_ielr(resolved_conflict) + end + end + + # @rbs () -> void + def compute_reduce_reduce_conflicts + states.each do |state| + state.reduces.combination(2) do |reduce1, reduce2| + next if reduce1.look_ahead.nil? || reduce2.look_ahead.nil? + + intersection = reduce1.look_ahead & reduce2.look_ahead + + unless intersection.empty? + state.conflicts << State::ReduceReduceConflict.new(symbols: intersection, reduce1: reduce1, reduce2: reduce2) + end + end + end + end + + # @rbs () -> void + def compute_default_reduction + states.each do |state| + next if state.reduces.empty? + # Do not set, if conflict exist + next unless state.conflicts.empty? + # Do not set, if shift with `error` exists. + next if state.term_transitions.map {|shift| shift.next_sym }.include?(@grammar.error_symbol) + + state.default_reduction_rule = state.reduces.map do |r| + [r.rule, r.rule.id, (r.look_ahead || []).count] + end.min_by do |rule, rule_id, count| + [-count, rule_id] + end.first + end + end + + # @rbs () -> void + def clear_conflicts + states.each(&:clear_conflicts) + end + + # Definition 3.15 (Predecessors) + # + # @rbs () -> void + def compute_predecessors + @states.each do |state| + state.transitions.each do |transition| + transition.to_state.append_predecessor(state) + end + end + end + + # Definition 3.16 (follow_kernel_items) + # + # @rbs () -> void + def compute_follow_kernel_items + set = nterm_transitions + relation = compute_goto_internal_relation + base_function = compute_goto_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, follow_kernel_items| + state = goto.from_state + state.follow_kernel_items[goto] = state.kernels.map {|kernel| + [kernel, Bitmap.to_bool_array(follow_kernel_items, state.kernels.count)] + }.to_h + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_goto_bitmaps + nterm_transitions.map {|goto| + bools = goto.from_state.kernels.map.with_index {|kernel, i| i if kernel.next_sym == goto.next_sym && kernel.symbols_after_transition.all?(&:nullable) }.compact + [goto, Bitmap.from_array(bools)] + }.to_h + end + + # Definition 3.20 (always_follows, one closure) + # + # @rbs () -> void + def compute_always_follows + set = nterm_transitions + relation = compute_goto_successor_or_internal_relation + base_function = compute_transition_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, always_follows_bitmap| + goto.from_state.always_follows[goto] = bitmap_to_terms(always_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_successor_or_internal_relation + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.successor_dependencies(goto) + state.internal_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_transition_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.to_state.term_transitions.map {|shift| shift.next_sym.number })] + }.to_h + end + + # Definition 3.24 (goto_follows, via always_follows) + # + # @rbs () -> void + def compute_goto_follows + set = nterm_transitions + relation = compute_goto_internal_or_predecessor_dependencies + base_function = compute_always_follows_bitmaps + Digraph.new(set, relation, base_function).compute.each do |goto, goto_follows_bitmap| + goto.from_state.goto_follows[goto] = bitmap_to_terms(goto_follows_bitmap) + end + end + + # @rbs () -> Hash[State::Action::Goto, Array[State::Action::Goto]] + def compute_goto_internal_or_predecessor_dependencies + relations = {} + + @states.each do |state| + state.nterm_transitions.each do |goto| + relations[goto] = state.internal_dependencies(goto) + state.predecessor_dependencies(goto) + end + end + + relations + end + + # @rbs () -> Hash[State::Action::Goto, Bitmap::bitmap] + def compute_always_follows_bitmaps + nterm_transitions.map {|goto| + [goto, Bitmap.from_array(goto.from_state.always_follows[goto].map(&:number))] + }.to_h + end + + # @rbs () -> void + def split_states + @states.each do |state| + state.transitions.each do |transition| + compute_state(state, transition, transition.to_state) + end + end + end + + # @rbs () -> void + def compute_inadequacy_annotations + @states.each do |state| + state.annotate_manifestation + end + + queue = @states.reject {|state| state.annotation_list.empty? } + + while (curr = queue.shift) do + curr.predecessors.each do |pred| + cache = pred.annotation_list.dup + curr.annotate_predecessor(pred) + queue << pred if cache != pred.annotation_list && !queue.include?(pred) + end + end + end + + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> void + def merge_lookaheads(state, filtered_lookaheads) + return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? } + + state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 } + state.transitions.each do |transition| + next if transition.to_state.lookaheads_recomputed + compute_state(state, transition, transition.to_state) + end + end + + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void + def compute_state(state, transition, next_state) + propagating_lookaheads = state.propagate_lookaheads(next_state) + s = next_state.ielr_isocores.find {|st| st.is_compatible?(propagating_lookaheads) } + + if s.nil? + s = next_state.lalr_isocore + new_state = State.new(@states.count, s.accessing_symbol, s.kernels) + new_state.closure = s.closure + new_state.compute_transitions_and_reduces + s.transitions.each do |transition| + new_state.set_items_to_state(transition.to_items, transition.to_state) + end + @states << new_state + new_state.lalr_isocore = s + s.ielr_isocores << new_state + s.ielr_isocores.each do |st| + st.ielr_isocores = s.ielr_isocores + end + new_state.lookaheads_recomputed = true + new_state.item_lookahead_set = propagating_lookaheads + state.update_transition(transition, new_state) + elsif(!s.lookaheads_recomputed) + s.lookaheads_recomputed = true + s.item_lookahead_set = propagating_lookaheads + else + merge_lookaheads(s, propagating_lookaheads) + state.update_transition(transition, s) if state.items_to_state[transition.to_items].id != s.id + end + end + + # @rbs (Logger logger) -> void + def validate_conflicts_within_threshold!(logger) + exit false unless conflicts_within_threshold?(logger) + end + + # @rbs (Logger logger) -> bool + def conflicts_within_threshold?(logger) + return true unless @grammar.expect + + [sr_conflicts_within_threshold?(logger), rr_conflicts_within_threshold?(logger)].all? + end + + # @rbs (Logger logger) -> bool + def sr_conflicts_within_threshold?(logger) + return true if @grammar.expect == sr_conflicts_count + + logger.error("shift/reduce conflicts: #{sr_conflicts_count} found, #{@grammar.expect} expected") + false + end + + # @rbs (Logger logger) -> bool + def rr_conflicts_within_threshold?(logger, expected: 0) + return true if expected == rr_conflicts_count + + logger.error("reduce/reduce conflicts: #{rr_conflicts_count} found, #{expected} expected") + false + end + + # @rbs () -> void + def clear_look_ahead_sets + @direct_read_sets.clear + @reads_relation.clear + @read_sets.clear + @includes_relation.clear + @lookback_relation.clear + @follow_sets.clear + @la.clear + + @_direct_read_sets = nil + @_read_sets = nil + @_follow_sets = nil + @_la = nil + end + end +end diff --git a/tool/lrama/lib/lrama/tracer.rb b/tool/lrama/lib/lrama/tracer.rb new file mode 100644 index 0000000000..fda699a665 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer.rb @@ -0,0 +1,51 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative "tracer/actions" +require_relative "tracer/closure" +require_relative "tracer/duration" +require_relative "tracer/only_explicit_rules" +require_relative "tracer/rules" +require_relative "tracer/state" + +module Lrama + class Tracer + # @rbs (IO io, **bool options) -> void + def initialize(io, **options) + @io = io + @options = options + @only_explicit_rules = OnlyExplicitRules.new(io, **options) + @rules = Rules.new(io, **options) + @actions = Actions.new(io, **options) + @closure = Closure.new(io, **options) + @state = State.new(io, **options) + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + @only_explicit_rules.trace(grammar) + @rules.trace(grammar) + @actions.trace(grammar) + end + + # @rbs (Lrama::State state) -> void + def trace_closure(state) + @closure.trace(state) + end + + # @rbs (Lrama::State state) -> void + def trace_state(state) + @state.trace(state) + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_state_list_append(state_count, state) + @state.trace_list_append(state_count, state) + end + + # @rbs () -> void + def enable_duration + Duration.enable if @options[:time] + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/actions.rb b/tool/lrama/lib/lrama/tracer/actions.rb new file mode 100644 index 0000000000..7b9c9b9f53 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/actions.rb @@ -0,0 +1,22 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Actions + # @rbs (IO io, ?actions: bool, **bool options) -> void + def initialize(io, actions: false, **options) + @io = io + @actions = actions + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @actions + + @io << "Grammar rules with actions:" << "\n" + grammar.rules.each { |rule| @io << rule.with_actions << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/closure.rb b/tool/lrama/lib/lrama/tracer/closure.rb new file mode 100644 index 0000000000..5b2f0b27e6 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/closure.rb @@ -0,0 +1,30 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Closure + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @closure = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @closure + + @io << "Closure: input" << "\n" + state.kernels.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + @io << "Closure: output" << "\n" + state.items.each do |item| + @io << " #{item.display_rest}" << "\n" + end + @io << "\n\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/duration.rb b/tool/lrama/lib/lrama/tracer/duration.rb new file mode 100644 index 0000000000..91c49625b2 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/duration.rb @@ -0,0 +1,38 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + module Duration + # TODO: rbs-inline 0.11.0 doesn't support instance variables. + # Move these type declarations above instance variable definitions, once it's supported. + # see: https://github.com/soutaro/rbs-inline/pull/149 + # + # @rbs! + # @_report_duration_enabled: bool + + # @rbs () -> void + def self.enable + @_report_duration_enabled = true + end + + # @rbs () -> bool + def self.enabled? + !!@_report_duration_enabled + end + + # @rbs [T] (_ToS message) { -> T } -> T + def report_duration(message) + time1 = Time.now.to_f + result = yield + time2 = Time.now.to_f + + if Duration.enabled? + STDERR.puts sprintf("%s %10.5f s", message, time2 - time1) + end + + return result + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb new file mode 100644 index 0000000000..4f64e7d2f4 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/only_explicit_rules.rb @@ -0,0 +1,24 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class OnlyExplicitRules + # @rbs (IO io, ?only_explicit: bool, **bool) -> void + def initialize(io, only_explicit: false, **_) + @io = io + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return unless @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each do |rule| + @io << rule.display_name_without_action << "\n" if rule.lhs.first_set.any? + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/rules.rb b/tool/lrama/lib/lrama/tracer/rules.rb new file mode 100644 index 0000000000..d6e85b8432 --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class Rules + # @rbs (IO io, ?rules: bool, ?only_explicit: bool, **bool) -> void + def initialize(io, rules: false, only_explicit: false, **_) + @io = io + @rules = rules + @only_explicit = only_explicit + end + + # @rbs (Lrama::Grammar grammar) -> void + def trace(grammar) + return if !@rules || @only_explicit + + @io << "Grammar rules:" << "\n" + grammar.rules.each { |rule| @io << rule.display_name << "\n" } + end + end + end +end diff --git a/tool/lrama/lib/lrama/tracer/state.rb b/tool/lrama/lib/lrama/tracer/state.rb new file mode 100644 index 0000000000..21c0047f8e --- /dev/null +++ b/tool/lrama/lib/lrama/tracer/state.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Tracer + class State + # @rbs (IO io, ?automaton: bool, ?closure: bool, **bool) -> void + def initialize(io, automaton: false, closure: false, **_) + @io = io + @state = automaton || closure + end + + # @rbs (Lrama::State state) -> void + def trace(state) + return unless @state + + # Bison 3.8.2 renders "(reached by "end-of-input")" for State 0 but + # I think it is not correct... + previous = state.kernels.first.previous_sym + @io << "Processing state #{state.id} (reached by #{previous.display_name})" << "\n" + end + + # @rbs (Integer state_count, Lrama::State state) -> void + def trace_list_append(state_count, state) + return unless @state + + previous = state.kernels.first.previous_sym + @io << sprintf("state_list_append (state = %d, symbol = %d (%s))", + state_count, previous.number, previous.display_name) << "\n" + end + end + end +end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb new file mode 100644 index 0000000000..d649b74939 --- /dev/null +++ b/tool/lrama/lib/lrama/version.rb @@ -0,0 +1,6 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + VERSION = "0.7.1".freeze #: String +end diff --git a/tool/lrama/lib/lrama/warnings.rb b/tool/lrama/lib/lrama/warnings.rb new file mode 100644 index 0000000000..52f09144ef --- /dev/null +++ b/tool/lrama/lib/lrama/warnings.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require_relative 'warnings/conflicts' +require_relative 'warnings/implicit_empty' +require_relative 'warnings/name_conflicts' +require_relative 'warnings/redefined_rules' +require_relative 'warnings/required' +require_relative 'warnings/useless_precedence' + +module Lrama + class Warnings + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @conflicts = Conflicts.new(logger, warnings) + @implicit_empty = ImplicitEmpty.new(logger, warnings) + @name_conflicts = NameConflicts.new(logger, warnings) + @redefined_rules = RedefinedRules.new(logger, warnings) + @required = Required.new(logger, warnings) + @useless_precedence = UselessPrecedence.new(logger, warnings) + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + @conflicts.warn(states) + @implicit_empty.warn(grammar) + @name_conflicts.warn(grammar) + @redefined_rules.warn(grammar) + @required.warn(grammar) + @useless_precedence.warn(grammar, states) + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/conflicts.rb b/tool/lrama/lib/lrama/warnings/conflicts.rb new file mode 100644 index 0000000000..6ba0de6f9c --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/conflicts.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Conflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::States states) -> void + def warn(states) + return unless @warnings + + if states.sr_conflicts_count != 0 + @logger.warn("shift/reduce conflicts: #{states.sr_conflicts_count} found") + end + + if states.rr_conflicts_count != 0 + @logger.warn("reduce/reduce conflicts: #{states.rr_conflicts_count} found") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/implicit_empty.rb b/tool/lrama/lib/lrama/warnings/implicit_empty.rb new file mode 100644 index 0000000000..ba81adca01 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/implicit_empty.rb @@ -0,0 +1,29 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Empty rules are easily overlooked and ambiguous + # - Empty alternatives like `rule: | "token";` can be missed during code reading + # - Difficult to distinguish between intentional empty rules vs. omissions + # - Explicit marking with %empty directive comment improves clarity + class ImplicitEmpty + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.rule_builders.each do |builder| + if builder.rhs.empty? + @logger.warn("warning: empty rule without %empty") + end + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/name_conflicts.rb b/tool/lrama/lib/lrama/warnings/name_conflicts.rb new file mode 100644 index 0000000000..c0754ab551 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/name_conflicts.rb @@ -0,0 +1,63 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + # Warning rationale: Parameterized rule names conflicting with symbol names + # - When a %rule name is identical to a terminal or non-terminal symbol name, + # it reduces grammar readability and may cause unintended behavior + # - Detecting these conflicts helps improve grammar definition quality + class NameConflicts + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return if grammar.parameterized_rules.empty? + + symbol_names = collect_symbol_names(grammar) + check_conflicts(grammar.parameterized_rules, symbol_names) + end + + private + + # @rbs (Lrama::Grammar grammar) -> Set[String] + def collect_symbol_names(grammar) + symbol_names = Set.new + + collect_term_names(grammar.terms, symbol_names) + collect_nterm_names(grammar.nterms, symbol_names) + + symbol_names + end + + # @rbs (Array[untyped] terms, Set[String] symbol_names) -> void + def collect_term_names(terms, symbol_names) + terms.each do |term| + symbol_names.add(term.id.s_value) + symbol_names.add(term.alias_name) if term.alias_name + end + end + + # @rbs (Array[untyped] nterms, Set[String] symbol_names) -> void + def collect_nterm_names(nterms, symbol_names) + nterms.each do |nterm| + symbol_names.add(nterm.id.s_value) + end + end + + # @rbs (Array[untyped] parameterized_rules, Set[String] symbol_names) -> void + def check_conflicts(parameterized_rules, symbol_names) + parameterized_rules.each do |param_rule| + next unless symbol_names.include?(param_rule.name) + + @logger.warn("warning: parameterized rule name \"#{param_rule.name}\" conflicts with symbol name") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/redefined_rules.rb b/tool/lrama/lib/lrama/warnings/redefined_rules.rb new file mode 100644 index 0000000000..8ac2f1f103 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/redefined_rules.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class RedefinedRules + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + grammar.parameterized_resolver.redefined_rules.each do |rule| + @logger.warn("parameterized rule redefined: #{rule}") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/required.rb b/tool/lrama/lib/lrama/warnings/required.rb new file mode 100644 index 0000000000..4ab1ed787e --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/required.rb @@ -0,0 +1,23 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class Required + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings = false, **_) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + + if grammar.required + @logger.warn("currently, %require is simply valid as a grammar but does nothing") + end + end + end + end +end diff --git a/tool/lrama/lib/lrama/warnings/useless_precedence.rb b/tool/lrama/lib/lrama/warnings/useless_precedence.rb new file mode 100644 index 0000000000..2913d6d7e5 --- /dev/null +++ b/tool/lrama/lib/lrama/warnings/useless_precedence.rb @@ -0,0 +1,25 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class UselessPrecedence + # @rbs (Lrama::Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void + def warn(grammar, states) + return unless @warnings + + grammar.precedences.each do |precedence| + unless precedence.used_by? + @logger.warn("Precedence #{precedence.s_value} (line: #{precedence.lineno}) is defined but not used in any rule.") + end + end + end + end + end +end diff --git a/tool/lrama/template/bison/_yacc.h b/tool/lrama/template/bison/_yacc.h new file mode 100644 index 0000000000..3e270c9171 --- /dev/null +++ b/tool/lrama/template/bison/_yacc.h @@ -0,0 +1,79 @@ +<%# b4_shared_declarations -%> + <%-# b4_cpp_guard_open([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#ifndef <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> +# define <%= output.b4_cpp_guard__b4_spec_mapped_header_file %> + <%- end -%> + <%-# b4_declare_yydebug & b4_YYDEBUG_define -%> +/* Debug traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif +#if YYDEBUG && !defined(yydebug) +extern int yydebug; +#endif +<%= output.percent_code("requires") %> + + <%-# b4_token_enums_defines -%> +/* Token kinds. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + enum yytokentype + { +<%= output.token_enums -%> + }; + typedef enum yytokentype yytoken_kind_t; +#endif + + <%-# b4_declare_yylstype -%> + <%-# b4_value_type_define -%> +/* Value type. */ +<% if output.grammar.union %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +union YYSTYPE +{ +#line <%= output.grammar.union.lineno %> "<%= output.grammar_file_path %>" +<%= output.grammar.union.braces_less_code %> +#line [@oline@] [@ofile@] + +}; +typedef union YYSTYPE YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% else %> +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef int YYSTYPE; +# define YYSTYPE_IS_TRIVIAL 1 +# define YYSTYPE_IS_DECLARED 1 +#endif +<% end %> + + <%-# b4_location_type_define -%> +/* Location type. */ +#if ! defined YYLTYPE && ! defined YYLTYPE_IS_DECLARED +typedef struct YYLTYPE YYLTYPE; +struct YYLTYPE +{ + int first_line; + int first_column; + int last_line; + int last_column; +}; +# define YYLTYPE_IS_DECLARED 1 +# define YYLTYPE_IS_TRIVIAL 1 +#endif + + + + + <%-# b4_declare_yyerror_and_yylex. Not supported -%> + <%-# b4_declare_yyparse -%> +int yyparse (<%= output.parse_param %>); + + +<%= output.percent_code("provides") %> + <%-# b4_cpp_guard_close([b4_spec_mapped_header_file]) -%> + <%- if output.spec_mapped_header_file -%> +#endif /* !<%= output.b4_cpp_guard__b4_spec_mapped_header_file %> */ + <%- end -%> diff --git a/tool/lrama/template/bison/yacc.c b/tool/lrama/template/bison/yacc.c new file mode 100644 index 0000000000..6edd59a0d5 --- /dev/null +++ b/tool/lrama/template/bison/yacc.c @@ -0,0 +1,2068 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison implementation for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* C LALR(1) parser skeleton written by Richard Stallman, by + simplifying the original so-called "semantic" parser. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + +/* All symbols defined below should begin with yy or YY, to avoid + infringing on user name space. This should be done even for local + variables, as they might otherwise be expanded by user macros. + There are some unavoidable exceptions within include files to + define necessary library symbols; they are noted "INFRINGES ON + USER NAME SPACE" below. */ + +<%# b4_identification -%> +/* Identify Bison output, and Bison version. */ +#define YYBISON 30802 + +/* Bison version string. */ +#define YYBISON_VERSION "3.8.2" + +/* Skeleton name. */ +#define YYSKELETON_NAME "<%= output.template_basename %>" + +/* Pure parsers. */ +#define YYPURE 1 + +/* Push parsers. */ +#define YYPUSH 0 + +/* Pull parsers. */ +#define YYPULL 1 + + +<%# b4_user_pre_prologue -%> +<%- if output.aux.prologue -%> +/* First part of user prologue. */ +#line <%= output.aux.prologue_first_lineno %> "<%= output.grammar_file_path %>" +<%= output.aux.prologue %> +#line [@oline@] [@ofile@] +<%- end -%> + +<%# b4_cast_define -%> +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast<Type> (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast<Type> (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif +<%# b4_null_define -%> +# ifndef YY_NULLPTR +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif +# else +# define YY_NULLPTR ((void*)0) +# endif +# endif + +<%# b4_header_include_if -%> +<%- if output.include_header -%> +#include "<%= output.include_header %>" +<%- else -%> +/* Use api.header.include to #include this header + instead of duplicating it here. */ +<%= output.render_partial("bison/_yacc.h") %> +<%- end -%> +<%# b4_declare_symbol_enum -%> +/* Symbol kind. */ +enum yysymbol_kind_t +{ +<%= output.symbol_enum -%> +}; +typedef enum yysymbol_kind_t yysymbol_kind_t; + + + + +<%# b4_user_post_prologue -%> +<%# b4_c99_int_type_define -%> +#ifdef short +# undef short +#endif + +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + <limits.h> and (if available) <stdint.h> are included + so that the code can choose integer types of a good width. */ + +#ifndef __PTRDIFF_MAX__ +# include <limits.h> /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stdint.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif + +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ + +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; +#endif + +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; +#else +typedef short yytype_int16; +#endif + +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + <https://h20195.www2.hpe.com/V2/getpdf.aspx/4AA4-7673ENW.pdf>. */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 +#endif + +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; +#else +typedef short yytype_uint8; +#endif + +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; +#else +typedef int yytype_uint16; +#endif + +<%# b4_sizes_types_define -%> +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif +#endif + +#ifndef YYSIZE_T +# ifdef __SIZE_TYPE__ +# define YYSIZE_T __SIZE_TYPE__ +# elif defined size_t +# define YYSIZE_T size_t +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include <stddef.h> /* INFRINGES ON USER NAME SPACE */ +# define YYSIZE_T size_t +# else +# define YYSIZE_T unsigned +# endif +#endif + +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef <%= output.int_type_for([output.yynstates - 1]) %> yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include <libintl.h> /* INFRINGES ON USER NAME SPACE */ +# define YY_(Msgid) dgettext ("bison-runtime", Msgid) +# endif +# endif +# ifndef YY_ +# define YY_(Msgid) Msgid +# endif +#endif + + +<%# b4_attribute_define -%> +#ifndef YY_ATTRIBUTE_PURE +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif +#endif + +#ifndef YY_ATTRIBUTE_UNUSED +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) +# else +# define YY_ATTRIBUTE_UNUSED +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YY_USE(E) ((void) (E)) +#else +# define YY_USE(E) /* empty */ +#endif + +/* Suppress an incorrect diagnostic about yylval being uninitialized. */ +#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__ +# if __GNUC__ * 100 + __GNUC_MINOR__ < 407 +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") +# else +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ + _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") +# endif +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ + _Pragma ("GCC diagnostic pop") +#else +# define YY_INITIAL_VALUE(Value) Value +#endif +#ifndef YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +# define YY_IGNORE_MAYBE_UNINITIALIZED_END +#endif +#ifndef YY_INITIAL_VALUE +# define YY_INITIAL_VALUE(Value) /* Nothing. */ +#endif + +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + + +#define YY_ASSERT(E) ((void) (0 && (E))) + +#if 1 + +/* The parser invokes alloca or malloc; define the necessary symbols. */ + +# ifdef YYSTACK_USE_ALLOCA +# if YYSTACK_USE_ALLOCA +# ifdef __GNUC__ +# define YYSTACK_ALLOC __builtin_alloca +# elif defined __BUILTIN_VA_ARG_INCR +# include <alloca.h> /* INFRINGES ON USER NAME SPACE */ +# elif defined _AIX +# define YYSTACK_ALLOC __alloca +# elif defined _MSC_VER +# include <malloc.h> /* INFRINGES ON USER NAME SPACE */ +# define alloca _alloca +# else +# define YYSTACK_ALLOC alloca +# if ! defined _ALLOCA_H && ! defined EXIT_SUCCESS +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ + /* Use EXIT_SUCCESS as a witness for stdlib.h. */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# endif +# endif +# endif + +# ifdef YYSTACK_ALLOC + /* Pacify GCC's 'empty if-body' warning. */ +# define YYSTACK_FREE(Ptr) do { /* empty */; } while (0) +# ifndef YYSTACK_ALLOC_MAXIMUM + /* The OS might guarantee only one guard page at the bottom of the stack, + and a page size can be as small as 4096 bytes. So we cannot safely + invoke alloca (N) if N exceeds 4096. Use a slightly smaller number + to allow for a few compiler-allocated temporary stack slots. */ +# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +# endif +# else +# define YYSTACK_ALLOC YYMALLOC +# define YYSTACK_FREE YYFREE +# ifndef YYSTACK_ALLOC_MAXIMUM +# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +# endif +# if (defined __cplusplus && ! defined EXIT_SUCCESS \ + && ! ((defined YYMALLOC || defined malloc) \ + && (defined YYFREE || defined free))) +# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */ +# ifndef EXIT_SUCCESS +# define EXIT_SUCCESS 0 +# endif +# endif +# ifndef YYMALLOC +# define YYMALLOC malloc +# if ! defined malloc && ! defined EXIT_SUCCESS +void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# ifndef YYFREE +# define YYFREE free +# if ! defined free && ! defined EXIT_SUCCESS +void free (void *); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# endif +#endif /* 1 */ + +#if (! defined yyoverflow \ + && (! defined __cplusplus \ + || (defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL \ + && defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) + +/* A type that is properly aligned for any stack member. */ +union yyalloc +{ + yy_state_t yyss_alloc; + YYSTYPE yyvs_alloc; + YYLTYPE yyls_alloc; +}; + +/* The size of the maximum gap between one aligned stack and the next. */ +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) + +/* The size of an array large to enough to hold all stacks, each with + N elements. */ +# define YYSTACK_BYTES(N) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE) \ + + YYSIZEOF (YYLTYPE)) \ + + 2 * YYSTACK_GAP_MAXIMUM) + +# define YYCOPY_NEEDED 1 + +/* Relocate STACK from its old location to the new one. The + local variables YYSIZE and YYSTACKSIZE give the old and new number of + elements in the stack, and YYPTR gives the new location of the + stack. Advance YYPTR to a properly aligned location for the next + stack. */ +# define YYSTACK_RELOCATE(Stack_alloc, Stack) \ + do \ + { \ + YYPTRDIFF_T yynewbytes; \ + YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ + Stack = &yyptr->Stack_alloc; \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ + } \ + while (0) + +#endif + +#if defined YYCOPY_NEEDED && YYCOPY_NEEDED +/* Copy COUNT objects from SRC to DST. The source and destination do + not overlap. */ +# ifndef YYCOPY +# if defined __GNUC__ && 1 < __GNUC__ +# define YYCOPY(Dst, Src, Count) \ + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) +# else +# define YYCOPY(Dst, Src, Count) \ + do \ + { \ + YYPTRDIFF_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (Dst)[yyi] = (Src)[yyi]; \ + } \ + while (0) +# endif +# endif +#endif /* !YYCOPY_NEEDED */ + +/* YYFINAL -- State number of the termination state. */ +#define YYFINAL <%= output.yyfinal %> +/* YYLAST -- Last index in YYTABLE. */ +#define YYLAST <%= output.yylast %> + +/* YYNTOKENS -- Number of terminals. */ +#define YYNTOKENS <%= output.yyntokens %> +/* YYNNTS -- Number of nonterminals. */ +#define YYNNTS <%= output.yynnts %> +/* YYNRULES -- Number of rules. */ +#define YYNRULES <%= output.yynrules %> +/* YYNSTATES -- Number of states. */ +#define YYNSTATES <%= output.yynstates %> + +/* YYMAXUTOK -- Last valid token kind. */ +#define YYMAXUTOK <%= output.yymaxutok %> + + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) + +/* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM + as returned by yylex. */ +static const <%= output.int_type_for(output.context.yytranslate) %> yytranslate[] = +{ +<%= output.yytranslate %> +}; + +<%- if output.error_recovery -%> +/* YYTRANSLATE_INVERTED[SYMBOL-NUM] -- Token number corresponding to SYMBOL-NUM */ +static const <%= output.int_type_for(output.context.yytranslate_inverted) %> yytranslate_inverted[] = +{ +<%= output.yytranslate_inverted %> +}; +<%- end -%> +#if YYDEBUG +/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ +static const <%= output.int_type_for(output.context.yyrline) %> yyrline[] = +{ +<%= output.yyrline %> +}; +#endif + +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if 1 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + +/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. + First, the terminals, then, starting at YYNTOKENS, nonterminals. */ +static const char *const yytname[] = +{ +<%= output.yytname %> +}; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} +#endif + +#define YYPACT_NINF (<%= output.yypact_ninf %>) + +#define yypact_value_is_default(Yyn) \ + <%= output.table_value_equals(output.context.yypact, "Yyn", output.yypact_ninf, "YYPACT_NINF") %> + +#define YYTABLE_NINF (<%= output.yytable_ninf %>) + +#define yytable_value_is_error(Yyn) \ + <%= output.table_value_equals(output.context.yytable, "Yyn", output.yytable_ninf, "YYTABLE_NINF") %> + +<%# b4_parser_tables_define -%> +/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + STATE-NUM. */ +static const <%= output.int_type_for(output.context.yypact) %> yypact[] = +{ +<%= output.int_array_to_string(output.context.yypact) %> +}; + +/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. + Performed when YYTABLE does not specify something else to do. Zero + means the default is an error. */ +static const <%= output.int_type_for(output.context.yydefact) %> yydefact[] = +{ +<%= output.int_array_to_string(output.context.yydefact) %> +}; + +/* YYPGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yypgoto) %> yypgoto[] = +{ +<%= output.int_array_to_string(output.context.yypgoto) %> +}; + +/* YYDEFGOTO[NTERM-NUM]. */ +static const <%= output.int_type_for(output.context.yydefgoto) %> yydefgoto[] = +{ +<%= output.int_array_to_string(output.context.yydefgoto) %> +}; + +/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If + positive, shift that token. If negative, reduce the rule whose + number is the opposite. If YYTABLE_NINF, syntax error. */ +static const <%= output.int_type_for(output.context.yytable) %> yytable[] = +{ +<%= output.int_array_to_string(output.context.yytable) %> +}; + +static const <%= output.int_type_for(output.context.yycheck) %> yycheck[] = +{ +<%= output.int_array_to_string(output.context.yycheck) %> +}; + +/* YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of + state STATE-NUM. */ +static const <%= output.int_type_for(output.context.yystos) %> yystos[] = +{ +<%= output.int_array_to_string(output.context.yystos) %> +}; + +/* YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr1) %> yyr1[] = +{ +<%= output.int_array_to_string(output.context.yyr1) %> +}; + +/* YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. */ +static const <%= output.int_type_for(output.context.yyr2) %> yyr2[] = +{ +<%= output.int_array_to_string(output.context.yyr2) %> +}; + + +enum { YYENOMEM = -2 }; + +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab +#define YYNOMEM goto yyexhaustedlab + + +#define YYRECOVERING() (!!yyerrstatus) + +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (<%= output.yyerror_args %>, YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF + +<%# b4_yylloc_default_define -%> +/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. + If N is 0, then set CURRENT to the empty location which ends + the previous symbol: RHS[0] (always defined). */ + +#ifndef YYLLOC_DEFAULT +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = \ + YYRHSLOC (Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC (Rhs, 0).last_column; \ + } \ + while (0) +#endif + +#define YYRHSLOC(Rhs, K) ((Rhs)[K]) + + +/* Enable debugging if requested. */ +#if YYDEBUG + +# ifndef YYFPRINTF +# include <stdio.h> /* INFRINGES ON USER NAME SPACE */ +# define YYFPRINTF fprintf +# endif + +# define YYDPRINTF(Args) \ +do { \ + if (yydebug) \ + YYFPRINTF Args; \ +} while (0) + + +<%# b4_yylocation_print_define -%> +/* YYLOCATION_PRINT -- Print the location on the stream. + This macro was not mandated originally: define only if we know + we won't break user code: when these are the locations we know. */ + +# ifndef YYLOCATION_PRINT + +# if defined YY_LOCATION_PRINT + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) YY_LOCATION_PRINT(File, *(Loc)<%= output.user_args %>) + +# elif defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + +/* Print *YYLOCP on YYO. Private, do not rely on its existence. */ + +YY_ATTRIBUTE_UNUSED +static int +yy_location_print_ (FILE *yyo, YYLTYPE const * const yylocp) +{ + int res = 0; + int end_col = 0 != yylocp->last_column ? yylocp->last_column - 1 : 0; + if (0 <= yylocp->first_line) + { + res += YYFPRINTF (yyo, "%d", yylocp->first_line); + if (0 <= yylocp->first_column) + res += YYFPRINTF (yyo, ".%d", yylocp->first_column); + } + if (0 <= yylocp->last_line) + { + if (yylocp->first_line < yylocp->last_line) + { + res += YYFPRINTF (yyo, "-%d", yylocp->last_line); + if (0 <= end_col) + res += YYFPRINTF (yyo, ".%d", end_col); + } + else if (0 <= end_col && yylocp->first_column < end_col) + res += YYFPRINTF (yyo, "-%d", end_col); + } + return res; +} + +# define YYLOCATION_PRINT yy_location_print_ + + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT(File, Loc<%= output.user_args %>) YYLOCATION_PRINT(File, &(Loc)<%= output.user_args %>) + +# else + +# define YYLOCATION_PRINT(File, Loc<%= output.user_args %>) ((void) 0) + /* Temporary convenience wrapper in case some people defined the + undocumented and private YY_LOCATION_PRINT macros. */ +# define YY_LOCATION_PRINT YYLOCATION_PRINT + +# endif +# endif /* !defined YYLOCATION_PRINT */ + + +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) \ +do { \ + if (yydebug) \ + { \ + YYFPRINTF (stderr, "%s ", Title); \ + yy_symbol_print (stderr, \ + Kind, Value, Location<%= output.user_args %>); \ + YYFPRINTF (stderr, "\n"); \ + } \ +} while (0) + + +<%# b4_yy_symbol_print_define -%> +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ + +static void +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + FILE *yyoutput = yyo; +<%= output.parse_param_use("yyoutput", "yylocationp") %> + if (!yyvaluep) + return; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +<%# b4_symbol_actions(printer) -%> +switch (yykind) + { +<%= output.symbol_actions_for_printer -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ + +static void +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, YYLTYPE const * const yylocationp<%= output.user_formals %>) +{ + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); + + YYLOCATION_PRINT (yyo, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ": "); + yy_symbol_value_print (yyo, yykind, yyvaluep, yylocationp<%= output.user_args %>); + YYFPRINTF (yyo, ")"); +} + +/*------------------------------------------------------------------. +| yy_stack_print -- Print the state stack from its BOTTOM up to its | +| TOP (included). | +`------------------------------------------------------------------*/ + +static void +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop<%= output.user_formals %>) +{ + YYFPRINTF (stderr, "Stack now"); + for (; yybottom <= yytop; yybottom++) + { + int yybot = *yybottom; + YYFPRINTF (stderr, " %d", yybot); + } + YYFPRINTF (stderr, "\n"); +} + +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_stack_print ((Bottom), (Top)<%= output.user_args %>); \ +} while (0) + + +/*------------------------------------------------. +| Report that the YYRULE is going to be reduced. | +`------------------------------------------------*/ + +static void +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, YYLTYPE *yylsp, + int yyrule<%= output.user_formals %>) +{ + int yylno = yyrline[yyrule]; + int yynrhs = yyr2[yyrule]; + int yyi; + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", + yyrule - 1, yylno); + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + YYFPRINTF (stderr, " $%d = ", yyi + 1); + yy_symbol_print (stderr, + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)], + &(yylsp[(yyi + 1) - (yynrhs)])<%= output.user_args %>); + YYFPRINTF (stderr, "\n"); + } +} + +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) \ +do { \ + if (yydebug) \ + yy_reduce_print (yyssp, yyvsp, yylsp, Rule<%= output.user_args %>); \ +} while (0) + +/* Nonzero means print parse trace. It is left uninitialized so that + multiple parsers can coexist. */ +#ifndef yydebug +int yydebug; +#endif +#else /* !YYDEBUG */ +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location<%= output.user_args %>) +# define YY_STACK_PRINT(Bottom, Top<%= output.user_args %>) +# define YY_REDUCE_PRINT(Rule<%= output.user_args %>) +#endif /* !YYDEBUG */ + + +/* YYINITDEPTH -- initial size of the parser's stacks. */ +#ifndef YYINITDEPTH +# define YYINITDEPTH 200 +#endif + +/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only + if the built-in stack extension method is used). + + Do not make this value too large; the results are undefined if + YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) + evaluated with infinite-precision integer arithmetic. */ + +#ifndef YYMAXDEPTH +# define YYMAXDEPTH 10000 +#endif + + +/* Context of a parse error. */ +typedef struct +{ + yy_state_t *yyssp; + yysymbol_kind_t yytoken; + YYLTYPE *yylloc; +} yypcontext_t; + +/* Put in YYARG at most YYARGN of the expected tokens given the + current YYCTX, and return the number of tokens stored in YYARG. If + YYARG is null, return the number of expected tokens (guaranteed to + be less than YYNTOKENS). Return YYENOMEM on memory exhaustion. + Return 0 if there are more than YYARGN expected tokens, yet fill + YYARG up to YYARGN. */ +static int +yypcontext_expected_tokens (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + int yyn = yypact[+*yyctx->yyssp]; + if (!yypact_value_is_default (yyn)) + { + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. In other words, skip the first -YYN actions for + this state because they are default actions. */ + int yyxbegin = yyn < 0 ? -yyn : 0; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYSYMBOL_YYerror + && !yytable_value_is_error (yytable[yyx + yyn])) + { + if (!yyarg) + ++yycount; + else if (yycount == yyargn) + return 0; + else + yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx); + } + } + if (yyarg && yycount == 0 && 0 < yyargn) + yyarg[0] = YYSYMBOL_YYEMPTY; + return yycount; +} + + + + +#ifndef yystrlen +# if defined __GLIBC__ && defined _STRING_H +# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S))) +# else +/* Return the length of YYSTR. */ +static YYPTRDIFF_T +yystrlen (const char *yystr) +{ + YYPTRDIFF_T yylen; + for (yylen = 0; yystr[yylen]; yylen++) + continue; + return yylen; +} +# endif +#endif + +#ifndef yystpcpy +# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +# define yystpcpy stpcpy +# else +/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in + YYDEST. */ +static char * +yystpcpy (char *yydest, const char *yysrc) +{ + char *yyd = yydest; + const char *yys = yysrc; + + while ((*yyd++ = *yys++) != '\0') + continue; + + return yyd - 1; +} +# endif +#endif + +#ifndef yytnamerr +/* Copy to YYRES the contents of YYSTR after stripping away unnecessary + quotes and backslashes, so that it's suitable for yyerror. The + heuristic is that double-quoting is unnecessary unless the string + contains an apostrophe, a comma, or backslash (other than + backslash-backslash). YYSTR is taken from yytname. If YYRES is + null, do not copy; instead, return the length of what the result + would have been. */ +static YYPTRDIFF_T +yytnamerr (char *yyres, const char *yystr) +{ + if (*yystr == '"') + { + YYPTRDIFF_T yyn = 0; + char const *yyp = yystr; + for (;;) + switch (*++yyp) + { + case '\'': + case ',': + goto do_not_strip_quotes; + + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; + else + goto append; + + append: + default: + if (yyres) + yyres[yyn] = *yyp; + yyn++; + break; + + case '"': + if (yyres) + yyres[yyn] = '\0'; + return yyn; + } + do_not_strip_quotes: ; + } + + if (yyres) + return yystpcpy (yyres, yystr) - yyres; + else + return yystrlen (yystr); +} +#endif + + +static int +yy_syntax_error_arguments (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + /* There are many possibilities here to consider: + - If this state is a consistent state with a default action, then + the only way this function was invoked is if the default action + is an error action. In that case, don't check for expected + tokens because there are none. + - The only way there can be no lookahead present (in yychar) is if + this state is a consistent state with a default action. Thus, + detecting the absence of a lookahead is sufficient to determine + that there is no unexpected or expected token to report. In that + case, just report a simple "syntax error". + - Don't assume there isn't a lookahead just because this state is a + consistent state with a default action. There might have been a + previous inconsistent state, consistent state with a non-default + action, or user semantic action that manipulated yychar. + - Of course, the expected token list depends on states to have + correct lookahead information, and it depends on the parser not + to perform extra reductions after fetching a lookahead from the + scanner and before detecting a syntax error. Thus, state merging + (from LALR or IELR) and default reductions corrupt the expected + token list. However, the list is correct for canonical LR with + one exception: it will still contain any token that will not be + accepted due to an error action in a later state. + */ + if (yyctx->yytoken != YYSYMBOL_YYEMPTY) + { + int yyn; + if (yyarg) + yyarg[yycount] = yyctx->yytoken; + ++yycount; + yyn = yypcontext_expected_tokens (yyctx, + yyarg ? yyarg + 1 : yyarg, yyargn - 1); + if (yyn == YYENOMEM) + return YYENOMEM; + else + yycount += yyn; + } + return yycount; +} + +/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message + about the unexpected token YYTOKEN for the state stack whose top is + YYSSP. + + Return 0 if *YYMSG was successfully written. Return -1 if *YYMSG is + not large enough to hold the message. In that case, also set + *YYMSG_ALLOC to the required number of bytes. Return YYENOMEM if the + required number of bytes is too large to store. */ +static int +yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg, + const yypcontext_t *yyctx<%= output.user_formals %>) +{ + enum { YYARGS_MAX = 5 }; + /* Internationalized format string. */ + const char *yyformat = YY_NULLPTR; + /* Arguments of yyformat: reported tokens (one for the "unexpected", + one per "expected"). */ + yysymbol_kind_t yyarg[YYARGS_MAX]; + /* Cumulated lengths of YYARG. */ + YYPTRDIFF_T yysize = 0; + + /* Actual size of YYARG. */ + int yycount = yy_syntax_error_arguments (yyctx, yyarg, YYARGS_MAX); + if (yycount == YYENOMEM) + return YYENOMEM; + + switch (yycount) + { +#define YYCASE_(N, S) \ + case N: \ + yyformat = S; \ + break + default: /* Avoid compiler warnings. */ + YYCASE_(0, YY_("syntax error")); + YYCASE_(1, YY_("syntax error, unexpected %s")); + YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); + YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s")); + YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); + YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); +#undef YYCASE_ + } + + /* Compute error message size. Don't count the "%s"s, but reserve + room for the terminator. */ + yysize = yystrlen (yyformat) - 2 * yycount + 1; + { + int yyi; + for (yyi = 0; yyi < yycount; ++yyi) + { + YYPTRDIFF_T yysize1 + = yysize + yytnamerr (YY_NULLPTR, yytname[yyarg[yyi]]); + if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM) + yysize = yysize1; + else + return YYENOMEM; + } + } + + if (*yymsg_alloc < yysize) + { + *yymsg_alloc = 2 * yysize; + if (! (yysize <= *yymsg_alloc + && *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM)) + *yymsg_alloc = YYSTACK_ALLOC_MAXIMUM; + return -1; + } + + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + { + char *yyp = *yymsg; + int yyi = 0; + while ((*yyp = *yyformat) != '\0') + if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount) + { + yyp += yytnamerr (yyp, yytname[yyarg[yyi++]]); + yyformat += 2; + } + else + { + ++yyp; + ++yyformat; + } + } + return 0; +} + +<%# b4_yydestruct_define %> +/*-----------------------------------------------. +| Release the memory associated to this symbol. | +`-----------------------------------------------*/ + +static void +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep, YYLTYPE *yylocationp<%= output.user_formals %>) +{ +<%= output.parse_param_use("yyvaluep", "yylocationp") %> + if (!yymsg) + yymsg = "Deleting"; + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp<%= output.user_args %>); + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + switch (yykind) + { +<%= output.symbol_actions_for_destructor -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + + + +<%- if output.error_recovery -%> +#ifndef YYMAXREPAIR +# define YYMAXREPAIR(<%= output.parse_param_name %>) (3) +#endif + +#ifndef YYERROR_RECOVERY_ENABLED +# define YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>) (1) +#endif + +enum yy_repair_type { + inserting, + deleting, + shifting, +}; + +struct yy_repair { + enum yy_repair_type type; + yysymbol_kind_t term; +}; +typedef struct yy_repair yy_repair; + +struct yy_repairs { + /* For debug */ + int id; + /* For breadth-first traversing */ + struct yy_repairs *next; + YYPTRDIFF_T stack_length; + /* Bottom of states */ + yy_state_t *states; + /* Top of states */ + yy_state_t *state; + /* repair length */ + int repair_length; + /* */ + struct yy_repairs *prev_repair; + struct yy_repair repair; +}; +typedef struct yy_repairs yy_repairs; + +struct yy_term { + yysymbol_kind_t kind; + YYSTYPE value; + YYLTYPE location; +}; +typedef struct yy_term yy_term; + +struct yy_repair_terms { + int id; + int length; + yy_term terms[]; +}; +typedef struct yy_repair_terms yy_repair_terms; + +static void +yy_error_token_initialize (yysymbol_kind_t yykind, YYSTYPE * const yyvaluep, YYLTYPE * const yylocationp<%= output.user_formals %>) +{ + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN +switch (yykind) + { +<%= output.symbol_actions_for_error_token -%> + default: + break; + } + YY_IGNORE_MAYBE_UNINITIALIZED_END +} + +static yy_repair_terms * +yy_create_repair_terms(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + yy_repair_terms *rep_terms; + int count = 0; + + while (r->prev_repair) + { + count++; + r = r->prev_repair; + } + + rep_terms = (yy_repair_terms *) YYMALLOC (sizeof (yy_repair_terms) + sizeof (yy_term) * count); + rep_terms->id = reps->id; + rep_terms->length = count; + + r = reps; + while (r->prev_repair) + { + rep_terms->terms[count-1].kind = r->repair.term; + count--; + r = r->prev_repair; + } + + return rep_terms; +} + +static void +yy_print_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + yy_repairs *r = reps; + + YYDPRINTF ((stderr, + "id: %d, repair_length: %d, repair_state: %d, prev_repair_id: %d\n", + reps->id, reps->repair_length, *reps->state, reps->prev_repair->id)); + + while (r->prev_repair) + { + YYDPRINTF ((stderr, "%s ", yysymbol_name (r->repair.term))); + r = r->prev_repair; + } + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_print_repair_terms(yy_repair_terms *rep_terms<%= output.user_formals %>) +{ + for (int i = 0; i < rep_terms->length; i++) + YYDPRINTF ((stderr, "%s ", yysymbol_name (rep_terms->terms[i].kind))); + + YYDPRINTF ((stderr, "\n")); +} + +static void +yy_free_repairs(yy_repairs *reps<%= output.user_formals %>) +{ + while (reps) + { + yy_repairs *r = reps; + reps = reps->next; + YYFREE (r->states); + YYFREE (r); + } +} + +static int +yy_process_repairs(yy_repairs *reps, yysymbol_kind_t token) +{ + int yyn; + int yystate = *reps->state; + int yylen = 0; + yysymbol_kind_t yytoken = token; + + goto yyrecover_backup; + +yyrecover_newstate: + // TODO: check reps->stack_length + reps->state += 1; + *reps->state = (yy_state_t) yystate; + + +yyrecover_backup: + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yyrecover_default; + + /* "Reading a token" */ + if (yytoken == YYSYMBOL_YYEMPTY) + return 1; + + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yyrecover_default; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyrecover_errlab; + yyn = -yyn; + goto yyrecover_reduce; + } + + /* shift */ + yystate = yyn; + yytoken = YYSYMBOL_YYEMPTY; + goto yyrecover_newstate; + + +yyrecover_default: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyrecover_errlab; + goto yyrecover_reduce; + + +yyrecover_reduce: + yylen = yyr2[yyn]; + /* YYPOPSTACK */ + reps->state -= yylen; + yylen = 0; + + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *reps->state; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *reps->state + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yyrecover_newstate; + +yyrecover_errlab: + return 0; +} + +static yy_repair_terms * +yyrecover(yy_state_t *yyss, yy_state_t *yyssp, int yychar<%= output.user_formals %>) +{ + yysymbol_kind_t yytoken = YYTRANSLATE (yychar); + yy_repair_terms *rep_terms = YY_NULLPTR; + int count = 0; + + yy_repairs *head = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + yy_repairs *current = head; + yy_repairs *tail = head; + YYPTRDIFF_T stack_length = yyssp - yyss + 1; + + head->id = count; + head->next = 0; + head->stack_length = stack_length; + head->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + head->state = head->states + (yyssp - yyss); + YYCOPY (head->states, yyss, stack_length); + head->repair_length = 0; + head->prev_repair = 0; + + stack_length = (stack_length * 2 > 100) ? (stack_length * 2) : 100; + count++; + + while (current) + { + int yystate = *current->state; + int yyn = yypact[yystate]; + /* See also: yypcontext_expected_tokens */ + if (!yypact_value_is_default (yyn)) + { + int yyxbegin = yyn < 0 ? -yyn : 0; + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + { + if (yyx != YYSYMBOL_YYerror) + { + if (current->repair_length + 1 > YYMAXREPAIR(<%= output.parse_param_name %>)) + continue; + + yy_repairs *reps = (yy_repairs *) YYMALLOC (sizeof (yy_repairs)); + reps->id = count; + reps->next = 0; + reps->stack_length = stack_length; + reps->states = (yy_state_t *) YYMALLOC (sizeof (yy_state_t) * (stack_length)); + reps->state = reps->states + (current->state - current->states); + YYCOPY (reps->states, current->states, current->state - current->states + 1); + reps->repair_length = current->repair_length + 1; + reps->prev_repair = current; + reps->repair.type = inserting; + reps->repair.term = (yysymbol_kind_t) yyx; + + /* Process PDA assuming next token is yyx */ + if (! yy_process_repairs (reps, (yysymbol_kind_t)yyx)) + { + YYFREE (reps); + continue; + } + + tail->next = reps; + tail = reps; + count++; + + if (yyx == yytoken) + { + rep_terms = yy_create_repair_terms (current<%= output.user_args %>); + YYDPRINTF ((stderr, "repair_terms found. id: %d, length: %d\n", rep_terms->id, rep_terms->length)); + yy_print_repairs (current<%= output.user_args %>); + yy_print_repair_terms (rep_terms<%= output.user_args %>); + + goto done; + } + + YYDPRINTF ((stderr, + "New repairs is enqueued. count: %d, yystate: %d, yyx: %d\n", + count, yystate, yyx)); + yy_print_repairs (reps<%= output.user_args %>); + } + } + } + + current = current->next; + } + +done: + + yy_free_repairs(head<%= output.user_args %>); + + if (!rep_terms) + { + YYDPRINTF ((stderr, "repair_terms not found\n")); + } + + return rep_terms; +} +<%- end -%> + + + +/*----------. +| yyparse. | +`----------*/ + +int +yyparse (<%= output.parse_param %>) +{ +<%# b4_declare_scanner_communication_variables -%> +/* Lookahead token kind. */ +int yychar; + + +/* The semantic value of the lookahead symbol. */ +/* Default value used for initialization, for pacifying older GCCs + or non-GCC compilers. */ +#ifdef __cplusplus +static const YYSTYPE yyval_default = {}; +(void) yyval_default; +#else +YY_INITIAL_VALUE (static const YYSTYPE yyval_default;) +#endif +YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); + +/* Location data for the lookahead symbol. */ +static const YYLTYPE yyloc_default +# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL + = { 1, 1, 1, 1 } +# endif +; +YYLTYPE yylloc = yyloc_default; + +<%# b4_declare_parser_state_variables -%> + /* Number of syntax errors so far. */ + int yynerrs = 0; + YY_USE (yynerrs); /* Silence compiler warning. */ + + yy_state_fast_t yystate = 0; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus = 0; + + /* Refer to the stacks through separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ + + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; + + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; + + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; + + /* The location stack: array, bottom, top. */ + YYLTYPE yylsa[YYINITDEPTH]; + YYLTYPE *yyls = yylsa; + YYLTYPE *yylsp = yyls; + + int yyn; + /* The return value of yyparse. */ + int yyresult; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; + YYLTYPE yyloc; + + /* The locations where the error started and ended. */ + YYLTYPE yyerror_range[3]; +<%- if output.error_recovery -%> + yy_repair_terms *rep_terms = 0; + yy_term term_backup; + int rep_terms_index; + int yychar_backup; +<%- end -%> + + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char *yymsg = yymsgbuf; + YYPTRDIFF_T yymsg_alloc = sizeof yymsgbuf; + +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N), yylsp -= (N)) + + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; + + YYDPRINTF ((stderr, "Starting parse\n")); + + yychar = YYEMPTY; /* Cause a token to be read. */ + + +<%# b4_user_initial_action -%> +<%= output.user_initial_action("/* User initialization code. */") %> +#line [@oline@] [@ofile@] + + yylsp[0] = yylloc; + goto yysetstate; + + +/*------------------------------------------------------------. +| yynewstate -- push a new state, which is found in yystate. | +`------------------------------------------------------------*/ +yynewstate: + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; + + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + + if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + YYNOMEM; +#else + { + /* Get the current used size of the three stacks, in elements. */ + YYPTRDIFF_T yysize = yyssp - yyss + 1; + +# if defined yyoverflow + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + yy_state_t *yyss1 = yyss; + YYSTYPE *yyvs1 = yyvs; + YYLTYPE *yyls1 = yyls; + + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow (YY_("memory exhausted"), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), + &yyls1, yysize * YYSIZEOF (*yylsp), + &yystacksize); + yyss = yyss1; + yyvs = yyvs1; + yyls = yyls1; + } +# else /* defined YYSTACK_RELOCATE */ + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + YYNOMEM; + yystacksize *= 2; + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yy_state_t *yyss1 = yyss; + union yyalloc *yyptr = + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); + if (! yyptr) + YYNOMEM; + YYSTACK_RELOCATE (yyss_alloc, yyss); + YYSTACK_RELOCATE (yyvs_alloc, yyvs); + YYSTACK_RELOCATE (yyls_alloc, yyls); +# undef YYSTACK_RELOCATE + if (yyss1 != yyssa) + YYSTACK_FREE (yyss1); + } +# endif + + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; + yylsp = yyls + yysize - 1; + + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END + + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ + + + if (yystate == YYFINAL) + YYACCEPT; + + goto yybackup; + + +/*-----------. +| yybackup. | +`-----------*/ +yybackup: + /* Do appropriate processing given the current state. Read a + lookahead token if we need one and don't already have one. */ + + /* First try to decide what to do without reference to lookahead token. */ + yyn = yypact[yystate]; + if (yypact_value_is_default (yyn)) + goto yydefault; + + /* Not known => get a lookahead token if don't already have one. */ + +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + if (yychar == YYEMPTY && rep_terms) + { + + if (rep_terms_index < rep_terms->length) + { + YYDPRINTF ((stderr, "An error recovery token is used\n")); + yy_term term = rep_terms->terms[rep_terms_index]; + yytoken = term.kind; + yylval = term.value; + yylloc = term.location; + yychar = yytranslate_inverted[yytoken]; + YY_SYMBOL_PRINT ("Next error recovery token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + rep_terms_index++; + } + else + { + YYDPRINTF ((stderr, "Error recovery is completed\n")); + yytoken = term_backup.kind; + yylval = term_backup.value; + yylloc = term_backup.location; + yychar = yychar_backup; + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + + YYFREE (rep_terms); + rep_terms = 0; + yychar_backup = 0; + } + } + } +<%- end -%> + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token\n")); + yychar = yylex <%= output.yylex_formals %>; + } + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + yychar = <%= output.eof_symbol.id.s_value %>; + yytoken = <%= output.eof_symbol.enum_name %>; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else if (yychar == <%= output.error_symbol.id.s_value %>) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = <%= output.undef_symbol.id.s_value %>; + yytoken = <%= output.error_symbol.enum_name %>; + yyerror_range[1] = yylloc; + goto yyerrlab1; + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yytable_value_is_error (yyn)) + goto yyerrlab; + yyn = -yyn; + goto yyreduce; + } + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the lookahead token. */ + YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc<%= output.user_args %>); + yystate = yyn; + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + *++yylsp = yylloc; +<%= output.after_shift_function("/* %after-shift code. */") %> + + /* Discard the shifted token. */ + yychar = YYEMPTY; + goto yynewstate; + + +/*-----------------------------------------------------------. +| yydefault -- do the default action for the current state. | +`-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + +/*-----------------------------. +| yyreduce -- do a reduction. | +`-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + '$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1-yylen]; +<%= output.before_reduce_function("/* %before-reduce function. */") %> + + /* Default location. */ + YYLLOC_DEFAULT (yyloc, (yylsp - yylen), yylen); + yyerror_range[1] = yyloc; + YY_REDUCE_PRINT (yyn<%= output.user_args %>); + switch (yyn) + { +<%= output.user_actions -%> + + default: break; + } + /* User semantic actions sometimes alter yychar, and that requires + that yytoken be updated with the new translation. We take the + approach of translating immediately before every use of yytoken. + One alternative is translating here after every semantic action, + but that translation would be missed if the semantic action invokes + YYABORT, YYACCEPT, or YYERROR immediately after altering yychar or + if it invokes YYBACKUP. In the case of YYABORT or YYACCEPT, an + incorrect destructor might then be invoked immediately. In the + case of YYERROR or YYBACKUP, subsequent parser actions might lead + to an incorrect destructor call or verbose syntax error message + before the lookahead is translated. */ + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc<%= output.user_args %>); + + YYPOPSTACK (yylen); +<%= output.after_reduce_function("/* %after-reduce function. */") %> + yylen = 0; + + *++yyvsp = yyval; + *++yylsp = yyloc; + + /* Now 'shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + goto yynewstate; + + +/*--------------------------------------. +| yyerrlab -- here on detecting error. | +`--------------------------------------*/ +yyerrlab: + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; + { + yypcontext_t yyctx + = {yyssp, yytoken, &yylloc}; + char const *yymsgp = YY_("syntax error"); + int yysyntax_error_status; + yysyntax_error_status = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + if (yysyntax_error_status == 0) + yymsgp = yymsg; + else if (yysyntax_error_status == -1) + { + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + yymsg = YY_CAST (char *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, yymsg_alloc))); + if (yymsg) + { + yysyntax_error_status + = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); + yymsgp = yymsg; + } + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + yysyntax_error_status = YYENOMEM; + } + } + yyerror (<%= output.yyerror_args %>, yymsgp); + if (yysyntax_error_status == YYENOMEM) + YYNOMEM; + } + } + + yyerror_range[1] = yylloc; + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse lookahead token after an + error, discard it. */ + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + /* Return failure if at end of input. */ + if (yychar == <%= output.eof_symbol.id.s_value %>) + YYABORT; + } + else + { + yydestruct ("Error: discarding", + yytoken, &yylval, &yylloc<%= output.user_args %>); + yychar = YYEMPTY; + } + } + + /* Else will try to reuse lookahead token after shifting the error + token. */ + goto yyerrlab1; + + +/*---------------------------------------------------. +| yyerrorlab -- error raised explicitly by YYERROR. | +`---------------------------------------------------*/ +yyerrorlab: + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; + ++yynerrs; + + /* Do not reclaim the symbols of the rule whose action triggered + this YYERROR. */ + YYPOPSTACK (yylen); +<%= output.after_pop_stack_function("yylen", "/* %after-pop-stack function. */") %> + yylen = 0; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + yystate = *yyssp; + goto yyerrlab1; + + +/*-------------------------------------------------------------. +| yyerrlab1 -- common code for both syntax error and YYERROR. | +`-------------------------------------------------------------*/ +yyerrlab1: +<%- if output.error_recovery -%> + if (YYERROR_RECOVERY_ENABLED(<%= output.parse_param_name %>)) + { + rep_terms = yyrecover (yyss, yyssp, yychar<%= output.user_args %>); + if (rep_terms) + { + for (int i = 0; i < rep_terms->length; i++) + { + yy_term *term = &rep_terms->terms[i]; + yy_error_token_initialize (term->kind, &term->value, &term->location<%= output.user_args %>); + } + + yychar_backup = yychar; + /* Can be packed into (the tail of) rep_terms? */ + term_backup.kind = yytoken; + term_backup.value = yylval; + term_backup.location = yylloc; + rep_terms_index = 0; + yychar = YYEMPTY; + + goto yybackup; + } + } +<%- end -%> + yyerrstatus = 3; /* Each real token shifted decrements this. */ + + /* Pop stack until we find a state that shifts the error token. */ + for (;;) + { + yyn = yypact[yystate]; + if (!yypact_value_is_default (yyn)) + { + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) + { + yyn = yytable[yyn]; + if (0 < yyn) + break; + } + } + + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + + yyerror_range[1] = *yylsp; + yydestruct ("Error: popping", + YY_ACCESSING_SYMBOL (yystate), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); +<%= output.after_pop_stack_function(1, "/* %after-pop-stack function. */") %> + yystate = *yyssp; + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + } + + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + *++yyvsp = yylval; + YY_IGNORE_MAYBE_UNINITIALIZED_END + + yyerror_range[2] = yylloc; + ++yylsp; + YYLLOC_DEFAULT (*yylsp, yyerror_range, 2); + + /* Shift the error token. */ + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp<%= output.user_args %>); +<%= output.after_shift_error_token_function("/* %after-shift-error-token code. */") %> + + yystate = yyn; + goto yynewstate; + + +/*-------------------------------------. +| yyacceptlab -- YYACCEPT comes here. | +`-------------------------------------*/ +yyacceptlab: + yyresult = 0; + goto yyreturnlab; + + +/*-----------------------------------. +| yyabortlab -- YYABORT comes here. | +`-----------------------------------*/ +yyabortlab: + yyresult = 1; + goto yyreturnlab; + + +/*-----------------------------------------------------------. +| yyexhaustedlab -- YYNOMEM (memory exhaustion) comes here. | +`-----------------------------------------------------------*/ +yyexhaustedlab: + yyerror (<%= output.yyerror_args %>, YY_("memory exhausted")); + yyresult = 2; + goto yyreturnlab; + + +/*----------------------------------------------------------. +| yyreturnlab -- parsing is finished, clean up and return. | +`----------------------------------------------------------*/ +yyreturnlab: + if (yychar != YYEMPTY) + { + /* Make sure we have latest lookahead translation. See comments at + user semantic actions for why this is necessary. */ + yytoken = YYTRANSLATE (yychar); + yydestruct ("Cleanup: discarding lookahead", + yytoken, &yylval, &yylloc<%= output.user_args %>); + } + /* Do not reclaim the symbols of the rule whose action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK (yylen); + YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); + while (yyssp != yyss) + { + yydestruct ("Cleanup: popping", + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp, yylsp<%= output.user_args %>); + YYPOPSTACK (1); + } +#ifndef yyoverflow + if (yyss != yyssa) + YYSTACK_FREE (yyss); +#endif + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + return yyresult; +} + +<%# b4_percent_code_get([[epilogue]]) -%> +<%- if output.aux.epilogue -%> +#line <%= output.aux.epilogue_first_lineno - 1 %> "<%= output.grammar_file_path %>" +<%= output.aux.epilogue -%> +<%- end -%> + diff --git a/tool/lrama/template/bison/yacc.h b/tool/lrama/template/bison/yacc.h new file mode 100644 index 0000000000..848dbf5961 --- /dev/null +++ b/tool/lrama/template/bison/yacc.h @@ -0,0 +1,40 @@ +<%# b4_generated_by -%> +/* A Bison parser, made by Lrama <%= Lrama::VERSION %>. */ + +<%# b4_copyright -%> +/* Bison interface for Yacc-like parsers in C + + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +<%# b4_disclaimer -%> +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ +<%= output.render_partial("bison/_yacc.h") %> diff --git a/tool/lrama/template/diagram/diagram.html b/tool/lrama/template/diagram/diagram.html new file mode 100644 index 0000000000..3e87e6e519 --- /dev/null +++ b/tool/lrama/template/diagram/diagram.html @@ -0,0 +1,102 @@ +<!DOCTYPE html> +<html> +<head> + <title>Lrama syntax diagrams</title> + + <style> + <%= output.default_style %> + .diagram-header { + display: inline-block; + font-weight: bold; + font-size: 18px; + margin-bottom: -8px; + text-align: center; + } + + svg { + width: 100%; + } + + svg.railroad-diagram g.non-terminal text { + cursor: pointer; + } + + h2.hover-header { + background-color: #90ee90; + } + + svg.railroad-diagram g.non-terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + + svg.railroad-diagram g.terminal.hover-g rect { + fill: #eded91; + stroke: 5; + } + </style> +</head> + +<body align="center"> + <%= output.diagrams %> + <script> + document.addEventListener("DOMContentLoaded", () => { + function addHoverEffect(selector, hoverClass, relatedSelector, relatedHoverClass, getTextElements) { + document.querySelectorAll(selector).forEach(element => { + element.addEventListener("mouseenter", () => { + element.classList.add(hoverClass); + getTextElements(element).forEach(textEl => { + if (!relatedSelector) return; + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.add(relatedHoverClass); + }); + }); + }); + + element.addEventListener("mouseleave", () => { + element.classList.remove(hoverClass); + if (!relatedSelector) return; + getTextElements(element).forEach(textEl => { + getElementsByText(relatedSelector, textEl.textContent).forEach(related => { + related.classList.remove(relatedHoverClass); + }); + }); + }); + }); + } + + function getElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => el.textContent.trim() === text.trim()); + } + + function getParentElementsByText(selector, text) { + return [...document.querySelectorAll(selector)].filter(el => + [...el.querySelectorAll("text")].some(textEl => textEl.textContent.trim() === text.trim()) + ); + } + + function scrollToMatchingHeader() { + document.querySelectorAll("g.non-terminal").forEach(element => { + element.addEventListener("click", () => { + const textElements = [...element.querySelectorAll("text")]; + for (const textEl of textElements) { + const targetHeader = getElementsByText("h2", textEl.textContent)[0]; + if (targetHeader) { + targetHeader.scrollIntoView({ behavior: "smooth", block: "start" }); + break; + } + } + }); + }); + } + + addHoverEffect("h2", "hover-header", "g.non-terminal", "hover-g", element => [element]); + addHoverEffect("g.non-terminal", "hover-g", "h2", "hover-header", + element => [...element.querySelectorAll("text")] + ); + addHoverEffect("g.terminal", "hover-g", "", "", element => [element]); + scrollToMatchingHeader(); + }); + </script> +</body> +</html> diff --git a/tool/m4/_colorize_result_prepare.m4 b/tool/m4/_colorize_result_prepare.m4 new file mode 100644 index 0000000000..8439acf3ed --- /dev/null +++ b/tool/m4/_colorize_result_prepare.m4 @@ -0,0 +1,34 @@ +dnl -*- Autoconf -*- +AC_DEFUN([_COLORIZE_RESULT_PREPARE], [ + msg_checking= msg_result_yes= msg_result_no= msg_result_other= msg_reset= + AS_CASE(["x${CONFIGURE_TTY}"], + [xyes|xalways],[configure_tty=1], + [xno|xnever], [configure_tty=0], + [AS_IF([test -t 1], + [configure_tty=1], + [configure_tty=0])]) + AS_IF([test $configure_tty -eq 1], [ + msg_begin="`tput smso 2>/dev/null`" + AS_IF([test -z "$msg_begin"], [msg_begin="`tput so 2>/dev/null`"]) + AS_CASE(["$msg_begin"], ['@<:@'*m], + [msg_begin="`echo "$msg_begin" | sed ['s/[0-9]*m$//']`" + msg_checking="${msg_begin}33m" + AS_IF([test ${TEST_COLORS:+set}], [ + msg_result_yes=[`expr ":$TEST_COLORS:" : ".*:pass=\([^:]*\):"`] + msg_result_no=[`expr ":$TEST_COLORS:" : ".*:fail=\([^:]*\):"`] + msg_result_other=[`expr ":$TEST_COLORS:" : ".*:skip=\([^:]*\):"`] + ]) + msg_result_yes="${msg_begin}${msg_result_yes:-32;1}m" + msg_result_no="${msg_begin}${msg_result_no:-31;1}m" + msg_result_other="${msg_begin}${msg_result_other:-33;1}m" + msg_reset="${msg_begin}m" + ]) + AS_UNSET(msg_begin) + ]) + AS_REQUIRE_SHELL_FN([colorize_result], + [AS_FUNCTION_DESCRIBE([colorize_result], [MSG], [Colorize result])], + [AS_CASE(["$[]1"], + [yes], [_AS_ECHO([${msg_result_yes}$[]1${msg_reset}])], + [no], [_AS_ECHO([${msg_result_no}$[]1${msg_reset}])], + [_AS_ECHO([${msg_result_other}$[]1${msg_reset}])])]) +])dnl diff --git a/tool/m4/ac_msg_result.m4 b/tool/m4/ac_msg_result.m4 new file mode 100644 index 0000000000..12a3617c0d --- /dev/null +++ b/tool/m4/ac_msg_result.m4 @@ -0,0 +1,5 @@ +dnl -*- Autoconf -*- +AC_DEFUN([AC_MSG_RESULT], [dnl +{ _AS_ECHO_LOG([result: $1]) +COLORIZE_RESULT([$1]); dnl +}])dnl diff --git a/tool/m4/colorize_result.m4 b/tool/m4/colorize_result.m4 new file mode 100644 index 0000000000..83912040e5 --- /dev/null +++ b/tool/m4/colorize_result.m4 @@ -0,0 +1,9 @@ +dnl -*- Autoconf -*- +AC_DEFUN([COLORIZE_RESULT], [AC_REQUIRE([_COLORIZE_RESULT_PREPARE])dnl + AS_LITERAL_IF([$1], + [m4_case([$1], + [yes], [_AS_ECHO([${msg_result_yes}$1${msg_reset}])], + [no], [_AS_ECHO([${msg_result_no}$1${msg_reset}])], + [_AS_ECHO([${msg_result_other}$1${msg_reset}])])], + [colorize_result "$1"]) dnl +])dnl diff --git a/tool/m4/ruby_append_option.m4 b/tool/m4/ruby_append_option.m4 new file mode 100644 index 0000000000..8cd2741ae8 --- /dev/null +++ b/tool/m4/ruby_append_option.m4 @@ -0,0 +1,9 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_APPEND_OPTION], + [# RUBY_APPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="[$]$1 $2"])])dnl +AC_DEFUN([RUBY_PREPEND_OPTION], + [# RUBY_PREPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_append_options.m4 b/tool/m4/ruby_append_options.m4 new file mode 100644 index 0000000000..14213111ca --- /dev/null +++ b/tool/m4/ruby_append_options.m4 @@ -0,0 +1,7 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_APPEND_OPTIONS], + [# RUBY_APPEND_OPTIONS($1) + for rb_opt in $2; do + AS_CASE([" [$]{$1-} "], + [*" [$]{rb_opt} "*], [], [' '], [ $1="[$]{rb_opt}"], [ $1="[$]$1 [$]{rb_opt}"]) + done])dnl diff --git a/tool/m4/ruby_check_builtin_func.m4 b/tool/m4/ruby_check_builtin_func.m4 new file mode 100644 index 0000000000..40abc78ef8 --- /dev/null +++ b/tool/m4/ruby_check_builtin_func.m4 @@ -0,0 +1,10 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_FUNC], [dnl +AC_CACHE_CHECK([for $1], AS_TR_SH(rb_cv_builtin_$1), + [AC_LINK_IFELSE( + [AC_LANG_PROGRAM([int foo;], [$2;])], + [AS_TR_SH(rb_cv_builtin_$1)=yes], + [AS_TR_SH(rb_cv_builtin_$1)=no])]) +AS_IF([test "${AS_TR_SH(rb_cv_builtin_$1)}" != no], [ + AC_DEFINE(AS_TR_CPP(HAVE_BUILTIN_$1)) +])])dnl diff --git a/tool/m4/ruby_check_builtin_overflow.m4 b/tool/m4/ruby_check_builtin_overflow.m4 new file mode 100644 index 0000000000..8568d2c6d9 --- /dev/null +++ b/tool/m4/ruby_check_builtin_overflow.m4 @@ -0,0 +1,28 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_OVERFLOW], [dnl +{ # $0($1) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow, [int x;__builtin_[$1]_overflow(0,0,&x)]) + RUBY_CHECK_BUILTIN_FUNC(__builtin_[$1]_overflow_p, [__builtin_[$1]_overflow_p(0,0,(int)0)]) + + AS_IF([test "$rb_cv_builtin___builtin_[$1]_overflow" != no], [ + AC_CACHE_CHECK(for __builtin_[$1]_overflow with long long arguments, rb_cv_use___builtin_[$1]_overflow_long_long, [ + AC_LINK_IFELSE([AC_LANG_SOURCE([[ +@%:@pragma clang optimize off + +int +main(void) +{ + long long x = 0, y; + __builtin_$1_overflow(x, x, &y); + + return 0; +} +]])], + rb_cv_use___builtin_[$1]_overflow_long_long=yes, + rb_cv_use___builtin_[$1]_overflow_long_long=no)]) + ]) + AS_IF([test "$rb_cv_use___builtin_[$1]_overflow_long_long" = yes], [ + AC_DEFINE(USE___BUILTIN_[]AS_TR_CPP($1)_OVERFLOW_LONG_LONG, 1) + ]) +} +])dnl diff --git a/tool/m4/ruby_check_builtin_setjmp.m4 b/tool/m4/ruby_check_builtin_setjmp.m4 new file mode 100644 index 0000000000..1e5d9b3028 --- /dev/null +++ b/tool/m4/ruby_check_builtin_setjmp.m4 @@ -0,0 +1,27 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_BUILTIN_SETJMP], [ +AS_IF([test x"${ac_cv_func___builtin_setjmp}" = xyes], [ + unset ac_cv_func___builtin_setjmp +]) +AC_CACHE_CHECK(for __builtin_setjmp, ac_cv_func___builtin_setjmp, + [ + ac_cv_func___builtin_setjmp=no + for cast in "" "(void **)"; do + RUBY_WERROR_FLAG( + [AC_LINK_IFELSE([AC_LANG_PROGRAM([[@%:@include <setjmp.h> + @%:@include <stdio.h> + jmp_buf jb; + @%:@ifdef NORETURN + NORETURN(void t(void)); + @%:@endif + void t(void) {__builtin_longjmp($cast jb, 1);} + int jump(void) {(void)(__builtin_setjmp($cast jb) ? 1 : 0); return 0;}]], + [[ + void (*volatile f)(void) = t; + if (!jump()) printf("%d\n", f != 0); + ]])], + [ac_cv_func___builtin_setjmp="yes${cast:+ with cast ($cast)}"]) + ]) + test "$ac_cv_func___builtin_setjmp" = no || break + done]) +])dnl diff --git a/tool/m4/ruby_check_header.m4 b/tool/m4/ruby_check_header.m4 new file mode 100644 index 0000000000..6fec9d16c5 --- /dev/null +++ b/tool/m4/ruby_check_header.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_HEADER], + [# RUBY_CHECK_HEADER($@) + save_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS m4_if([$5], [], [$INCFLAGS], [$5])" + AC_CHECK_HEADERS([$1], [$2], [$3], [$4]) + CPPFLAGS="$save_CPPFLAGS" + unset save_CPPFLAGS]) diff --git a/tool/m4/ruby_check_printf_prefix.m4 b/tool/m4/ruby_check_printf_prefix.m4 new file mode 100644 index 0000000000..15bb4aee87 --- /dev/null +++ b/tool/m4/ruby_check_printf_prefix.m4 @@ -0,0 +1,29 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_PRINTF_PREFIX], [ +AC_CACHE_CHECK([for printf prefix for $1], [rb_cv_pri_prefix_]AS_TR_SH($1),[ + [rb_cv_pri_prefix_]AS_TR_SH($1)=[NONE] + RUBY_WERROR_FLAG(RUBY_APPEND_OPTIONS(CFLAGS, $rb_cv_wsuppress_flags) + for pri in $2; do + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <stdio.h> + @%:@include <stddef.h> + @%:@ifdef __GNUC__ + @%:@if defined __MINGW_PRINTF_FORMAT + @%:@define PRINTF_ARGS(decl, string_index, first_to_check) \ + decl __attribute__((format(__MINGW_PRINTF_FORMAT, string_index, first_to_check))) + @%:@else + @%:@define PRINTF_ARGS(decl, string_index, first_to_check) \ + decl __attribute__((format(printf, string_index, first_to_check))) + @%:@endif + @%:@else + @%:@define PRINTF_ARGS(decl, string_index, first_to_check) decl + @%:@endif + PRINTF_ARGS(void test_sprintf(const char*, ...), 1, 2);]], + [[printf("%]${pri}[d", (]$1[)42); + test_sprintf("%]${pri}[d", (]$1[)42);]])], + [rb_cv_pri_prefix_]AS_TR_SH($1)[=[$pri]; break]) + done)]) +AS_IF([test "[$rb_cv_pri_prefix_]AS_TR_SH($1)" != NONE], [ + AC_DEFINE_UNQUOTED([PRI_]m4_ifval($3,$3,AS_TR_CPP(m4_bpatsubst([$1],[_t$])))[_PREFIX], + "[$rb_cv_pri_prefix_]AS_TR_SH($1)") +]) +])dnl diff --git a/tool/m4/ruby_check_setjmp.m4 b/tool/m4/ruby_check_setjmp.m4 new file mode 100644 index 0000000000..6020b766b8 --- /dev/null +++ b/tool/m4/ruby_check_setjmp.m4 @@ -0,0 +1,17 @@ +dnl -*- Autoconf -*- +dnl used for AC_ARG_WITH(setjmp-type) +AC_DEFUN([RUBY_CHECK_SETJMP], [ +AC_CACHE_CHECK([for ]$1[ as a macro or function], ac_cv_func_$1, + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +@%:@include <setjmp.h> +]AC_INCLUDES_DEFAULT([$3])[ +@%:@define JMPARGS_1 env +@%:@define JMPARGS_2 env,1 +@%:@define JMPARGS JMPARGS_]m4_ifval($2,2,1)[ +]], + [m4_ifval($2,$2,jmp_buf)[ env; $1(JMPARGS);]])], + ac_cv_func_$1=yes, + ac_cv_func_$1=no)] +) +AS_IF([test "$ac_cv_func_]$1[" = yes], [AC_DEFINE([HAVE_]AS_TR_CPP($1), 1)]) +])dnl diff --git a/tool/m4/ruby_check_signedness.m4 b/tool/m4/ruby_check_signedness.m4 new file mode 100644 index 0000000000..f9fbb3c088 --- /dev/null +++ b/tool/m4/ruby_check_signedness.m4 @@ -0,0 +1,5 @@ +dnl -*- Autoconf -*- +dnl RUBY_CHECK_SIGNEDNESS [typename] [if-signed] [if-unsigned] [included] +AC_DEFUN([RUBY_CHECK_SIGNEDNESS], [dnl + AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([AC_INCLUDES_DEFAULT([$4])], [($1)-1 > 0])], + [$3], [$2])])dnl diff --git a/tool/m4/ruby_check_sizeof.m4 b/tool/m4/ruby_check_sizeof.m4 new file mode 100644 index 0000000000..975ac6c9be --- /dev/null +++ b/tool/m4/ruby_check_sizeof.m4 @@ -0,0 +1,108 @@ +dnl -*- Autoconf -*- +dnl RUBY_CHECK_SIZEOF [typename], [maybe same size types], [macros], [include] +AC_DEFUN([RUBY_CHECK_SIZEOF], +[dnl +AS_VAR_PUSHDEF([rbcv_var], [rbcv_sizeof_var])dnl +AS_VAR_PUSHDEF([cond], [rbcv_sizeof_cond])dnl +AS_VAR_PUSHDEF([t], [rbcv_sizeof_type])dnl +AS_VAR_PUSHDEF([s], [rbcv_sizeof_size])dnl +] +[m4_bmatch([$1], [\.], [], [if test "$universal_binary" = yes; then]) +AC_CACHE_CHECK([size of $1], [AS_TR_SH([ac_cv_sizeof_$1])], [ + unset AS_TR_SH(ac_cv_sizeof_$1) + rbcv_var=" +typedef m4_bpatsubst([$1], [\..*]) ac__type_sizeof_; +static ac__type_sizeof_ *rbcv_ptr; +@%:@define AS_TR_CPP(SIZEOF_$1) sizeof((*rbcv_ptr)[]m4_bmatch([$1], [\.], .m4_bpatsubst([$1], [^[^.]*\.]))) +" + m4_ifval([$2], [test -z "${AS_TR_SH(ac_cv_sizeof_$1)+set}" && { + for t in $2; do + AC_COMPILE_IFELSE( + [AC_LANG_BOOL_COMPILE_TRY(AC_INCLUDES_DEFAULT([$4] + [$rbcv_var]), + [AS_TR_CPP(SIZEOF_$1) == sizeof($t)])], [ + AS_TR_SH(ac_cv_sizeof_$1)=AS_TR_CPP([SIZEOF_]$t) + break]) + done + }], [ + AC_COMPUTE_INT([AS_TR_SH(ac_cv_sizeof_$1)], [AS_TR_CPP(SIZEOF_$1)], + [AC_INCLUDES_DEFAULT([$4]) +$rbcv_var], + [AS_TR_SH(ac_cv_sizeof_$1)=]) + ]) + unset cond + m4_ifval([$3], [test -z "${AS_TR_SH(ac_cv_sizeof_$1)+set}" && { + for s in 32 64 128; do + for t in $3; do + cond="${cond} +@%:@${cond+el}if defined(__${t}${s}__) || defined(__${t}${s}) || defined(_${t}${s}) || defined(${t}${s})" + hdr="AC_INCLUDES_DEFAULT([$4 +@%:@if defined(__${t}${s}__) || defined(__${t}${s}) || defined(_${t}${s}) || defined(${t}${s}) +@%:@ define AS_TR_CPP(HAVE_$1) 1 +@%:@else +@%:@ define AS_TR_CPP(HAVE_$1) 0 +@%:@endif])" + AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([$hdr], [!AS_TR_CPP(HAVE_$1)])], [continue]) + AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([$hdr] + [$rbcv_var], + [AS_TR_CPP(HAVE_$1) == (AS_TR_CPP(SIZEOF_$1) == ($s / $rb_cv_char_bit))])], + [AS_TR_SH(ac_cv_sizeof_$1)="${AS_TR_SH(ac_cv_sizeof_$1)+${AS_TR_SH(ac_cv_sizeof_$1)-} }${t}${s}"; continue]) + AC_COMPILE_IFELSE([AC_LANG_BOOL_COMPILE_TRY([$hdr] +[ +@%:@if AS_TR_CPP(HAVE_$1) +$rbcv_var +@%:@else +@%:@define AS_TR_CPP(SIZEOF_$1) 0 +@%:@endif +], + [AS_TR_CPP(HAVE_$1) == (AS_TR_CPP(SIZEOF_$1) == (m4_bmatch([$2], [^[0-9][0-9]*$], [$2], [($s / $rb_cv_char_bit)])))])], + [AS_TR_SH(ac_cv_sizeof_$1)="${AS_TR_SH(ac_cv_sizeof_$1)+${AS_TR_SH(ac_cv_sizeof_$1)-} }${t}${s}m4_bmatch([$2], [^[0-9][0-9]*$], [:$2])"]) + done + done + }]) + test "${AS_TR_SH(ac_cv_sizeof_$1)@%:@@<:@1-9@:>@}" = "${AS_TR_SH(ac_cv_sizeof_$1)}" && + m4_ifval([$2][$3], + [test "${AS_TR_SH(ac_cv_sizeof_$1)@%:@SIZEOF_}" = "${AS_TR_SH(ac_cv_sizeof_$1)}" && ]){ + test "$universal_binary" = yes && cross_compiling=yes + AC_COMPUTE_INT([t], AS_TR_CPP(SIZEOF_$1), [AC_INCLUDES_DEFAULT([$4])] +[${cond+$cond +@%:@else} +$rbcv_var +${cond+@%:@endif} +@%:@ifndef AS_TR_CPP(SIZEOF_$1) +@%:@define AS_TR_CPP(SIZEOF_$1) 0 +@%:@endif], [t=0]) + test "$universal_binary" = yes && cross_compiling=$real_cross_compiling + AS_IF([test ${t-0} != 0], [ + AS_TR_SH(ac_cv_sizeof_$1)="${AS_TR_SH(ac_cv_sizeof_$1)+${AS_TR_SH(ac_cv_sizeof_$1)-} }${t}" + ]) + } + : ${AS_TR_SH(ac_cv_sizeof_$1)=0} +]) +{ + unset cond + for t in ${AS_TR_SH(ac_cv_sizeof_$1)-}; do + AS_CASE(["$t"], + [[[0-9]*|SIZEOF_*]], [ + ${cond+echo "@%:@else"} + echo "[@%:@define ]AS_TR_CPP(SIZEOF_$1) $t" + break + ], + [ + s=`expr $t : ['.*[^0-9]\([0-9][0-9]*\)$']` + AS_CASE([$t], [*:*], [t="${t%:*}"], [s=`expr $s / $rb_cv_char_bit`]) + echo "@%:@${cond+el}if defined(__${t}__) || defined(__${t}) || defined(_${t}) || defined($t)" + echo "@%:@define AS_TR_CPP(SIZEOF_$1) $s" + cond=1 + ]) + done + ${cond+echo "@%:@endif"} +} >> confdefs.h +m4_bmatch([$1], [\.], [], [else +AC_CHECK_SIZEOF([$1], 0, [$4]) +fi]) +AS_VAR_POPDEF([rbcv_var])dnl +AS_VAR_POPDEF([cond])dnl +AS_VAR_POPDEF([t])dnl +AS_VAR_POPDEF([s])dnl +])dnl diff --git a/tool/m4/ruby_check_sysconf.m4 b/tool/m4/ruby_check_sysconf.m4 new file mode 100644 index 0000000000..f554786e77 --- /dev/null +++ b/tool/m4/ruby_check_sysconf.m4 @@ -0,0 +1,13 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CHECK_SYSCONF], [dnl +AC_CACHE_CHECK([whether _SC_$1 is supported], rb_cv_have_sc_[]m4_tolower($1), + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> + ]], + [[_SC_$1 >= 0]])], + rb_cv_have_sc_[]m4_tolower($1)=yes, + rb_cv_have_sc_[]m4_tolower($1)=no) + ]) +AS_IF([test "$rb_cv_have_sc_[]m4_tolower($1)" = yes], [ + AC_DEFINE(HAVE__SC_$1) +]) +])dnl diff --git a/tool/m4/ruby_cppoutfile.m4 b/tool/m4/ruby_cppoutfile.m4 new file mode 100644 index 0000000000..976cbb1c43 --- /dev/null +++ b/tool/m4/ruby_cppoutfile.m4 @@ -0,0 +1,18 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_CPPOUTFILE], +[AC_CACHE_CHECK(whether ${CPP} accepts -o, rb_cv_cppoutfile, +[save_CPPFLAGS="$CPPFLAGS" +CPPFLAGS='-o conftest-1.i' +rb_cv_cppoutfile=no +AC_PREPROC_IFELSE([AC_LANG_SOURCE([[test-for-cppout]])], + [grep test-for-cppout conftest-1.i > /dev/null && rb_cv_cppoutfile=yes]) +CPPFLAGS="$save_CPPFLAGS" +rm -f conftest*]) +AS_IF([test "$rb_cv_cppoutfile" = yes], [ + CPPOUTFILE='-o conftest.i' +], [test "$rb_cv_cppoutfile" = no], [ + CPPOUTFILE='> conftest.i' +], [test -n "$rb_cv_cppoutfile"], [ + CPPOUTFILE="$rb_cv_cppoutfile" +]) +AC_SUBST(CPPOUTFILE)])dnl diff --git a/tool/m4/ruby_decl_attribute.m4 b/tool/m4/ruby_decl_attribute.m4 new file mode 100644 index 0000000000..a8a73dc870 --- /dev/null +++ b/tool/m4/ruby_decl_attribute.m4 @@ -0,0 +1,45 @@ +dnl -*- Autoconf -*- +dnl RUBY_DECL_ATTRIBUTE(attrib, macroname, cachevar, condition, type, code) +AC_DEFUN([RUBY_DECL_ATTRIBUTE], [dnl +m4_ifval([$2], dnl + [AS_VAR_PUSHDEF([attrib], m4_bpatsubst([$2], [(.*)], []))], dnl + [AS_VAR_PUSHDEF([attrib], m4_toupper(m4_format(%.4s, [$5]))[_]AS_TR_CPP($1))] dnl +)dnl +m4_ifval([$3], dnl + [AS_VAR_PUSHDEF([rbcv],[$3])], dnl + [AS_VAR_PUSHDEF([rbcv],[rb_cv_]m4_format(%.4s, [$5])[_][$1])]dnl +)dnl +m4_pushdef([attrib_code],[m4_bpatsubst([$1],["],[\\"])])dnl +m4_pushdef([attrib_params],[m4_bpatsubst([$2(x)],[^[^()]*(\([^()]*\)).*],[\1])])dnl +m4_ifval([$4], [rbcv_cond=["$4"]; test "$rbcv_cond" || unset rbcv_cond]) +AC_CACHE_CHECK(for m4_ifval([$2],[m4_bpatsubst([$2], [(.*)], [])],[$1]) [$5] attribute, rbcv, dnl +[rbcv=x +RUBY_WERROR_FLAG([ +for mac in \ + "__attribute__ ((attrib_code)) x" \ + "x __attribute__ ((attrib_code))" \ + "__declspec(attrib_code) x" \ + x; do + m4_ifval([$4],mac="$mac"${rbcv_cond+" /* only if $rbcv_cond */"}) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([ + m4_ifval([$4],${rbcv_cond+[@%:@if ]$rbcv_cond}) +[@%:@define ]attrib[](attrib_params)[ $mac] +m4_ifval([$4],${rbcv_cond+[@%:@else]} +${rbcv_cond+[@%:@define ]attrib[](attrib_params)[ x]} +${rbcv_cond+[@%:@endif]}) +$6 +@%:@define mesg ("") +@%:@define san "address" + attrib[](attrib_params)[;]], [[]])], + [rbcv="$mac"; break]) +done +])]) +AS_IF([test "$rbcv" != x], [ + RUBY_DEFINE_IF(m4_ifval([$4],[${rbcv_cond}]), attrib[](attrib_params)[], $rbcv) +]) +m4_ifval([$4], [unset rbcv_cond]) dnl +m4_popdef([attrib_params])dnl +m4_popdef([attrib_code])dnl +AS_VAR_POPDEF([attrib])dnl +AS_VAR_POPDEF([rbcv])dnl +])dnl diff --git a/tool/m4/ruby_default_arch.m4 b/tool/m4/ruby_default_arch.m4 new file mode 100644 index 0000000000..2f25ba81ee --- /dev/null +++ b/tool/m4/ruby_default_arch.m4 @@ -0,0 +1,21 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_DEFAULT_ARCH], [ +# Set ARCH_FLAG for different width but family CPU +AC_MSG_CHECKING([arch option]) +AS_CASE([$1:"$host_cpu"], + [arm64:arm*], [ARCH_FLAG=-m64], + [arm*:arm*], [ARCH_FLAG=-m32], + [x86_64:[i[3-6]86]], [ARCH_FLAG=-m64], + [x64:x86_64], [], + [[i[3-6]86]:x86_64], [ARCH_FLAG=-m32], + [ppc64:ppc*], [ARCH_FLAG=-m64], + [ppc*:ppc64], [ARCH_FLAG=-m32], + [ + ARCH_FLAG= + for flag in "-arch "$1 -march=$1; do + _RUBY_TRY_CFLAGS([$]flag, [ARCH_FLAG="[$]flag"]) + test x"$ARCH_FLAG" = x || break + done] +) +AC_MSG_RESULT([${ARCH_FLAG:-'(none)'}]) +])dnl diff --git a/tool/m4/ruby_define_if.m4 b/tool/m4/ruby_define_if.m4 new file mode 100644 index 0000000000..aba55783a2 --- /dev/null +++ b/tool/m4/ruby_define_if.m4 @@ -0,0 +1,6 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_DEFINE_IF], [dnl + m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])printf "@%:@if %s\n" "$1" >>confdefs.h]) +AC_DEFINE_UNQUOTED($2, $3)dnl + m4_ifval([$1], [AS_LITERAL_IF([$1], [], [test "X$1" = X || ])printf "@%:@endif /* %s */\n" "$1" >>confdefs.h]) +])dnl diff --git a/tool/m4/ruby_defint.m4 b/tool/m4/ruby_defint.m4 new file mode 100644 index 0000000000..7f262a73fc --- /dev/null +++ b/tool/m4/ruby_defint.m4 @@ -0,0 +1,41 @@ +dnl -*- Autoconf -*- +dnl RUBY_DEFINT TYPENAME, SIZE, [UNSIGNED], [INCLUDES = DEFAULT-INCLUDES] +AC_DEFUN([RUBY_DEFINT], [dnl +AS_VAR_PUSHDEF([cond], [rb_defint_cond])dnl +AS_VAR_PUSHDEF([type], [rb_defint_type])dnl +AC_CACHE_CHECK([for $1], [rb_cv_type_$1], +[AC_COMPILE_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT([$4]) +typedef $1 t; int s = sizeof(t) == 42;])], + [rb_cv_type_$1=yes], + [AS_CASE([m4_bmatch([$2], [^[1-9][0-9]*$], $2, [$ac_cv_sizeof_]AS_TR_SH($2))], + ["1"], [ rb_cv_type_$1="m4_if([$3], [], [signed ], [$3 ])char"], + ["$ac_cv_sizeof_short"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])short"], + ["$ac_cv_sizeof_int"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])int"], + ["$ac_cv_sizeof_long"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])long"], + ["$ac_cv_sizeof_long_long"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])long long"], + ["${ac_cv_sizeof___int64@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int64"], + ["${ac_cv_sizeof___int128@%:@*:}"], [ rb_cv_type_$1="m4_if([$3], [], [], [$3 ])__int128"], + [ rb_cv_type_$1=no])])]) +AS_IF([test "${rb_cv_type_$1}" != no], [ + type="${rb_cv_type_$1@%:@@%:@*signed }" + AS_IF([test "$type" = "long long"], [type=long_long]) + AS_IF([test "$type" != yes && eval 'test -n "${ac_cv_sizeof_'$type'+set}"'], [ + eval cond='"${ac_cv_sizeof_'$type'}"' + AS_CASE([$cond], [*:*], [ + cond=AS_TR_CPP($type) + echo "@%:@if defined SIZEOF_"$cond" && SIZEOF_"$cond" > 0" >> confdefs.h + ], [cond=]) + ], [cond=]) + AC_DEFINE([HAVE_]AS_TR_CPP($1), 1) + AS_IF([test "${rb_cv_type_$1}" = yes], [ + m4_bmatch([$2], [^[1-9][0-9]*$], [AC_CHECK_SIZEOF([$1], 0, [AC_INCLUDES_DEFAULT([$4])])], + [RUBY_CHECK_SIZEOF([$1], [$2], [], [AC_INCLUDES_DEFAULT([$4])])]) + ], [ + AC_DEFINE_UNQUOTED($1, [$rb_cv_type_$1]) + AC_DEFINE_UNQUOTED([SIZEOF_]AS_TR_CPP($1), [SIZEOF_]AS_TR_CPP([$type])) + ]) + test -n "$cond" && echo "@%:@endif /* $cond */" >> confdefs.h +]) +AS_VAR_POPDEF([cond])dnl +AS_VAR_POPDEF([type])dnl +])dnl diff --git a/tool/m4/ruby_dtrace_available.m4 b/tool/m4/ruby_dtrace_available.m4 new file mode 100644 index 0000000000..e03b7762fc --- /dev/null +++ b/tool/m4/ruby_dtrace_available.m4 @@ -0,0 +1,20 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_DTRACE_AVAILABLE], +[AC_CACHE_CHECK(whether dtrace USDT is available, rb_cv_dtrace_available, +[ + echo "provider conftest{ probe fire(); };" > conftest_provider.d + rb_cv_dtrace_available=no + AS_FOR(opt, rb_dtrace_opt, ["-xnolibs" ""], [dnl + AS_IF([$DTRACE opt -h -o conftest_provider.h -s conftest_provider.d >/dev/null 2>/dev/null], + [], [continue]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include "conftest_provider.h"]], [[CONFTEST_FIRE();]])], + [], [continue]) + # DTrace is available on the system + rb_cv_dtrace_available=yes${rb_dtrace_opt:+"(opt)"} + break + ]) + rm -f conftest.[co] conftest_provider.[dho] +]) +AS_CASE(["$rb_cv_dtrace_available"], ["yes("*")"], + [DTRACE_OPT=`expr "$rb_cv_dtrace_available" : "yes(\(.*\))"`]) +])dnl diff --git a/tool/m4/ruby_dtrace_postprocess.m4 b/tool/m4/ruby_dtrace_postprocess.m4 new file mode 100644 index 0000000000..6fd6de7c9a --- /dev/null +++ b/tool/m4/ruby_dtrace_postprocess.m4 @@ -0,0 +1,30 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_DTRACE_POSTPROCESS], +[AC_CACHE_CHECK(whether $DTRACE needs post processing, rb_cv_prog_dtrace_g, +[ + rb_cv_prog_dtrace_g=no + AS_IF([{ + cat >conftest_provider.d <<_PROBES && + provider conftest { + probe fire(); + }; +_PROBES + $DTRACE ${DTRACE_OPT} -h -o conftest_provider.h -s conftest_provider.d >/dev/null 2>/dev/null && + : + }], [ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include "conftest_provider.h"]], [[CONFTEST_FIRE();]])],[ + AS_IF([{ + cp -p conftest.${ac_objext} conftest.${ac_objext}.save && + $DTRACE ${DTRACE_OPT} -G -s conftest_provider.d conftest.${ac_objext} 2>/dev/null && + : + }], [ + AS_IF([cmp -s conftest.o conftest.${ac_objext}.save], [ + rb_cv_prog_dtrace_g=yes + ], [ + rb_cv_prog_dtrace_g=rebuild + ]) + ])]) + ]) + rm -f conftest.[co] conftest_provider.[dho] +]) +])dnl diff --git a/tool/m4/ruby_func_attribute.m4 b/tool/m4/ruby_func_attribute.m4 new file mode 100644 index 0000000000..bce26fc16a --- /dev/null +++ b/tool/m4/ruby_func_attribute.m4 @@ -0,0 +1,7 @@ +dnl -*- Autoconf -*- +dnl RUBY_FUNC_ATTRIBUTE(attrib, macroname, cachevar, condition) +AC_DEFUN([RUBY_FUNC_ATTRIBUTE], [dnl + RUBY_DECL_ATTRIBUTE([$1], [$2], [$3], [$4], + [function], [@%:@define x int conftest_attribute_check(void)] + ) +])dnl diff --git a/tool/m4/ruby_mingw32.m4 b/tool/m4/ruby_mingw32.m4 new file mode 100644 index 0000000000..98e922340b --- /dev/null +++ b/tool/m4/ruby_mingw32.m4 @@ -0,0 +1,24 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_MINGW32], +[AS_CASE(["$host_os"], +[cygwin*], [ +AC_CACHE_CHECK(for mingw32 environment, rb_cv_mingw32, +[AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ +#ifndef __MINGW32__ +# error +#endif +]])],[rb_cv_mingw32=yes],[rb_cv_mingw32=no]) +rm -f conftest*]) +AS_IF([test "$rb_cv_mingw32" = yes], [ + target_os="mingw32" + : ${ac_tool_prefix:="`expr "$CC" : ['\(.*-\)g\?cc[^/]*$']`"} + AC_DEFINE(__USE_MINGW_ANSI_STDIO, 1) dnl for gnu_printf +]) +]) +AS_CASE(["$target_os"], [mingw*msvc], [ +target_os="`echo ${target_os} | sed 's/msvc$//'`" +]) +AS_CASE(["$target_cpu-$target_os"], [x86_64-mingw*], [ +target_cpu=x64 +]) +])dnl diff --git a/tool/m4/ruby_modular_gc.m4 b/tool/m4/ruby_modular_gc.m4 new file mode 100644 index 0000000000..661fce2e60 --- /dev/null +++ b/tool/m4/ruby_modular_gc.m4 @@ -0,0 +1,41 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_MODULAR_GC],[ +AC_ARG_WITH(modular-gc, + AS_HELP_STRING([--with-modular-gc=DIR], + [Enable replacement of Ruby's GC from a modular library in the specified directory.]), + [modular_gc_dir=$withval], [unset modular_gc_dir] +) + +AS_IF([test "$modular_gc_dir" = yes], [ + AC_MSG_ERROR(you must specify a directory when using --with-modular-gc) +]) + +AC_MSG_CHECKING([if building with modular GC support]) +AS_IF([test x"$modular_gc_dir" != x], [ + AC_MSG_RESULT([yes]) + + # Ensure that modular_gc_dir is always an absolute path so that Ruby + # never loads a modular GC from a relative path + AS_CASE(["$modular_gc_dir"], + [/*], [], + [test "$load_relative" = yes || modular_gc_dir="$prefix/$modular_gc_dir"] + ) + + # Ensure that modular_gc_dir always terminates with a / + AS_CASE(["$modular_gc_dir"], + [*/], [], + [modular_gc_dir="$modular_gc_dir/"] + ) + + AC_DEFINE([USE_MODULAR_GC], [1]) + + modular_gc_summary="yes (in $modular_gc_dir)" +], [ + AC_MSG_RESULT([no]) + AC_DEFINE([USE_MODULAR_GC], [0]) + + modular_gc_summary="no" +]) + +AC_SUBST(modular_gc_dir, "${modular_gc_dir}") +])dnl diff --git a/tool/m4/ruby_prepend_option.m4 b/tool/m4/ruby_prepend_option.m4 new file mode 100644 index 0000000000..3b7030a473 --- /dev/null +++ b/tool/m4/ruby_prepend_option.m4 @@ -0,0 +1,5 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_PREPEND_OPTION], + [# RUBY_PREPEND_OPTION($1) + AS_CASE([" [$]{$1-} "], + [*" $2 "*], [], [' '], [ $1="$2"], [ $1="$2 [$]$1"])])dnl diff --git a/tool/m4/ruby_prog_gnu_ld.m4 b/tool/m4/ruby_prog_gnu_ld.m4 new file mode 100644 index 0000000000..b38fb3d527 --- /dev/null +++ b/tool/m4/ruby_prog_gnu_ld.m4 @@ -0,0 +1,10 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_PROG_GNU_LD], +[AC_CACHE_CHECK(whether the linker is GNU ld, rb_cv_prog_gnu_ld, +[AS_IF([`$CC $CFLAGS $CPPFLAGS $LDFLAGS --print-prog-name=ld 2>&1` -v 2>&1 | grep "GNU ld" > /dev/null], [ + rb_cv_prog_gnu_ld=yes +], [ + rb_cv_prog_gnu_ld=no +])]) +GNU_LD=$rb_cv_prog_gnu_ld +AC_SUBST(GNU_LD)])dnl diff --git a/tool/m4/ruby_prog_makedirs.m4 b/tool/m4/ruby_prog_makedirs.m4 new file mode 100644 index 0000000000..132ff7768f --- /dev/null +++ b/tool/m4/ruby_prog_makedirs.m4 @@ -0,0 +1,9 @@ +dnl -*- Autoconf -*- +m4_defun([RUBY_PROG_MAKEDIRS], + [m4_bpatsubst(m4_defn([AC_PROG_MKDIR_P]), + [MKDIR_P=\"$ac_install_sh -d\"], [ + AS_IF([test "x$MKDIR_P" = "xfalse"], [AC_MSG_ERROR([mkdir -p is required])]) + MKDIR_P="mkdir -p"]) + ]dnl + AC_SUBST(MAKEDIRS, ["$MKDIR_P"]) +) diff --git a/tool/m4/ruby_replace_funcs.m4 b/tool/m4/ruby_replace_funcs.m4 new file mode 100644 index 0000000000..33b3ca70c1 --- /dev/null +++ b/tool/m4/ruby_replace_funcs.m4 @@ -0,0 +1,13 @@ +dnl -*- Autoconf -*- +dnl RUBY_REPLACE_FUNC [func] [included] +AC_DEFUN([RUBY_REPLACE_FUNC], [dnl + AC_CHECK_DECL([$1],dnl + [AC_DEFINE(AS_TR_CPP(HAVE_[$1]))],dnl + [AC_REPLACE_FUNCS($1)],dnl + [$2])dnl +])dnl +dnl +dnl RUBY_REPLACE_FUNCS [funcs] [included] +AC_DEFUN([RUBY_REPLACE_FUNCS], [dnl + m4_map_args_w([$1], [RUBY_REPLACE_FUNC(], [, [$2])])dnl +])dnl diff --git a/tool/m4/ruby_replace_type.m4 b/tool/m4/ruby_replace_type.m4 new file mode 100644 index 0000000000..3df0f3994f --- /dev/null +++ b/tool/m4/ruby_replace_type.m4 @@ -0,0 +1,68 @@ +dnl -*- Autoconf -*- +dnl RUBY_REPLACE_TYPE [typename] [default type] [macro type] [included] +AC_DEFUN([RUBY_REPLACE_TYPE], [dnl + AC_CHECK_TYPES([$1], + [n="patsubst([$1],["],[\\"])"], + [n="patsubst([$2],["],[\\"])"], + [$4]) + AC_CACHE_CHECK([for convertible type of [$1]], rb_cv_[$1]_convertible, [ + AC_COMPILE_IFELSE( + [AC_LANG_BOOL_COMPILE_TRY([AC_INCLUDES_DEFAULT([$4])] + [typedef $n rbcv_conftest_target_type; + extern rbcv_conftest_target_type rbcv_conftest_var; + ], [sizeof(&*rbcv_conftest_var)])], + [rb_cv_[$1]_convertible=PTR], + [ + u= t= + AS_CASE(["$n "], + [*" signed "*], [ ], + [*" unsigned "*], [ + u=U], + [RUBY_CHECK_SIGNEDNESS($n, [], [u=U], [$4])]) + AS_IF([test x"$t" = x], [ + for t in "long long" long int short; do + test -n "$u" && t="unsigned $t" + AC_COMPILE_IFELSE( + [AC_LANG_BOOL_COMPILE_TRY([AC_INCLUDES_DEFAULT([$4])] + [typedef $n rbcv_conftest_target_type; + typedef $t rbcv_conftest_replace_type; + extern rbcv_conftest_target_type rbcv_conftest_var; + extern rbcv_conftest_replace_type rbcv_conftest_var; + extern rbcv_conftest_target_type rbcv_conftest_func(void); + extern rbcv_conftest_replace_type rbcv_conftest_func(void); + ], [sizeof(rbcv_conftest_target_type) == sizeof(rbcv_conftest_replace_type)])], + [n="$t"; break]) + done + ]) + AS_CASE([" $n "], + [*" long long "*], [ + t=LL], + [*" long "*], [ + t=LONG], + [*" short "*], [ + t=SHORT], + [ + t=INT]) + rb_cv_[$1]_convertible=${u}${t}]) + ]) + AS_IF([test "${AS_TR_SH(ac_cv_type_[$1])}" = "yes"], [ + n="$1" + ], [ + AS_CASE(["${rb_cv_[$1]_convertible}"], + [*LL], [n="long long"], + [*LONG], [n="long"], + [*SHORT], [n="short"], + [n="int"]) + AS_CASE(["${rb_cv_[$1]_convertible}"], + [U*], [n="unsigned $n"]) + ]) + AS_CASE("${rb_cv_[$1]_convertible}", [PTR], [u=], [U*], [u=+1], [u=-1]) + AC_DEFINE_UNQUOTED(rb_[$1], $n) + AS_IF([test $u], [ + AC_DEFINE_UNQUOTED([SIGNEDNESS_OF_]AS_TR_CPP($1), $u) + AC_DEFINE_UNQUOTED([$3]2NUM[(v)], [${rb_cv_[$1]_convertible}2NUM(v)]) + AC_DEFINE_UNQUOTED(NUM2[$3][(v)], [NUM2${rb_cv_[$1]_convertible}(v)]) + AC_DEFINE_UNQUOTED(PRI_[$3]_PREFIX, + [PRI_`echo ${rb_cv_[$1]_convertible} | sed ['s/^U//']`_PREFIX]) + ]) +])dnl diff --git a/tool/m4/ruby_require_funcs.m4 b/tool/m4/ruby_require_funcs.m4 new file mode 100644 index 0000000000..78372d57d8 --- /dev/null +++ b/tool/m4/ruby_require_funcs.m4 @@ -0,0 +1,13 @@ +dnl -*- Autoconf -*- +dnl RUBY_REQUIRE_FUNC [func] [included] +AC_DEFUN([RUBY_REQUIRE_FUNC], [ +# RUBY_REQUIRE_FUNC([$1], [$2]) + AC_CHECK_FUNCS([$1]) + AS_IF([test "$ac_cv_func_[]AS_TR_SH($1)" = yes], [], + [AC_MSG_ERROR($1[() must be supported])]) +])dnl +dnl +dnl RUBY_REQUIRE_FUNCS [funcs] [included] +AC_DEFUN([RUBY_REQUIRE_FUNCS], [dnl + m4_map_args_w([$1], [RUBY_REQUIRE_FUNC(], [, [$2])])dnl +])dnl diff --git a/tool/m4/ruby_rm_recursive.m4 b/tool/m4/ruby_rm_recursive.m4 new file mode 100644 index 0000000000..b97701f88e --- /dev/null +++ b/tool/m4/ruby_rm_recursive.m4 @@ -0,0 +1,18 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_RM_RECURSIVE], [dnl +m4_version_prereq([2.70], [], [dnl +# suppress error messages, rm: cannot remove 'conftest.dSYM', from +# AC_EGREP_CPP with CFLAGS=-g on Darwin. +AS_CASE([$build_os], [darwin*], [ +rm() { + rm_recursive='' + for arg do + AS_CASE("$arg", + [--*], [], + [-*r*], [break], + [conftest.*], [AS_IF([test -d "$arg"], [rm_recursive=-r; break])], + []) + done + command rm $rm_recursive "[$]@" +} +])])])dnl diff --git a/tool/m4/ruby_setjmp_type.m4 b/tool/m4/ruby_setjmp_type.m4 new file mode 100644 index 0000000000..d26af34ea0 --- /dev/null +++ b/tool/m4/ruby_setjmp_type.m4 @@ -0,0 +1,43 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_SETJMP_TYPE], [ +RUBY_CHECK_BUILTIN_SETJMP +RUBY_CHECK_SETJMP(_setjmpex, [], [@%:@include <setjmpex.h>]) +RUBY_CHECK_SETJMP(_setjmp) +AC_MSG_CHECKING(for setjmp type) +setjmp_suffix= +AC_ARG_WITH(setjmp-type, + AS_HELP_STRING([--with-setjmp-type], [select setjmp type]), + [ + AS_CASE([$withval], + [__builtin_setjmp], [setjmp=__builtin_setjmp], + [_setjmp], [ setjmp_prefix=_], + [sigsetjmp*], [ AC_MSG_WARN(No longer use sigsetjmp; use setjmp instead); setjmp_prefix=], + [setjmp], [ setjmp_prefix=], + [setjmpex], [ setjmp_prefix= setjmp_suffix=ex], + [''], [ unset setjmp_prefix], + [ AC_MSG_ERROR(invalid setjmp type: $withval)])], [unset setjmp_prefix]) +setjmp_cast= +AS_IF([test ${setjmp_prefix+set}], [ + AS_IF([test "${setjmp_prefix}" && eval test '$ac_cv_func_'${setjmp_prefix}setjmp${setjmp_suffix} = no], [ + AC_MSG_ERROR(${setjmp_prefix}setjmp${setjmp_suffix} is not available) + ]) +], [{ AS_CASE("$ac_cv_func___builtin_setjmp", [yes*], [true], [false]) }], [ + setjmp_cast=`expr "$ac_cv_func___builtin_setjmp" : "yes with cast (\(.*\))"` + setjmp_prefix=__builtin_ + setjmp_suffix= +], [test "$ac_cv_header_setjmpex_h:$ac_cv_func__setjmpex" = yes:yes], [ + setjmp_prefix= + setjmp_suffix=ex +], [test "$ac_cv_func__setjmp" = yes], [ + setjmp_prefix=_ + setjmp_suffix= +], [ + setjmp_prefix= + setjmp_suffix= +]) +AC_MSG_RESULT(${setjmp_prefix}setjmp${setjmp_suffix}${setjmp_cast:+\($setjmp_cast\)}) +AC_DEFINE_UNQUOTED([RUBY_SETJMP(env)], [${setjmp_prefix}setjmp${setjmp_suffix}($setjmp_cast(env))]) +AC_DEFINE_UNQUOTED([RUBY_LONGJMP(env,val)], [${setjmp_prefix}longjmp($setjmp_cast(env),val)]) +AS_CASE(["$GCC:$setjmp_prefix"], [yes:__builtin_], [], AC_DEFINE_UNQUOTED(RUBY_JMP_BUF, jmp_buf)) +AS_IF([test x$setjmp_suffix = xex], [AC_DEFINE_UNQUOTED(RUBY_USE_SETJMPEX, 1)]) +])dnl diff --git a/tool/m4/ruby_stack_grow_direction.m4 b/tool/m4/ruby_stack_grow_direction.m4 new file mode 100644 index 0000000000..8c6fdd5722 --- /dev/null +++ b/tool/m4/ruby_stack_grow_direction.m4 @@ -0,0 +1,30 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_STACK_GROW_DIRECTION], [ + AS_VAR_PUSHDEF([stack_grow_dir], [rb_cv_stack_grow_dir_$1]) + AC_CACHE_CHECK(stack growing direction on $1, stack_grow_dir, [ +AS_CASE(["$1"], +[m68*|x86*|x64|i?86|ppc*|sparc*|alpha*|arm*|aarch*], [ $2=-1], +[hppa*], [ $2=+1], +[ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ +/* recurse to get rid of inlining */ +static int +stack_growup_p(addr, n) + volatile int *addr, n; +{ + volatile int end; + if (n > 0) + return *addr = stack_growup_p(addr, n - 1); + else + return (&end > addr); +} +int main() +{ + int x; + return stack_growup_p(&x, 10); +} +]])],[$2=-1],[$2=+1],[$2=0]) + ]) +eval stack_grow_dir=\$$2]) +eval $2=\$stack_grow_dir +AS_VAR_POPDEF([stack_grow_dir])])dnl diff --git a/tool/m4/ruby_thread.m4 b/tool/m4/ruby_thread.m4 new file mode 100644 index 0000000000..439c63bc22 --- /dev/null +++ b/tool/m4/ruby_thread.m4 @@ -0,0 +1,80 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_THREAD], [ +AC_ARG_WITH(thread, + AS_HELP_STRING([--with-thread=IMPLEMENTATION], [specify the thread implementation to use]), + [THREAD_MODEL=$withval], [ + THREAD_MODEL= + AS_CASE(["$target_os"], + [freebsd*], [ + AC_CACHE_CHECK([whether pthread should be enabled by default], + rb_cv_enable_pthread_default, + [AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ +@%:@include <osreldate.h> +@%:@if __FreeBSD_version < 502102 +@%:@error pthread should be disabled on this platform +@%:@endif + ]])], + rb_cv_enable_pthread_default=yes, + rb_cv_enable_pthread_default=no)]) + AS_IF([test $rb_cv_enable_pthread_default = yes], + [THREAD_MODEL=pthread], + [THREAD_MODEL=none]) + ], + [mingw*], [ + THREAD_MODEL=win32 + ], + [wasi*], [ + THREAD_MODEL=none + ], + [ + THREAD_MODEL=pthread + ] + ) +]) + +AS_IF([test x"$THREAD_MODEL" = xpthread], [ + AC_CHECK_HEADERS(pthread.h) + AS_IF([test x"$ac_cv_header_pthread_h" = xyes], [], [ + AC_MSG_WARN("Don't know how to find pthread header on your system -- thread support disabled") + THREAD_MODEL=none + ]) +]) +AS_IF([test x"$THREAD_MODEL" = xpthread], [ + THREAD_MODEL=none + for pthread_lib in thr pthread pthreads c c_r root; do + AC_CHECK_LIB($pthread_lib, pthread_create, + [THREAD_MODEL=pthread; break]) + done + AS_IF([test x"$THREAD_MODEL" = xpthread], [ + AC_DEFINE(_REENTRANT) + AC_DEFINE(_THREAD_SAFE) + AC_DEFINE(HAVE_LIBPTHREAD) + AC_CHECK_HEADERS(pthread_np.h, [], [], [@%:@include <pthread.h>]) + AS_CASE(["$pthread_lib:$target_os"], + [c:*], [], + [root:*], [], + [c_r:*|*:openbsd*|*:mirbsd*], [LIBS="-pthread $LIBS"], + [LIBS="-l$pthread_lib $LIBS"]) + ], [ + AC_MSG_WARN("Don't know how to find pthread library on your system -- thread support disabled") + ]) +]) + +AS_CASE(["$THREAD_MODEL"], +[pthread], [], +[win32], [], +[none], [], +[""], [AC_MSG_ERROR(thread model is missing)], + [AC_MSG_ERROR(unknown thread model $THREAD_MODEL)]) +AC_MSG_CHECKING(thread model) +AC_MSG_RESULT($THREAD_MODEL) + +THREAD_IMPL_H=thread_$THREAD_MODEL.h +AS_IF([test ! -f "$srcdir/$THREAD_IMPL_H"], + [AC_MSG_ERROR('$srcdir/$THREAD_IMPL_H' must exist)]) +THREAD_IMPL_SRC=thread_$THREAD_MODEL.c +AS_IF([test ! -f "$srcdir/$THREAD_IMPL_SRC"], + [AC_MSG_ERROR('$srcdir/$THREAD_IMPL_SRC' must exist)]) +AC_DEFINE_UNQUOTED(THREAD_IMPL_H, ["$THREAD_IMPL_H"]) +AC_DEFINE_UNQUOTED(THREAD_IMPL_SRC, ["$THREAD_IMPL_SRC"]) +])dnl diff --git a/tool/m4/ruby_try_cflags.m4 b/tool/m4/ruby_try_cflags.m4 new file mode 100644 index 0000000000..b397642aad --- /dev/null +++ b/tool/m4/ruby_try_cflags.m4 @@ -0,0 +1,41 @@ +dnl -*- Autoconf -*- +dnl +dnl Autoconf 2.67 fails to detect `-Werror=old-style-definition` due +dnl to the old style definition of `main`. +m4_version_prereq([2.70], [], [ +m4_defun([AC_LANG_PROGRAM(C)], m4_bpatsubst(m4_defn([AC_LANG_PROGRAM(C)]), [main ()], [main (void)])) +])dnl +dnl +AC_DEFUN([_RUBY_TRY_CFLAGS], [ + RUBY_WERROR_FLAG([ + CFLAGS="[$]CFLAGS $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS([$1], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)], + [$4], [$5]) +])dnl + +AC_DEFUN([_RUBY_TRY_CFLAGS_PREPEND], [ + RUBY_WERROR_FLAG([ + CFLAGS="$1 [$]CFLAGS" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2], [$3]) + ])dnl +])dnl +AC_DEFUN([RUBY_TRY_CFLAGS_PREPEND], [ + AC_MSG_CHECKING([whether ]$1[ is accepted as CFLAGS])dnl + _RUBY_TRY_CFLAGS_PREPEND([$1], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)], + [$4], [$5]) +])dnl diff --git a/tool/m4/ruby_try_cxxflags.m4 b/tool/m4/ruby_try_cxxflags.m4 new file mode 100644 index 0000000000..06f645f546 --- /dev/null +++ b/tool/m4/ruby_try_cxxflags.m4 @@ -0,0 +1,17 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_TRY_CXXFLAGS], [ + save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="[$]CXXFLAGS $1" + AC_MSG_CHECKING([whether ]$1[ is accepted as CXXFLAGS]) + RUBY_WERROR_FLAG([ + AC_LANG_PUSH([C++]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)]) + ]) + AC_LANG_POP([C++]) + CXXFLAGS="$save_CXXFLAGS" + save_CXXFLAGS= +])dnl diff --git a/tool/m4/ruby_try_ldflags.m4 b/tool/m4/ruby_try_ldflags.m4 new file mode 100644 index 0000000000..c3a6be0fb3 --- /dev/null +++ b/tool/m4/ruby_try_ldflags.m4 @@ -0,0 +1,15 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_TRY_LDFLAGS], [ + save_LDFLAGS="$LDFLAGS" + LDFLAGS="[$]LDFLAGS $1" + AC_MSG_CHECKING([whether $1 is accepted as LDFLAGS]) + RUBY_WERROR_FLAG([ + AC_LINK_IFELSE([AC_LANG_PROGRAM([[$4]], [[$5]])], + [$2 + AC_MSG_RESULT(yes)], + [$3 + AC_MSG_RESULT(no)]) + ]) + LDFLAGS="$save_LDFLAGS" + save_LDFLAGS= +])dnl diff --git a/tool/m4/ruby_type_attribute.m4 b/tool/m4/ruby_type_attribute.m4 new file mode 100644 index 0000000000..5ea1219c6e --- /dev/null +++ b/tool/m4/ruby_type_attribute.m4 @@ -0,0 +1,8 @@ +dnl -*- Autoconf -*- +dnl RUBY_TYPE_ATTRIBUTE(attrib, macroname, cachevar, condition) +AC_DEFUN([RUBY_TYPE_ATTRIBUTE], [dnl + RUBY_DECL_ATTRIBUTE([$1], [$2], [$3], [$4], + [type], [ +@%:@define x struct conftest_attribute_check {int i;} +]) +])dnl diff --git a/tool/m4/ruby_universal_arch.m4 b/tool/m4/ruby_universal_arch.m4 new file mode 100644 index 0000000000..d3e0dd0b47 --- /dev/null +++ b/tool/m4/ruby_universal_arch.m4 @@ -0,0 +1,122 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_UNIVERSAL_ARCH], [ +# RUBY_UNIVERSAL_ARCH begin +ARCH_FLAG=`expr " $CXXFLAGS " : ['.* \(-m[0-9][0-9]*\) ']` +test ${CXXFLAGS+set} && CXXFLAGS=`echo "$CXXFLAGS" | sed [-e 's/ *-arch *[^ ]*//g' -e 's/ *-m32//g' -e 's/ *-m64//g']` +ARCH_FLAG=`expr " $CFLAGS " : ['.* \(-m[0-9][0-9]*\) ']` +test ${CFLAGS+set} && CFLAGS=`echo "$CFLAGS" | sed [-e 's/ *-arch *[^ ]*//g' -e 's/ *-m32//g' -e 's/ *-m64//g']` +test ${LDFLAGS+set} && LDFLAGS=`echo "$LDFLAGS" | sed [-e 's/ *-arch *[^ ]*//g' -e 's/ *-m32//g' -e 's/ *-m64//g']` +unset universal_binary universal_archnames +AS_IF([test ${target_archs+set}], [ + AC_MSG_CHECKING([target architectures]) + target_archs=`echo $target_archs | tr , ' '` + # /usr/lib/arch_tool -archify_list $TARGET_ARCHS + for archs in $target_archs + do + AS_CASE([",$universal_binary,"],[*",$archs,"*], [],[ + cpu=$archs + cpu=`echo $cpu | sed 's/-.*-.*//'` + universal_binary="${universal_binary+$universal_binary,}$cpu" + universal_archnames="${universal_archnames:+$universal_archnames }${archs}=${cpu}" + ARCH_FLAG="${ARCH_FLAG+$ARCH_FLAG }-arch $archs" + ]) + done + target_archs="$universal_binary" + unset universal_binary + AS_CASE(["$target_archs"], + [*,*], [universal_binary=yes], + [unset universal_archnames]) + AC_MSG_RESULT([$target_archs]) + + target=`echo $target | sed "s/^$target_cpu-/-/"` + target_alias=`echo $target_alias | sed "s/^$target_cpu-/-/"` + AS_IF([test "${universal_binary-no}" = yes], [ + AC_SUBST(try_header,try_compile) + target_cpu=universal + real_cross_compiling=$cross_compiling + ], [ + AS_IF([test x"$target_cpu" != x"${target_archs}"], [ + echo 'int main(){return 0;}' > conftest.c + AS_IF([$CC $CFLAGS $ARCH_FLAG -o conftest conftest.c > /dev/null 2>&1], [ + rm -fr conftest.* + ], [test -z "$ARCH_FLAG"], [ + RUBY_DEFAULT_ARCH($target_archs) + ]) + ]) + target_cpu=${target_archs} + ]) + AS_CASE(["$target"], [-*], [ target="$target_cpu${target}"]) + AS_CASE(["$target_alias"], [-*], [ target_alias="$target_cpu${target_alias}"]) +], [ + AS_IF([test x"$target_alias" = x], [ + AS_CASE(["$target_os"], + [darwin*], [ + AC_MSG_CHECKING([for real target cpu]) + target=`echo $target | sed "s/^$target_cpu-/-/"` + target_cpu=`$CC -E - 2>/dev/null <<EOF | +#ifdef __x86_64__ +"processor-name=x86_64" +#endif +#ifdef __i386__ +"processor-name=i386" +#endif +#ifdef __ppc__ +"processor-name=powerpc" +#endif +#ifdef __ppc64__ +"processor-name=powerpc64" +#endif +#ifdef __arm64__ +"processor-name=arm64" +#endif +EOF + sed -n 's/^"processor-name=\(.*\)"/\1/p'` + target="$target_cpu${target}" + AC_MSG_RESULT([$target_cpu]) + ]) + ]) + target_archs="$target_cpu" +]) +AS_IF([test "${target_archs}" != "${rb_cv_target_archs-${target_archs}}"], [ + AC_MSG_ERROR([target arch(s) has changed from ${rb_cv_target_archs-nothing} to ${target_archs}]) +], [ + rb_cv_target_archs=${target_archs} +]) +AS_IF([test "x${ARCH_FLAG}" != x], [ + CFLAGS="${CFLAGS:+$CFLAGS }${ARCH_FLAG}" + LDFLAGS="${LDFLAGS:+$LDFLAGS }${ARCH_FLAG}" +]) +# RUBY_UNIVERSAL_ARCH end +])dnl +dnl +AC_DEFUN([RUBY_UNIVERSAL_CHECK_HEADER_COND], [ dnl + AC_CACHE_CHECK([for $2 when $1], [$3], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [AC_INCLUDES_DEFAULT([$6])[ + @%:@if ]$1[ + @%:@include <]$2[> + @%:@endif]], [[]])], + [AS_VAR_SET($3, yes)], + [AS_VAR_SET($3, no)])) + AS_VAR_IF([$3], [yes], [dnl + printf "@%:@if %s\n" "$1" >>confdefs.h + AC_DEFINE_UNQUOTED(HAVE_[]AS_TR_CPP($2), 1)dnl + printf "@%:@endif\n" >>confdefs.h dnl + $4], [$5]) +])dnl +dnl +dnl RUBY_UNIVERSAL_CHECK_HEADER(CPU-LIST, HEADER, +dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND], +dnl [INCLUDES = DEFAULT-INCLUDES]) +AC_DEFUN([RUBY_UNIVERSAL_CHECK_HEADER], [ dnl + m4_if([$# dnl + ], [0], [], [ dnl + m4_foreach([rb_Header], [$1], + [AS_CASE([",$target_archs,"], [*,]rb_Header[,*], + [RUBY_UNIVERSAL_CHECK_HEADER_COND]([defined(__[]rb_Header[]__)], + [$2], [rb_cv_header_[]AS_TR_SH($2)_on_[]AS_TR_SH(rb_Header)], + [$3], [$4], [$5]) + ) + ]) + ])dnl +])dnl diff --git a/tool/m4/ruby_wasm_tools.m4 b/tool/m4/ruby_wasm_tools.m4 new file mode 100644 index 0000000000..efc017e771 --- /dev/null +++ b/tool/m4/ruby_wasm_tools.m4 @@ -0,0 +1,25 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_WASM_TOOLS], +[AS_CASE(["$target_os"], +[wasi*], [ + AC_CHECK_TOOL(WASMOPT, wasm-opt) + AS_IF([test x"${WASMOPT}" = x], [ + AC_MSG_ERROR([wasm-opt is required]) + ]) + AC_SUBST(wasmoptflags) + : ${wasmoptflags=-O3} + + AC_MSG_CHECKING([whether \$WASI_SDK_PATH is set]) + AS_IF([test x"${WASI_SDK_PATH}" = x], [ + AC_MSG_RESULT([no]) + AC_MSG_ERROR([WASI_SDK_PATH environment variable is required]) + ], [ + AC_MSG_RESULT([yes]) + CC="${CC:-${WASI_SDK_PATH}/bin/clang}" + LD="${LD:-${WASI_SDK_PATH}/bin/clang}" + AR="${AR:-${WASI_SDK_PATH}/bin/llvm-ar}" + RANLIB="${RANLIB:-${WASI_SDK_PATH}/bin/llvm-ranlib}" + OBJCOPY="${OBJCOPY:-${WASI_SDK_PATH}/bin/llvm-objcopy}" + ]) +]) +])dnl diff --git a/tool/m4/ruby_werror_flag.m4 b/tool/m4/ruby_werror_flag.m4 new file mode 100644 index 0000000000..616a7f6abf --- /dev/null +++ b/tool/m4/ruby_werror_flag.m4 @@ -0,0 +1,18 @@ +dnl -*- Autoconf -*- +AC_DEFUN([RUBY_WERROR_FLAG], [dnl +save_CFLAGS="$CFLAGS" +CFLAGS="$CFLAGS $rb_cv_warnflags" +AS_IF([test "${ac_c_werror_flag+set}"], [ + rb_c_werror_flag="$ac_c_werror_flag" +], [ + unset rb_c_werror_flag +]) +ac_c_werror_flag=yes +$1 +CFLAGS="$save_CFLAGS" +save_CFLAGS= +AS_IF([test "${rb_c_werror_flag+set}"], [ + ac_c_werror_flag="$rb_c_werror_flag" +], [ + unset ac_c_werror_flag +])])dnl diff --git a/tool/make-snapshot b/tool/make-snapshot index 865fdeb965..4af6a855eb 100755 --- a/tool/make-snapshot +++ b/tool/make-snapshot @@ -1,62 +1,78 @@ #!/usr/bin/ruby -s # -*- coding: us-ascii -*- +require 'rubygems' +require 'rubygems/package' +require 'rubygems/package/tar_writer' require 'uri' require 'digest/sha1' require 'digest/sha2' require 'fileutils' require 'shellwords' require 'tmpdir' -require File.expand_path("../vcs", __FILE__) +require 'pathname' +require 'date' +require 'yaml' +require 'json' +require File.expand_path("../lib/vcs", __FILE__) +require File.expand_path("../lib/colorize", __FILE__) STDOUT.sync = true $srcdir ||= nil -$exported = nil if ($exported ||= nil) == "" $archname = nil if ($archname ||= nil) == "" $keep_temp ||= nil $patch_file ||= nil $packages ||= nil $digests ||= nil +$no7z ||= nil $tooldir = File.expand_path("..", __FILE__) +$unicode_version = nil if ($unicode_version ||= nil) == "" +$colorize = Colorize.new def usage <<USAGE usage: #{File.basename $0} [option...] new-directory-to-save [version ...] options: -srcdir=PATH source directory path - -exported=PATH make snapshot from already exported working directory -archname=NAME make the basename of snapshots NAME -keep_temp keep temporary working directory -patch_file=PATCH apply PATCH file after export -packages=PKG[,...] make PKG packages (#{PACKAGES.keys.join(", ")}) -digests=ALG[,...] show ALG digests (#{DIGESTS.join(", ")}) + -unicode_version=VER Unicode version to generate encodings + -help, --help show this message version: - trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL + master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL each versions may be followed by optional @revision. USAGE end DIGESTS = %w[SHA1 SHA256 SHA512] PACKAGES = { + "tar" => %w".tar", "bzip" => %w".tar.bz2 bzip2 -c", "gzip" => %w".tar.gz gzip -c", "xz" => %w".tar.xz xz -c", - "zip" => %w".zip zip -qr", + "zip" => %w".zip zip -Xqr", } +DEFAULT_PACKAGES = PACKAGES.keys - ["tar"] +if !$no7z and system("7z", out: IO::NULL) + PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so" + PACKAGES["zip"] = %w".zip 7z a -tzip -mx -mtc=off" << {out: IO::NULL} +elsif gzip = ENV.delete("GZIP") + PACKAGES["gzip"].concat(gzip.shellsplit) +end +if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags + mflags = mflags.gsub(/(\A|\s)(-\S*)j\d*/, '\1\2') + mflags.strip! + ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?) +end ENV["LC_ALL"] = ENV["LANG"] = "C" -SVNURL = URI.parse("http://svn.ruby-lang.org/repos/ruby/") +# https git clone is disabled at git.ruby-lang.org/ruby.git. +GITURL = URI.parse("https://github.com/ruby/ruby.git") RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/ -ENV["VPATH"] ||= "include/ruby" -YACC = ENV["YACC"] ||= "bison" -ENV["BASERUBY"] ||= "ruby" -ENV["RUBY"] ||= "ruby" -ENV["MV"] ||= "mv" -ENV["RM"] ||= "rm -f" -ENV["MINIRUBY"] ||= "ruby" -ENV["PROGRAM"] ||= "ruby" -ENV["AUTOCONF"] ||= "autoconf" -ENV["BUILTIN_TRANSOBJS"] ||= "newline.o" +ENV["TZ"] = "UTC" class String # for older ruby @@ -77,10 +93,13 @@ class Dir end $packages &&= $packages.split(/[, ]+/).tap {|pkg| + if all = pkg.index("all") + pkg[all, 1] = DEFAULT_PACKAGES - pkg + end pkg -= PACKAGES.keys pkg.empty? or abort "#{File.basename $0}: unknown packages - #{pkg.join(", ")}" } -$packages ||= PACKAGES.keys +$packages ||= DEFAULT_PACKAGES $digests &&= $digests.split(/[, ]+/).tap {|dig| dig -= DIGESTS @@ -89,22 +108,32 @@ $digests &&= $digests.split(/[, ]+/).tap {|dig| $digests ||= DIGESTS $patch_file &&= File.expand_path($patch_file) -path = ENV["PATH"].split(File::PATH_SEPARATOR) -%w[YACC BASERUBY RUBY MV MINIRUBY].each do |var| - cmd, = ENV[var].shellsplit - unless path.any? {|dir| +PATH = ENV["PATH"].split(File::PATH_SEPARATOR) +def PATH.executable_env(var, command = nil) + command = ENV[var] ||= (command or return) + cmd, = command.shellsplit + unless any? {|dir| file = File.expand_path(cmd, dir) File.file?(file) and File.executable?(file) } abort "#{File.basename $0}: #{var} command not found - #{cmd}" end + command end +PATH.executable_env("MV", "mv") +PATH.executable_env("RM", "rm -f") +PATH.executable_env("AUTOCONF", "autoconf") + %w[BASERUBY RUBY MINIRUBY].each do |var| - `#{ENV[var]} --disable-gem -e1 2>&1` - if $?.success? - ENV[var] += ' --disable-gem' + cmd = PATH.executable_env(var, "ruby") + help = IO.popen("#{cmd} --help", err: %i[child out], &:read) + unless $?.success? and /ruby/ =~ help + abort "#{File.basename $0}: #{var} ruby not found - #{cmd}" end + IO.popen("#{cmd} --disable-gem -eexit", err: %i[child out], &:read) + cmd += ' --disable-gem' if $?.success? + ENV[var] = cmd end if defined?($help) or defined?($_help) @@ -114,26 +143,124 @@ end unless destdir = ARGV.shift abort usage end -revisions = ARGV.empty? ? ["trunk"] : ARGV -unless tmp = $exported - FileUtils.mkpath(destdir) - destdir = File.expand_path(destdir) - tmp = Dir.mktmpdir("ruby-snapshot") - FileUtils.mkpath(tmp) - at_exit { - Dir.chdir "/" - FileUtils.rm_rf(tmp) - } unless $keep_temp +revisions = ARGV.empty? ? [nil] : ARGV + +if defined?($exported) + abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead" +end + +FileUtils.mkpath(destdir) +destdir = File.expand_path(destdir) +tmp = Dir.mktmpdir("ruby-snapshot") +FileUtils.mkpath(tmp) +at_exit { + Dir.chdir "/" + FileUtils.rm_rf(tmp) +} unless $keep_temp + +def tar_create(tarball, dir) + header = Gem::Package::TarHeader + dir_type = "5" + uname = gname = "ruby" + File.open(tarball, "wb") do |f| + w = Gem::Package::TarWriter.new(f) + list = Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH) + list.reject! {|name| name.end_with?("/.")} + list.sort_by! {|name| name.split("/")} + list.each do |path| + next if File.basename(path) == "." + s = File.stat(path) + mode = 0644 + case + when s.file? + type = nil + size = s.size + mode |= 0111 if s.executable? + when s.directory? + path += "/" + type = dir_type + size = 0 + mode |= 0111 + else + next + end + name, prefix = w.split_name(path) + h = header.new(name: name, prefix: prefix, typeflag: type, + mode: mode, size: size, mtime: s.mtime, + uname: uname, gname: gname) + f.write(h) + if size > 0 + IO.copy_stream(path, f) + f.write("\0" * (-size % 512)) + end + end + end + true +rescue => e + warn e.message + false +end + +def touch_all(time, pattern, opt, &cond) + Dir.glob(pattern, opt) do |n| + stat = File.stat(n) + if stat.file? or stat.directory? + next if cond and !yield(n, stat) + File.utime(time, time, n) + end + end +rescue + false +else + true +end + +class MAKE < Struct.new(:prog, :args) + def initialize(vars) + vars = vars.map {|arg| arg.join("=")} + super(ENV["MAKE"] || ENV["make"] || "make", vars) + end + + def run(target) + err = IO.pipe do |r, w| + begin + pid = Process.spawn(self.prog, *self.args, target, {:err => w, r => :close}) + w.close + r.read + ensure + Process.wait(pid) + end + end + if $?.success? + true + else + STDERR.puts err + $colorize.fail("#{target} failed") + false + end + end +end + +def measure + clock = Process::CLOCK_MONOTONIC + t0 = Process.clock_gettime(clock) + STDOUT.flush + result = yield + printf(" %6.3f", Process.clock_gettime(clock) - t0) + STDOUT.flush + result end def package(vcs, rev, destdir, tmp = nil) - patchlevel = false + pwd = Dir.pwd prerelease = false - if revision = rev[/@(\d+)\z/, 1] + if rev and revision = rev[/@(\h+)\z/, 1] rev = $` end case rev - when /\Atrunk\z/ + when nil + url = nil + when /\A(?:master|trunk)\z/ url = vcs.trunk when /\Abranches\// url = vcs.branch($') @@ -142,27 +269,37 @@ def package(vcs, rev, destdir, tmp = nil) when /\Astable\z/ vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]} url &&= vcs.branch(url) - when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/ + when /\A(\d+)\.(\d+)\.(\d+)-(preview|rc)(\d+)/ prerelease = true tag = "#{$4}#{$5}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") - when /\A(.*)\.(.*)\.(.*)-p(\d+)/ - patchlevel = true - tag = "p#{$4}" - url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}") - when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/ - if $3 && ($1 > "2" || $1 == "2" && $2 >= "1") - patchlevel = true - tag = "" - url = vcs.tag("v#{$1}_#{$2}_#{$3}") + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") else - url = vcs.branch("ruby_#{rev.tr('.', '_')}") + url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}") end + when /\A(\d+)\.(\d+)\.(\d+)\z/ + tag = "" + if Integer($1) >= 4 + url = vcs.tag("v#{rev}") + else + url = vcs.tag("v#{$1}_#{$2}_#{$3}") + end + when /\A(\d+)\.(\d+)\z/ + url = vcs.branch("ruby_#{rev.tr('.', '_')}") else warn "#{$0}: unknown version - #{rev}" return end - revision ||= vcs.get_revisions(url)[1] + if info = vcs.get_revisions(url) + modified = info[2] + else + _, _, modified = VCS::Null.new(nil).get_revisions(url) + end + if !revision and info + revision = info + url ||= vcs.branch(revision[3]) + revision = revision[1] + end version = nil unless revision url = vcs.trunk @@ -173,74 +310,111 @@ def package(vcs, rev, destdir, tmp = nil) end revision = vcs.get_revisions(url)[1] end - v = nil - if $exported - if String === $exported - v = $exported - end - else - v = "ruby" - puts "Exporting #{rev}@#{revision}" - exported = tmp ? File.join(tmp, v) : v - unless vcs.export(revision, url, exported, true) {|line| print line} - warn("Export failed") - return - end - if $srcdir - Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*}") do |file| - puts "copying #{file}" - dest = exported + file[$srcdir.size..-1] - FileUtils.mkpath(File.dirname(dest)) - begin - FileUtils.ln(file, dest, force: true) - rescue SystemCallError - FileUtils.cp(file, dest, preserve: true) - end + + v = "ruby" + puts "Exporting #{rev}@#{revision}" + exported = tmp ? File.join(tmp, v) : v + unless vcs.export(revision, url, exported, true) {|line| print line} + warn("Export failed") + return + end + if $srcdir + Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file| + puts "copying #{file}" if $VERBOSE + dest = exported + file[$srcdir.size..-1] + FileUtils.mkpath(File.dirname(dest)) + begin + FileUtils.cp_r(file, dest) + FileUtils.chmod_R("a+rwX,go-w", dest) + rescue SystemCallError end end end + Dir.glob("#{exported}/.*.yml") do |file| + FileUtils.rm(file, verbose: $VERBOSE) + end + + status = File.read(File.dirname(__FILE__) + "/prereq.status") Dir.chdir(tmp) if tmp if !File.directory?(v) v = Dir.glob("ruby-*").select(&File.method(:directory?)) - v.size == 1 or abort "not exported" + v.size == 1 or abort "#{File.basename $0}: not exported" v = v[0] end - open("#{v}/revision.h", "wb") {|f| f.puts "#define RUBY_REVISION #{revision}"} - version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] - version or return - if patchlevel - unless tag.empty? - versionhdr ||= IO.read("#{v}/version.h") - patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1] - tag = (patchlevel ? "p#{patchlevel}" : "r#{revision}") + + File.open("#{v}/revision.h", "wb") {|f| + f.puts vcs.revision_header(revision, modified) + } + version ||= (versionhdr = File.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1] + version ||= + begin + include_ruby_versionhdr = File.read("#{v}/include/ruby/version.h") + api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1] + api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1] + version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1] + [api_major_version, api_minor_version, version_teeny].join('.') end - elsif prerelease - versionhdr ||= IO.read("#{v}/version.h") - versionhdr.sub!(/^\#define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) - IO.write("#{v}/version.h", versionhdr) + version or return + if prerelease + versionhdr ||= File.read("#{v}/version.h") + versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace" + File.write("#{v}/version.h", versionhdr) else - tag ||= "r#{revision}" + tag ||= vcs.revision_name(revision) end - unless v == $exported - if $archname - n = $archname - elsif tag.empty? - n = "ruby-#{version}" - else - n = "ruby-#{version}-#{tag}" - end - File.directory?(n) or File.rename v, n - v = n + + if $archname + n = $archname + elsif tag.empty? + n = "ruby-#{version}" + else + n = "ruby-#{version}-#{tag}" + end + File.directory?(n) or File.rename v, n + v = n + + if $patch_file && !system(*%W"patch -d #{v} -p0 -i #{$patch_file}") + puts $colorize.fail("patching failed") + return end - system(*%W"patch -d #{v} -p0 -i #{$patch_file}") if $patch_file - if !$exported or $patch_file - "take a breath, and go ahead".scan(/./) {|c|print c; sleep(c == "," ? 0.7 : 0.05)}; puts + + class << (clean = []) + def add(n) push(n) + n + end + def create(file, content = "", &block) + add(file) + if block + File.open(file, "wb", &block) + else + File.binwrite(file, content) + end + end end - def (clean = []).add(n) push(n); n end + Dir.chdir(v) do - File.open(clean.add("cross.rb"), "w") do |f| + unless File.exist?("ChangeLog") + vcs.export_changelog(url, nil, revision, "ChangeLog") + end + + unless touch_all(modified, "**/*", File::FNM_DOTMATCH) + modified = nil + colors = %w[red yellow green cyan blue magenta] + "take a breath, and go ahead".scan(/./) do |c| + if c == ' ' + print c + else + colors.push(color = colors.shift) + print $colorize.decorate(c, color) + end + sleep(c == "," ? 0.7 : 0.05) + end + puts + end + + clean.create("cross.rb") do |f| f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)" f.puts "CROSS_COMPILING=true" f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)" @@ -248,102 +422,131 @@ def package(vcs, rev, destdir, tmp = nil) f.puts "Object.__send__(:remove_const, :RUBY_VERSION)" f.puts "RUBY_VERSION='#{version}'" end + puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE unless File.exist?("configure") print "creating configure..." - unless system([ENV["AUTOCONF"]]*2) - puts " failed" + unless system(File.exist?(gen = "./autogen.sh") ? gen : [ENV["AUTOCONF"]]*2) + puts $colorize.fail(" failed") return end - puts " done" + puts $colorize.pass(" done") end clean.add("autom4te.cache") + clean.add("enc/unicode/data") print "creating prerequisites..." - if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk") + if File.file?("common.mk") && /^prereq/ =~ commonmk = File.read("common.mk") puts extout = clean.add('tmp') - File.open(clean.add("config.status"), "w") {|f| - f.puts "s,@configure_args@,|#_!!_#|,g" - f.puts "s,@EXTOUT@,|#_!!_#|#{extout},g" - f.puts "s,@bindir@,|#_!!_#|,g" - f.puts "s,@ruby_install_name@,|#_!!_#|,g" - f.puts "s,@ARCH_FLAG@,|#_!!_#|,g" - f.puts "s,@CFLAGS@,|#_!!_#|,g" - f.puts "s,@CPPFLAGS@,|#_!!_#|,g" - f.puts "s,@CXXFLAGS@,|#_!!_#|,g" - f.puts "s,@LDFLAGS@,|#_!!_#|,g" - f.puts "s,@DLDFLAGS@,|#_!!_#|,g" - f.puts "s,@LIBEXT@,|#_!!_#|a,g" - f.puts "s,@OBJEXT@,|#_!!_#|o,g" - f.puts "s,@EXEEXT@,|#_!!_#|,g" - f.puts "s,@LIBRUBY@,|#_!!_#|libruby.a,g" - f.puts "s,@LIBRUBY_A@,|#_!!_#|libruby.a,g" - f.puts "s,@RM@,|#_!!_#|rm -f,g" - f.puts "s,@CP@,|#_!!_#|cp,g" - f.puts "s,@rubyarchdir@,|#_!!_#|,g" - f.puts "s,@rubylibprefix@,|#_!!_#|,g" - f.puts "s,@ruby_version@,|#_!!_#|#{version},g" - } + begin + status = File.read("tool/prereq.status") + rescue Errno::ENOENT + # use fallback file + end + clean.create("config.status", status) + clean.create("noarch-fake.rb", "require_relative 'cross'\n") FileUtils.mkpath(hdrdir = "#{extout}/include/ruby") - File.open("#{hdrdir}/config.h", "w") {} + File.binwrite("#{hdrdir}/config.h", "") FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults") - File.open("#{defaults}/operating_system.rb", "w") {} - File.open("#{defaults}/ruby.rb", "w") {} + File.binwrite("#{defaults}/operating_system.rb", "") + File.binwrite("#{defaults}/ruby.rb", "") miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross" baseruby = ENV["BASERUBY"] - mk = IO.read("Makefile.in").gsub(/^@.*\n/, '') + mk = (File.read("template/Makefile.in") rescue File.read("Makefile.in")). + gsub(/^@.*\n/, '') vars = { - "srcdir"=>".", - "CHDIR"=>"cd", - "NULLCMD"=>":", + "EXTOUT"=>extout, "PATH_SEPARATOR"=>File::PATH_SEPARATOR, - "IFCHANGE"=>"tool/ifchange", - "MKDIR_P"=>"mkdir -p", - "RMALL"=>"rm -fr", "MINIRUBY"=>miniruby, - "RUNRUBY"=>miniruby, "RUBY"=>ENV["RUBY"], - "HAVE_BASERUBY"=>"yes", "BASERUBY"=>baseruby, - "BOOTSTRAPRUBY"=>baseruby, "PWD"=>Dir.pwd, - "CONFIGURE"=>"configure", + "ruby_version"=>version, + "MAJOR"=>api_major_version, + "MINOR"=>api_minor_version, + "TEENY"=>version_teeny, + "VPATH"=>(ENV["VPATH"] || "include/ruby"), + "PROGRAM"=>(ENV["PROGRAM"] || "ruby"), + "BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"), } + status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do + vars[$2] ||= $3 + end + vars.delete("UNICODE_FILES") # for stable branches + vars["UNICODE_VERSION"] = $unicode_version if $unicode_version args = vars.dup mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]} - mk << commonmk.gsub(/(?<!#)\{[^{}]*\}/, "") + commonmk.gsub!(/^!(?:include \$\(srcdir\)\/(.*))?/) do + if inc = $1 and File.exist?(inc) + File.binread(inc).gsub(/^!/, '# !') + else + "#" + end + end + mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile') mk << <<-'APPEND' -prereq: clean-cache $(CLEAN_CACHE) -clean-cache $(CLEAN_CACHE): after-update +update-download:: touch-unicode-files +prepare-package: prereq after-update +clean-cache: $(CLEAN_CACHE) after-update:: extract-gems -extract-gems: +extract-gems: update-gems +update-gems: +$(UNICODE_SRC_DATA_DIR)/.unicode-tables.time: +touch-unicode-files: APPEND - open(clean.add("Makefile"), "w") do |f| - f.puts mk + clean.create("Makefile", mk) + clean.create("revision.tmp") + clean.create(".revision.time") + ENV["CACHE_SAVE"] = "no" + make = MAKE.new(args) + return unless make.run("update-download") + clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk", "ext/ripper/y.output", ".revision.time") + Dir.glob("**/*") do |dest| + next unless File.symlink?(dest) + orig = File.expand_path(File.readlink(dest), File.dirname(dest)) + File.unlink(dest) + FileUtils.cp_r(orig, dest) + end + File.utime(modified, modified, *Dir.glob(["tool/config.{guess,sub}", "gems/*.gem", "tool"])) + return unless make.run("prepare-package") + return unless make.run("clean-cache") + if modified + new_time = modified + 2 + touch_all(new_time, "**/*", File::FNM_DOTMATCH) do |name, stat| + stat.mtime > modified unless clean.include?(name) + end + modified = new_time end - system("make", "prereq", *args.map {|arg| arg.join("=")}) - clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk") print "prerequisites" else - system(*%W"#{YACC} -o parse.c parse.y") + system(*%W[#{PATH.executable_env("YACC", "bison")} -o parse.c parse.y]) end vcs.after_export(".") if exported - FileUtils.rm_rf(clean) unless $keep_temp - FileUtils.rm_rf(".downloaded-cache") + clean.concat(Dir.glob("ext/**/autom4te.cache")) + clean.add(".downloaded-cache") if File.exist?("gems/bundled_gems") gems = Dir.glob("gems/*.gem") gems -= File.readlines("gems/bundled_gems").map {|line| - 'gems/'+line.split(' ').join('-')+'.gem' + next if /^\s*(?:#|$)/ =~ line + name, version, _ = line.split(' ') + "gems/#{name}-#{version}.gem" } - FileUtils.rm_f(gems) + clean.concat(gems) else - FileUtils.rm_rf("gems") + clean.add("gems") + end + FileUtils.rm_rf(clean) + if modified + touch_all(modified, "**/*/", 0) do |name, stat| + stat.mtime > modified + end + File.utime(modified, modified, ".") end unless $?.success? - puts " failed" + puts $colorize.fail(" failed") return end - puts " done" + puts $colorize.pass(" done") end if v == "." @@ -364,35 +567,60 @@ extract-gems: if tarball next if tarball.empty? else - tarball = "#{$archname||v}.tar" + tarball = ext == ".tar" ? file : "#{$archname||v}.tar" print "creating tarball... #{tarball}" - if system("tar", "cf", tarball, v) - puts " done" + if measure {tar_create(tarball, v)} + puts $colorize.pass(" done") + File.utime(modified, modified, tarball) if modified + next if tarball == file else - puts " failed" + puts $colorize.fail(" failed") tarball = "" next end end print "creating #{mesg} tarball... #{file}" - done = system(*cmd, tarball, out: file) + done = measure {system(*cmd, tarball, out: file)} else print "creating #{mesg} archive... #{file}" - done = system(*cmd, file, v) + if Hash === cmd.last + *cmd, opt = *cmd + cmd << file << v << opt + else + (cmd = cmd.dup) << file << v + end + done = measure {system(*cmd)} end if done - puts " done" + puts $colorize.pass(" done") file else - puts " failed" + puts $colorize.fail(" failed") nil end end.compact ensure - FileUtils.rm_rf(v) if v and !$exported and !$keep_temp + FileUtils.rm_rf(tmp ? File.join(tmp, v) : v) if v and !$keep_temp + Dir.chdir(pwd) end -vcs = (VCS.detect($srcdir) rescue nil if $srcdir) || VCS::SVN.new(SVNURL) +if [$srcdir, ($git||=nil)].compact.size > 1 + abort "#{File.basename $0}: -srcdir and -git are exclusive" +end +if $srcdir + vcs = VCS.detect($srcdir) +elsif $git + abort "#{File.basename $0}: use -srcdir with cloned local repository" +else + begin + vcs = VCS.detect(File.expand_path("../..", __FILE__)) + rescue VCS::NotFoundError + abort "#{File.expand_path("../..", __FILE__)}: cannot find git repository" + end +end + +release_date = Time.now.getutc +info = {} success = true revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| @@ -400,14 +628,42 @@ revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name| success = false next end - str = open(name, "rb") {|f| f.read} - puts "* #{name}" - puts " SIZE: #{str.bytesize} bytes" + str = File.binread(name) + pathname = Pathname(name) + basename = pathname.basename.to_s + extname = pathname.extname.sub(/\A\./, '') + version = basename[/\Aruby-(.*)\.(?:tar|zip)/, 1] + key = basename[/\A(.*)\.(?:tar|zip)/, 1] + info[key] ||= Hash.new{|h,k|h[k]={}} + info[key]['version'] = version if version + info[key]['date'] = release_date.to_date + if version + info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/" + info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}" + else + info[key]['filename'][extname] = basename + end + info[key]['size'][extname] = str.bytesize + puts "* #{$colorize.pass(name)}" + puts " SIZE: #{str.bytesize} bytes" $digests.each do |alg| - printf " %-8s%s\n", "#{alg}:", Digest.const_get(alg).hexdigest(str) + digest = Digest(alg).hexdigest(str) + info[key][alg.downcase][extname] = digest + printf " %-8s%s\n", "#{alg}:", digest end end +yaml = info.values.to_yaml +json = info.values.to_json +puts "#{$colorize.pass('YAML:')}" +puts yaml +puts "#{$colorize.pass('JSON:')}" +puts json +infodir = Pathname(destdir) + 'info' +infodir.mkpath +(infodir+'info.yml').write(yaml) +(infodir+'info.json').write(json) + exit false if !success # vim:fileencoding=US-ASCII sw=2 ts=4 noexpandtab ff=unix diff --git a/tool/make_hgraph.rb b/tool/make_hgraph.rb index 0f388814dd..174fa5dd2f 100644 --- a/tool/make_hgraph.rb +++ b/tool/make_hgraph.rb @@ -83,13 +83,12 @@ module ObjectSpace def self.module_refenreces_image klass, file dot = module_refenreces_dot(klass) - img = nil - IO.popen("dot -Tpng", 'r+'){|io| + img = IO.popen(%W"dot -Tpng", 'r+b') {|io| # io.puts dot io.close_write - img = io.read + io.read } - open(File.expand_path(file), 'w+'){|f| f.puts img} + File.binwrite(file, img) end end diff --git a/tool/mdoc2man.rb b/tool/mdoc2man.rb index 274f2d5d08..e005fcf19a 100755 --- a/tool/mdoc2man.rb +++ b/tool/mdoc2man.rb @@ -82,6 +82,24 @@ class Mdoc2Man initialize end + def shift_arg(words) + case words[0] + when nil, RE_PUNCT + nil + when /\A"(.+)/ + words.shift + word = $1 + loop { + break if word.chomp!('"') + token = words.shift or break + word << ' ' << token + } + word + else + words.shift + end + end + def parse_macro(line) words = line.split retval = '' @@ -92,6 +110,7 @@ class Mdoc2Man while word = words.shift case word when RE_PUNCT + next retval << word if word == ':' while q = quote.pop case q when OPTION @@ -235,6 +254,13 @@ class Mdoc2Man when 'Ux' retval << "UNIX" next + when 'Bro' + retval << '{' + @nospace = 1 if @nospace == 0 + next + when 'Brc' + retval.sub!(/ *\z/, '}') + next end if @reference @@ -312,15 +338,29 @@ class Mdoc2Man next end - if word == 'Pa' && !quote.include?(OPTION) - retval << '\\fI' - retval << '\\&' if /^\./ =~ words[0] - retval << words.shift << '\\fP' - while RE_PUNCT =~ words[0] - retval << words.shift - end - # @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] - next + case word + when 'Pa' + if !quote.include?(OPTION) + retval << '\\fI' + retval << '\\&' if /^\./ =~ words[0] + retval << words.shift << '\\fP' + while RE_PUNCT =~ words[0] + retval << words.shift + end + # @nospace = 1 if @nospace == 0 && RE_PUNCT =~ words[0] + next + end + when 'Lk' + if !quote.include?(OPTION) + url = words.shift + if name = shift_arg(words) + retval << '\\fI' << name << ':\\fP ' + end + retval << '\\fB' + retval << '\\&' if /\A\./ =~ url + retval << url << '\\fP' + next + end end case word @@ -400,7 +440,7 @@ class Mdoc2Man # tags retval << ".TP\n" case words[0] - when 'Pa', 'Ev' + when 'Pa', 'Ev', 'Lk' words.shift retval << '.B' end diff --git a/tool/merger.rb b/tool/merger.rb index 367815fef2..4c096087fc 100755 --- a/tool/merger.rb +++ b/tool/merger.rb @@ -2,24 +2,23 @@ # -*- ruby -*- exec "${RUBY-ruby}" "-x" "$0" "$@" && [ ] if false #!ruby -# This needs ruby 1.9 and subversion. -# run this in a repository to commit. +# This needs ruby 2.0 and Git. +# As a Ruby committer, run this in a git repository to commit a change. require 'tempfile' +require 'net/http' +require 'uri' +require 'shellwords' -$repos = 'svn+ssh://svn@ci.ruby-lang.org/ruby/' ENV['LC_ALL'] = 'C' +ORIGIN = 'git@git.ruby-lang.org:ruby.git' +GITHUB = 'git@github.com:ruby/ruby.git' -def help - puts <<-end +class << Merger = Object.new + def help + puts <<-HELP \e[1msimple backport\e[0m - ruby #$0 1234 - -\e[1mrange backport\e[0m - ruby #$0 1234:5678 - -\e[1mbackport from other branch\e[0m - ruby #$0 17502 mvm + ruby #$0 1234abc \e[1mrevision increment\e[0m ruby #$0 revisionup @@ -28,166 +27,242 @@ def help ruby #$0 teenyup \e[1mtagging major release\e[0m - ruby #$0 tag 2.2.0 + ruby #$0 tag 3.2.0 -\e[1mtagging patch release\e[0m (about 2.1.0 or later, it means X.Y.Z (Z > 0) release) +\e[1mtagging patch release\e[0m (for 2.1.0 or later, it means X.Y.Z (Z > 0) release) ruby #$0 tag \e[1mtagging preview/RC\e[0m - ruby #$0 tag 2.2.0-preview1 + ruby #$0 tag 3.2.0-preview1 + +\e[1mremove tag\e[0m + ruby #$0 removetag 3.2.9 \e[33;1m* all operations shall be applied to the working directory.\e[0m -end -end + HELP + end -# Prints the version of Ruby found in version.h - -def version - v = p = nil - open 'version.h', 'rb' do |f| - f.each_line do |l| - case l - when /^#define RUBY_VERSION "(\d)\.(\d)\.(\d)"$/ - v = $~.captures - when /^#define RUBY_PATCHLEVEL (-?\d+)$/ - p = $1 + def interactive(str, editfile = nil) + loop do + yield if block_given? + STDERR.puts "\e[1;33m#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})\e[0m" + case STDIN.gets + when /\Aa/i then exit 1 + when /\Ar/i then redo + when /\Ay/i then break + when /\Ae/i then system(ENV['EDITOR'], editfile) + else exit 1 end end end - return v, p -end -def interactive str, editfile = nil - loop do - yield - STDERR.puts "#{str} ([y]es|[a]bort|[r]etry#{'|[e]dit' if editfile})" - case STDIN.gets - when /\Aa/i then exit - when /\Ar/i then redo - when /\Ay/i then break - when /\Ae/i then system(ENV["EDITOR"], editfile) - else exit + def version_up(teeny: false) + now = Time.now + now = now.localtime(9*60*60) # server is Japan Standard Time +09:00 + system('git', 'checkout', 'HEAD', 'version.h') + v, pl = version + + if teeny + v[2].succ! + end + # We stopped bumping RUBY_PATCHLEVEL at Ruby 4.0.0. + if Integer(v[0]) <= 3 + pl.succ! + end + + str = open('version.h', 'rb', &:read) + ruby_release_date = str[/RUBY_RELEASE_YEAR_STR"-"RUBY_RELEASE_MONTH_STR"-"RUBY_RELEASE_DAY_STR/] || now.strftime('"%Y-%m-%d"') + [%W[RUBY_VERSION "#{v.join('.')}"], + %W[RUBY_VERSION_CODE #{v.join('')}], + %W[RUBY_VERSION_MAJOR #{v[0]}], + %W[RUBY_VERSION_MINOR #{v[1]}], + %W[RUBY_VERSION_TEENY #{v[2]}], + %W[RUBY_RELEASE_DATE #{ruby_release_date}], + %W[RUBY_RELEASE_CODE #{now.strftime('%Y%m%d')}], + %W[RUBY_PATCHLEVEL #{pl}], + %W[RUBY_RELEASE_YEAR #{now.year}], + %W[RUBY_RELEASE_MONTH #{now.month}], + %W[RUBY_RELEASE_DAY #{now.day}], + ].each do |(k, i)| + str.sub!(/^(#define\s+#{k}\s+).*$/, "\\1#{i}") + end + str.sub!(/\s+\z/m, '') + fn = sprintf('version.h.tmp.%032b', rand(1 << 31)) + File.rename('version.h', fn) + open('version.h', 'wb') do |f| + f.puts(str) end + File.unlink(fn) end -end -def version_up(inc=nil) - d = Time.now - d = d.localtime(9*60*60) # server is Japan Standard Time +09:00 - system(*%w'svn revert version.h') - v, pl = version + def tag(relname) + # relname: + # * 2.2.0-preview1 + # * 2.2.0-rc1 + # * 2.2.0 + v, pl = version + if relname + abort "patchlevel is not -1 but '#{pl}' for preview or rc" if pl != '-1' && /-(?:preview|rc)/ =~ relname + abort "patchlevel is not 0 but '#{pl}' for the first release" if pl != '0' && relname.end_with?(".0") + pl = relname[/-(.*)\z/, 1] + curver = "#{v.join('.')}#{("-#{pl}" if pl)}" + if relname != curver + abort "given relname '#{relname}' conflicts current version '#{curver}'" + end + else + if pl == '-1' + abort 'no relname is given and not in a release branch even if this is patch release' + end + end + if /^(?:preview|rc)/ =~ pl + tagname = "v#{v.join('.')}-#{pl}" + elsif Integer(v[0]) >= 4 + tagname = "v#{v.join('.')}" + else + tagname = "v#{v.join('_')}" + end - if inc == :teeny - v[2].succ! + unless execute('git', 'diff', '--exit-code') + abort 'uncommitted changes' + end + unless execute('git', 'tag', tagname) + abort 'specfied tag already exists. check tag name and remove it if you want to force re-tagging' + end + execute('git', 'push', ORIGIN, tagname, interactive: true) end - # patchlevel - if pl != "-1" - pl.succ! + + def remove_tag(relname) + # relname: + # * 2.2.0-preview1 + # * 2.2.0-rc1 + # * 2.2.0 + # * v2_2_0_preview1 + # * v2_2_0_rc1 + # * v2_2_0 + unless relname + raise ArgumentError, 'relname is not specified' + end + if relname.start_with?('v') + tagname = relname + elsif Integer(relname.split('.', 2).first) >= 4 + tagname = "v#{relname}" + else + tagname = "v#{relname.gsub(/[.-]/, '_')}" + end + + execute('git', 'tag', '-d', tagname) + execute('git', 'push', ORIGIN, ":#{tagname}", interactive: true) + execute('git', 'push', GITHUB, ":#{tagname}", interactive: true) end - str = open 'version.h', 'rb' do |f| f.read end - [%W[RUBY_VERSION "#{v.join '.'}"], - %W[RUBY_VERSION_CODE #{v.join ''}], - %W[RUBY_VERSION_MAJOR #{v[0]}], - %W[RUBY_VERSION_MINOR #{v[1]}], - %W[RUBY_VERSION_TEENY #{v[2]}], - %W[RUBY_RELEASE_DATE "#{d.strftime '%Y-%m-%d'}"], - %W[RUBY_RELEASE_CODE #{d.strftime '%Y%m%d'}], - %W[RUBY_PATCHLEVEL #{pl}], - %W[RUBY_RELEASE_YEAR #{d.year}], - %W[RUBY_RELEASE_MONTH #{d.month}], - %W[RUBY_RELEASE_DAY #{d.day}], - ].each do |(k, i)| - str.sub!(/^(#define\s+#{k}\s+).*$/, "\\1#{i}") + def update_revision_h + execute('ruby tool/file2lastrev.rb --revision.h . > revision.tmp') + execute('tool/ifchange', '--timestamp=.revision.time', 'revision.h', 'revision.tmp') + execute('rm', '-f', 'revision.tmp') end - str.sub!(/\s+\z/m, '') - fn = sprintf 'version.h.tmp.%032b', rand(1 << 31) - File.rename 'version.h', fn - open 'version.h', 'wb' do |f| - f.puts str + + def stat + `git status --short` end - File.unlink fn -end -def tag intv_p = false, relname=nil - # relname: - # * 2.2.0-preview1 - # * 2.2.0-rc1 - # * 2.2.0 - v, pl = version - x = v.join('_') - if relname - abort "patchlevel is not -1 but '#{pl}' for preview or rc" if pl != '-1' && /-(?:preview|rc)/ =~ relname - abort "patchlevel is not 0 but '#{pl}' for the first release" if pl != '0' && /-(?:preview|rc)/ !~ relname - pl = relname[/-(.*)\z/, 1] - curver = v.join('.') + (pl ? '-' + pl : '') - if relname != curver - abort "given relname '#{relname}' conflicts current version '#{curver}'" - end - branch_url = `svn info`[/URL: (.*)/, 1] - else - if pl == '-1' - abort "no relname is given and not in a release branch even if this is patch release" + def diff(file = nil) + command = %w[git diff --color HEAD] + IO.popen(command + [file].compact, &:read) + end + + def commit(file) + current_branch = IO.popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], &:read).strip + execute('git', 'add', '.') && execute('git', 'commit', '-F', file) + end + + def has_conflicts? + changes = IO.popen(%w[git status --porcelain -z]) { |io| io.readlines("\0", chomp: true) } + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # AA: unmerged, both added + conflict = changes.grep(/\A(?:.U|AA) /) {$'} + !conflict.empty? + end + + private + + # Prints the version of Ruby found in version.h + def version + v = p = nil + open 'version.h', 'rb' do |f| + f.each_line do |l| + case l + when /^#define RUBY_VERSION "(\d+)\.(\d+)\.(\d+)"$/ + v = $~.captures + when /^#define RUBY_VERSION_TEENY (\d+)$/ + (v ||= [])[2] = $1 + when /^#define RUBY_PATCHLEVEL (-?\d+)$/ + p = $1 + end + end end - branch_url = $repos + 'branches/ruby_' - if v[0] < "2" || (v[0] == "2" && v[1] < "1") - abort "patchlevel must be greater than 0 for patch release" if pl == "0" - branch_url << x - else - abort "teeny must be greater than 0 for patch release" if v[2] == "0" - branch_url << x.sub(/_\d+$/, '') + if v and !v[0] + open 'include/ruby/version.h', 'rb' do |f| + f.each_line do |l| + case l + when /^#define RUBY_API_VERSION_MAJOR (\d+)/ + v[0] = $1 + when /^#define RUBY_API_VERSION_MINOR (\d+)/ + v[1] = $1 + end + end + end end + return v, p end - tagname = 'v' + x + (v[0] < "2" || (v[0] == "2" && v[1] < "1") || /^(?:preview|rc)/ =~ pl ? '_' + pl : '') - tag_url = $repos + 'tags/' + tagname - if intv_p - interactive "OK? svn cp -m \"add tag #{tagname}\" #{branch_url} #{tag_url}" do + + def execute(*cmd, interactive: false) + if interactive + Merger.interactive("OK?: #{cmd.shelljoin}") end + puts "+ #{cmd.shelljoin}" + system(*cmd) end - system(*%w'svn cp -m', "add tag #{tagname}", branch_url, tag_url) -end - -def default_merge_branch - %r{^URL: .*/branches/ruby_1_8_} =~ `svn info` ? 'branches/ruby_1_8' : 'trunk' end case ARGV[0] when "teenyup" - version_up(:teeny) - system 'svn diff version.h' -when "up", /\A(ver|version|rev|revision|lv|level|patch\s*level)\s*up/ - version_up - system 'svn diff version.h' + Merger.version_up(teeny: true) + puts Merger.diff('version.h') +when "up", /\A(ver|version|rev|revision|lv|level|patch\s*level)\s*up\z/ + Merger.version_up + puts Merger.diff('version.h') when "tag" - tag :interactive, ARGV[1] + Merger.tag(ARGV[1]) +when /\A(?:remove|rm|del)_?tag\z/ + Merger.remove_tag(ARGV[1]) when nil, "-h", "--help" - help + Merger.help exit else - system 'svn up' + Merger.update_revision_h - if /--ticket=(.*)/ =~ ARGV[0] - tickets = $1.split(/,/).map{|num| " [Backport ##{num}]"} + case ARGV[0] + when /--ticket=(.*)/ + tickets = $1.split(/,/) ARGV.shift else tickets = [] + detect_ticket = true end - q = $repos + (ARGV[1] || default_merge_branch) - revstr = ARGV[0].delete('^, :\-0-9') + revstr = ARGV[0].gsub(%r!https://github\.com/ruby/ruby/commit/|https://bugs\.ruby-lang\.org/projects/ruby-master/repository/git/revisions/!, '') + revstr = revstr.delete('^, :\-0-9a-fA-F') revs = revstr.split(/[,\s]+/) - log = '' - log_svn = '' + commit_message = '' revs.each do |rev| + git_rev = nil case rev - when /\A\d+:\d+\z/ - r = ['-r', rev] - when /\A(\d+)-(\d+)\z/ - rev = "#{$1.to_i-1}:#$2" - r = ['-r', rev] - when /\A\d+\z/ - r = ['-c', rev] + when /\A\h{7,40}\z/ + git_rev = rev when nil then puts "#$0 revision" exit @@ -196,66 +271,54 @@ else exit end - l = IO.popen %w'svn diff' + r + %w'--diff-cmd=diff -x -pU0' + [File.join(q, 'ChangeLog')] do |f| - f.read + # Merge revision from Git patch + git_uri = "https://github.com/ruby/ruby/commit/#{git_rev}.patch" + resp = Net::HTTP.get_response(URI(git_uri)) + if resp.code != '200' + abort "'#{git_uri}' returned status '#{resp.code}':\n#{resp.body}" end + patch = resp.body.sub(/^diff --git a\/version\.h b\/version\.h\nindex .*\n--- a\/version\.h\n\+\+\+ b\/version\.h\n@@ .* @@\n(?:[-\+ ].*\n|\n)+/, '') - log << l - log_svn << l.lines.grep(/^\+\t/).join.gsub(/^\+/, '').gsub(/^\t\*/, "\n\t\*") - - if log_svn.empty? - log_svn = IO.popen %w'svn log ' + r + [q] do |f| - f.read - end.sub(/\A-+\nr.*\n/, '').sub(/\n-+\n\z/, '').gsub(/^(?=\S)/, "\t") + if detect_ticket + tickets += patch.scan(/\[(?:Bug|Feature|Misc) #(\d+)\]/i).map(&:first) end - a = %w'svn merge --accept=postpone' + r + [q] - STDERR.puts a.join(' ') + message = "#{(patch[/^Subject: (.*)\n---\n /m, 1] || "Message not found for revision: #{git_rev}\n")}" + message.gsub!(/\G(.*)\n( .*)/, "\\1\\2") + message = "\n\n#{message}" - system(*a) - system(*%w'svn revert ChangeLog') if /^\+/ =~ l - end + puts '+ git apply' + IO.popen(['git', 'apply', '--3way'], 'wb') { |f| f.write(patch) } - if `svn diff --diff-cmd=diff -x -upw`.empty? - interactive 'Only ChangeLog is modified, right?' do - end + commit_message << message.sub(/\A-+\nr.*/, '').sub(/\n-+\n\z/, '').gsub(/^./, "\t\\&") end - if /^\+/ =~ log - system(*%w'svn revert ChangeLog') - IO.popen %w'patch -p0', 'wb' do |f| - f.write log.gsub(/\+(Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [ 123][0-9] [012][0-9]:[0-5][0-9]:[0-5][0-9] \d\d\d\d/, - # this format-time-string was from the file local variables of ChangeLog - '+'+Time.now.strftime('%a %b %e %H:%M:%S %Y')) - end - system(*%w'touch ChangeLog') # needed somehow, don't know why... - else - STDERR.puts '*** You should write ChangeLog NOW!!! ***' + if Merger.diff.empty? + Merger.interactive('Nothing is modified, right?') end - version_up + Merger.version_up f = Tempfile.new 'merger.rb' - f.printf "merge revision(s) %s:%s\n", revstr, tickets.join - f.write log_svn + f.printf "merge revision(s) %s:%s", revs.join(', '), tickets.map{|num| " [Backport ##{num}]"}.join + f.write commit_message f.flush f.close - interactive 'conflicts resolved?', f.path do - IO.popen(ENV["PAGER"] || "less", "w") do |g| - g << `svn stat` - g << "\n\n" - f.open - g << f.read - f.close - g << "\n\n" - g << `svn diff --diff-cmd=diff -x -upw` + if Merger.has_conflicts? + Merger.interactive('conflicts resolved?', f.path) do + IO.popen(ENV['PAGER'] || ['less', '-R'], 'w') do |g| + g << Merger.stat + g << "\n\n" + f.open + g << f.read + f.close + g << "\n\n" + g << Merger.diff + end end end - if system(*%w'svn ci -F', f.path) - # tag :interactive # no longer needed. - system 'rm -f subversion.commitlog' - else + unless Merger.commit(f.path) puts 'commit failed; try again.' end diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat new file mode 100755 index 0000000000..d39568fe86 --- /dev/null +++ b/tool/missing-baseruby.bat @@ -0,0 +1,30 @@ +:"" == " +@echo off || ( + :warn + echo>&2.%~1 + goto :eof + :abort + exit /b 1 +)||( +:)"||( + # necessary libraries + require 'erb' + require 'fileutils' + require 'tempfile' + s = %^# +) +: ; call() { local call=${1#:}; shift; $call "$@"; } +: ; warn() { echo "$1" >&2; } +: ; abort () { exit 1; } + +call :warn "executable host ruby is required. use --with-baseruby option." +call :warn "Note that BASERUBY must be Ruby 3.1.0 or later." +call :abort +(goto :eof ^;) +verbose = true if ARGV[0] == "--verbose" +case +when !defined?(RubyVM::InstructionSequence) + abort(*(["BASERUBY must be CRuby"] if verbose)) +when RUBY_VERSION < s[%r[warn .*\KBASERUBY .*Ruby ([\d.]+)(?:\.0)?.*(?=\")],1] + abort(*(["#{$&}. Found: #{RUBY_VERSION}"] if verbose)) +end diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb new file mode 100644 index 0000000000..5aa07962f9 --- /dev/null +++ b/tool/mk_builtin_loader.rb @@ -0,0 +1,429 @@ +# Parse built-in script and make rbinc file + +require 'ripper' +require 'stringio' +require_relative 'ruby_vm/helpers/c_escape' + +SUBLIBS = {} +REQUIRED = {} +BUILTIN_ATTRS = %w[leaf inline_block use_block c_trace] + +module CompileWarning + @@warnings = 0 + + def warn(message) + @@warnings += 1 + super + end + + def self.reset + w, @@warnings = @@warnings, 0 + w.nonzero? + end +end + +Warning.extend CompileWarning + +def string_literal(lit, str = []) + while lit + case lit.first + when :string_concat, :string_embexpr, :string_content + _, *lit = lit + lit.each {|s| string_literal(s, str)} + return str + when :string_literal + _, lit = lit + when :@tstring_content + str << lit[1] + return str + else + raise "unexpected #{lit.first}" + end + end +end + +# e.g. [:symbol_literal, [:symbol, [:@ident, "inline", [19, 21]]]] +def symbol_literal(lit) + symbol_literal, symbol_lit = lit + raise "#{lit.inspect} was not :symbol_literal" if symbol_literal != :symbol_literal + symbol, ident_lit = symbol_lit + raise "#{symbol_lit.inspect} was not :symbol" if symbol != :symbol + ident, symbol_name, = ident_lit + raise "#{ident.inspect} was not :@ident" if ident != :@ident + symbol_name +end + +def inline_text argc, arg1 + raise "argc (#{argc}) of inline! should be 1" unless argc == 1 + arg1 = string_literal(arg1) + raise "1st argument should be string literal" unless arg1 + arg1.join("").rstrip +end + +def inline_attrs(args) + raise "args was empty" if args.empty? + args.each do |arg| + attr = symbol_literal(arg) + unless BUILTIN_ATTRS.include?(attr) + raise "attr (#{attr}) was not in: #{BUILTIN_ATTRS.join(', ')}" + end + end +end + +def make_cfunc_name inlines, name, lineno + case name + when /\[\]/ + name = '_GETTER' + when /\[\]=/ + name = '_SETTER' + else + name = name.tr('!?', 'EP') + end + + base = "builtin_inline_#{name}_#{lineno}" + if inlines[base] + 1000.times{|i| + name = "#{base}_#{i}" + return name unless inlines[name] + } + raise "too many functions in same line..." + else + base + end +end + +def collect_locals tree + _type, name, (line, _cols) = tree + if locals = LOCALS_DB[[name, line]] + locals + else + if false # for debugging + pp LOCALS_DB + raise "not found: [#{name}, #{line}]" + end + end +end + +def collect_builtin base, tree, name, bs, inlines, locals = nil + while tree + recv = sep = mid = args = nil + case tree.first + when :def + locals = collect_locals(tree[1]) + tree = tree[3] + next + when :defs + locals = collect_locals(tree[3]) + tree = tree[5] + next + when :class + name = 'class' + tree = tree[3] + next + when :sclass, :module + name = 'class' + tree = tree[2] + next + when :method_add_arg + _method_add_arg, mid, (_arg_paren, args) = tree + case mid.first + when :call + _, recv, sep, mid = mid + when :fcall + _, mid = mid + else + mid = nil + end + # w/ trailing comma: [[:method_add_arg, ...]] + # w/o trailing comma: [:args_add_block, [[:method_add_arg, ...]], false] + if args && args.first == :args_add_block + args = args[1] + end + when :vcall + _, mid = tree + when :command # FCALL + _, mid, (_, args) = tree + when :call, :command_call # CALL + _, recv, sep, mid, (_, args) = tree + end + + if mid + raise "unknown sexp: #{mid.inspect}" unless %i[@ident @const].include?(mid.first) + _, mid, (lineno,) = mid + if recv + func_name = nil + case recv.first + when :var_ref + _, recv = recv + if recv.first == :@const and recv[1] == "Primitive" + func_name = mid.to_s + end + when :vcall + _, recv = recv + if recv.first == :@ident and recv[1] == "__builtin" + func_name = mid.to_s + end + end + collect_builtin(base, recv, name, bs, inlines) unless func_name + else + func_name = mid[/\A__builtin_(.+)/, 1] + end + if func_name + cfunc_name = func_name + args.pop unless (args ||= []).last + argc = args.size + + if /(.+)[\!\?]\z/ =~ func_name + case $1 + when 'attr' + # Compile-time validation only. compile.c will parse them. + inline_attrs(args) + break + when 'cstmt' + text = inline_text argc, args.first + + func_name = "_bi#{lineno}" + cfunc_name = make_cfunc_name(inlines, name, lineno) + inlines[cfunc_name] = [lineno, text, locals, func_name] + argc -= 1 + when 'cexpr', 'cconst' + text = inline_text argc, args.first + code = "return #{text};" + + func_name = "_bi#{lineno}" + cfunc_name = make_cfunc_name(inlines, name, lineno) + + locals = [] if $1 == 'cconst' + inlines[cfunc_name] = [lineno, code, locals, func_name] + argc -= 1 + when 'cinit' + text = inline_text argc, args.first + func_name = nil # required + inlines[inlines.size] = [lineno, text, nil, nil] + argc -= 1 + when 'mandatory_only' + func_name = nil + when 'arg' + argc == 1 or raise "unexpected argument number #{argc}" + (arg = args.first)[0] == :symbol_literal or raise "symbol literal expected #{args}" + (arg = arg[1])[0] == :symbol or raise "symbol expected #{arg}" + (var = arg[1] and var = var[1]) or raise "argument name expected #{arg}" + func_name = nil + end + end + + if bs[func_name] && + bs[func_name] != [argc, cfunc_name] + raise "same builtin function \"#{func_name}\", but different arity (was #{bs[func_name]} but #{argc})" + end + + bs[func_name] = [argc, cfunc_name] if func_name + elsif /\Arequire(?:_relative)\z/ =~ mid and args.size == 1 and + (arg1 = args[0])[0] == :string_literal and + (arg1 = arg1[1])[0] == :string_content and + (arg1 = arg1[1])[0] == :@tstring_content and + sublib = arg1[1] + if File.exist?(f = File.join(@dir, sublib)+".rb") + puts "- #{@base}.rb requires #{sublib}" + if REQUIRED[sublib] + warn "!!! #{sublib} is required from #{REQUIRED[sublib]} already; ignored" + else + REQUIRED[sublib] = @base + (SUBLIBS[@base] ||= []) << sublib + end + ARGV.push(f) + end + end + break unless tree = args + end + + tree.each do |t| + collect_builtin base, t, name, bs, inlines, locals if Array === t + end + break + end +end + +# ruby mk_builtin_loader.rb TARGET_FILE.rb +# #=> generate TARGET_FILE.rbinc +# + +LOCALS_DB = {} # [method_name, first_line] = locals + +def collect_iseq iseq_ary + # iseq_ary.each_with_index{|e, i| p [i, e]} + label = iseq_ary[5] + first_line = iseq_ary[8] + type = iseq_ary[9] + locals = iseq_ary[10] + insns = iseq_ary[13] + + if type == :method + LOCALS_DB[[label, first_line].freeze] = locals + end + + insns.each{|insn| + case insn + when Integer + # ignore + when Array + # p insn.shift # insn name + insn.each{|op| + if Array === op && op[0] == "YARVInstructionSequence/SimpleDataFormat" + collect_iseq op + end + } + end + } +end + +def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) + f = StringIO.new + + # Avoid generating fetches of lvars we don't need. This is imperfect as it + # will match text inside strings or other false positives. + local_ptrs = [] + local_candidates = text.gsub(/\bLOCAL_PTR\(\K[a-zA-Z_][a-zA-Z0-9_]*(?=\))/) { + local_ptrs << $&; '' + }.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + + f.puts '{' + lineno += 1 + # locals is nil outside methods + locals&.reverse_each&.with_index{|param, i| + next unless Symbol === param + param = param.to_s + lvar = local_candidates.include?(param) + next unless lvar or local_ptrs.include?(param) + f.puts "VALUE *const #{param}__ptr = (VALUE *)&ec->cfp->ep[#{-3 - i}];" + f.puts "MAYBE_UNUSED(const VALUE) #{param} = *#{param}__ptr;" if lvar + lineno += lvar ? 2 : 1 + } + f.puts "#line #{body_lineno} \"#{line_file}\"" + lineno += 1 + + f.puts text + lineno += text.count("\n") + 1 + + f.puts "#line #{lineno + 2} \"#{ofile}\"" # TODO: restore line number. + f.puts "}" + f.puts + lineno += 3 + + return lineno, f.string +end + +def mk_builtin_header file + @dir = File.dirname(file) + base = File.basename(file, '.rb') + @base = base + ofile = "#{file}inc" + + # bs = { func_name => argc } + code = File.read(file) + begin + verbose, $VERBOSE = $VERBOSE, true + collect_iseq RubyVM::InstructionSequence.compile(code, base).to_a + ensure + $VERBOSE = verbose + end + if warnings = CompileWarning.reset + raise "#{warnings} warnings in #{file}" + end + collect_builtin(base, Ripper.sexp(code), 'top', bs = {}, inlines = {}) + + StringIO.open do |f| + if File::ALT_SEPARATOR + file = file.tr(File::ALT_SEPARATOR, File::SEPARATOR) + ofile = ofile.tr(File::ALT_SEPARATOR, File::SEPARATOR) + end + lineno = __LINE__ + f.puts "// -*- c -*-" + f.puts "// DO NOT MODIFY THIS FILE DIRECTLY." + f.puts "// auto-generated file" + f.puts "// by #{__FILE__}" + f.puts "// with #{file}" + f.puts '#include "internal/compilers.h" /* for MAYBE_UNUSED */' + f.puts '#include "internal/warnings.h" /* for COMPILER_WARNING_PUSH */' + f.puts '#include "ruby/ruby.h" /* for VALUE */' + f.puts '#include "builtin.h" /* for RB_BUILTIN_FUNCTION */' + f.puts 'struct rb_execution_context_struct; /* in vm_core.h */' + f.puts + lineno = __LINE__ - lineno - 1 + line_file = file + + inlines.each{|cfunc_name, (body_lineno, text, locals, func_name)| + if String === cfunc_name + f.puts "static VALUE #{cfunc_name}(struct rb_execution_context_struct *ec, const VALUE self)" + lineno += 1 + lineno, str = generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) + f.write str + else + # cinit! + f.puts "#line #{body_lineno} \"#{line_file}\"" + lineno += 1 + f.puts text + lineno += text.count("\n") + 1 + f.puts "#line #{lineno + 2} \"#{ofile}\"" # TODO: restore line number. + lineno += 1 + end + } + + if SUBLIBS[base] + f.puts "// sub libraries" + SUBLIBS[base].each do |sub| + f.puts %[#include #{(sub+".rbinc").dump}] + end + f.puts + end + + f.puts "void Init_builtin_#{base}(void)" + f.puts "{" + + table = "#{base}_table" + f.puts " // table definition" + f.puts " static const struct rb_builtin_function #{table}[] = {" + bs.each.with_index{|(func, (argc, cfunc_name)), i| + f.puts " RB_BUILTIN_FUNCTION(#{i}, #{func}, #{cfunc_name}, #{argc})," + } + f.puts " RB_BUILTIN_FUNCTION(-1, NULL, NULL, 0)," + f.puts " };" + + f.puts + f.puts " // arity_check" + f.puts "COMPILER_WARNING_PUSH" + f.puts "#if GCC_VERSION_SINCE(5, 1, 0) || defined __clang__" + f.puts "COMPILER_WARNING_ERROR(-Wincompatible-pointer-types)" + f.puts "#endif" + bs.each{|func, (argc, cfunc_name)| + f.puts " if (0) rb_builtin_function_check_arity#{argc}(#{cfunc_name});" + } + f.puts "COMPILER_WARNING_POP" + + if SUBLIBS[base] + f.puts + f.puts " // sub libraries" + SUBLIBS[base].each do |sub| + f.puts " Init_builtin_#{sub}();" + end + end + + f.puts + f.puts " // load" + f.puts " rb_load_with_builtin_functions(#{base.dump}, #{table});" + + f.puts "}" + + begin + File.write(ofile, f.string) + rescue SystemCallError # EACCES, EPERM, EROFS, etc. + # Fall back to the current directory + File.write(File.basename(ofile), f.string) + end + end +end + +ARGV.each{|file| + # feature.rb => load_feature.inc + mk_builtin_header file +} diff --git a/tool/mk_call_iseq_optimized.rb b/tool/mk_call_iseq_optimized.rb deleted file mode 100644 index 4cfde2c00e..0000000000 --- a/tool/mk_call_iseq_optimized.rb +++ /dev/null @@ -1,72 +0,0 @@ - -puts <<EOS -#if 1 /* enable or disable this optimization */ - -/* DO NOT EDIT THIS FILE DIRECTLY - * - * This file is enerated by tool/mkcall_iseq.rb - */ - -EOS - -P = (0..3) -L = (1..6) - -def fname param, local - "vm_call_iseq_setup_normal_0start_#{param}params_#{local}locals" -end - -P.each{|param| - L.each{|local| - puts <<EOS -static VALUE -#{fname(param, local)}(rb_thread_t *th, rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_call_info *ci, struct rb_call_cache *cc) -{ - return vm_call_iseq_setup_normal(th, cfp, calling, ci, cc, 0, #{param}, #{local}); -} - -EOS - # - } -} - -puts <<EOS -/* vm_call_iseq_handlers[param][local] */ -static const vm_call_handler vm_call_iseq_handlers[][#{L.to_a.size}] = { -#{P.map{|param| '{' + L.map{|local| fname(param, local)}.join(",\n ") + '}'}.join(",\n")} -}; - -static inline vm_call_handler -vm_call_iseq_setup_func(const struct rb_call_info *ci, const int param_size, const int local_size) -{ - if (UNLIKELY(ci->flag & VM_CALL_TAILCALL)) { - return vm_call_iseq_setup_tailcall_0start; - } - else if (0) { /* to disable optimize */ - return vm_call_iseq_setup_normal_0start; - } - else { - if (param_size <= #{P.end} && - local_size <= #{L.end}) { - VM_ASSERT(local_size != 0); - return vm_call_iseq_handlers[param_size][local_size-1]; - } - return vm_call_iseq_setup_normal_0start; - } -} - -#else - - -static inline vm_call_handler -vm_call_iseq_setup_func(const struct rb_call_info *ci, struct rb_call_cache *cc) -{ - if (UNLIKELY(ci->flag & VM_CALL_TAILCALL)) { - return vm_call_iseq_setup_tailcall_0start; - } - else { - return vm_call_iseq_setup_normal_0start; - } -} -#endif -EOS diff --git a/tool/mk_rbbin.rb b/tool/mk_rbbin.rb new file mode 100755 index 0000000000..991230f094 --- /dev/null +++ b/tool/mk_rbbin.rb @@ -0,0 +1,48 @@ +#!ruby -s + +OPTIMIZATION = { + inline_const_cache: true, + peephole_optimization: true, + tailcall_optimization: false, + specialized_instruction: true, + operands_unification: true, + instructions_unification: true, + frozen_string_literal: true, + debug_frozen_string_literal: false, + coverage_enabled: false, + debug_level: 0, +} + +file = File.basename(ARGV[0], ".rb") +name = "<internal:#{file}>" +iseq = RubyVM::InstructionSequence.compile(ARGF.read, name, name, **OPTIMIZATION) +puts <<C +/* -*- C -*- */ + +static const char #{file}_builtin[] = { +C +iseq.to_binary.bytes.each_slice(8) do |b| + print " ", b.map {|c| "0x%.2x," % c}.join(" ") + if $comment + print " /* ", b.pack("C*").gsub(/([[ -~]&&[^\\]])|(?m:.)/) { + (c = $1) ? "#{c} " : (c = $&.dump).size == 2 ? c : ". " + }, "*/" + end + puts +end +puts <<C +}; + +#include "ruby/ruby.h" +#include "vm_core.h" + +void +Init_#{file}(void) +{ + const char *builtin = #{file}_builtin; + size_t size = sizeof(#{file}_builtin); + VALUE code = rb_str_new_static(builtin, (long)size); + VALUE iseq = rb_funcallv(rb_cISeq, rb_intern_const("load_from_binary"), 1, &code); + rb_funcallv(iseq, rb_intern_const("eval"), 0, 0); +} +C diff --git a/tool/mkconfig.rb b/tool/mkconfig.rb index b37241df61..db74115730 100755 --- a/tool/mkconfig.rb +++ b/tool/mkconfig.rb @@ -1,30 +1,25 @@ #!./miniruby -s +# frozen-string-literal: true + +# This script, which is run when ruby is built, generates rbconfig.rb by +# parsing information from config.status. rbconfig.rb contains build +# information for ruby (compiler flags, paths, etc.) and is used e.g. by +# mkmf to build compatible native extensions. # avoid warnings with -d. $install_name ||= nil $so_name ||= nil +$unicode_version ||= nil +$unicode_emoji_version ||= nil arch = $arch or raise "missing -arch" version = $version or raise "missing -version" srcdir = File.expand_path('../..', __FILE__) -$:.replace [srcdir+"/lib"] unless defined?(CROSS_COMPILING) $:.unshift(".") -require "fileutils" mkconfig = File.basename($0) -rbconfig_rb = ARGV[0] || 'rbconfig.rb' -unless File.directory?(dir = File.dirname(rbconfig_rb)) - FileUtils.makedirs(dir, :verbose => true) -end - -config = "" -def config.write(arg) - concat(arg.to_s) -end -$stdout = config - -fast = {'prefix'=>TRUE, 'ruby_install_name'=>TRUE, 'INSTALL'=>TRUE, 'EXEEXT'=>TRUE} +fast = {'prefix'=>true, 'ruby_install_name'=>true, 'INSTALL'=>true, 'EXEEXT'=>true} win32 = /mswin/ =~ arch universal = /universal.*darwin/ =~ arch @@ -35,6 +30,7 @@ continued_name = nil continued_line = nil install_name = nil so_name = nil +platform = nil File.foreach "config.status" do |line| next if /^#/ =~ line name = nil @@ -75,6 +71,7 @@ File.foreach "config.status" do |line| when /^arch$/; if val.empty? then val = arch else arch = val end when /^sitearch$/; val = '$(arch)' if val.empty? when /^DESTDIR$/; next + when /RUBYGEMS/; next end case val when /^\$\(ac_\w+\)$/; next @@ -90,7 +87,7 @@ File.foreach "config.status" do |line| unless $install_name $install_name = "ruby" val.gsub!(/\$\$/, '$') - val.scan(%r[\G[\s;]*(/(?:\\.|[^/])*/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*)\3((?:(?!\3)(?:\\.|.))*)\3([gi]*)]) do + val.scan(%r[\G[\s;]*(/(?:\\.|[^/])*+/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*+)\3((?:(?!\3)(?:\\.|.))*+)\3([gi]*)]) do |addr, cmd, sep, pat, rep, opt| if addr Regexp.new(addr[/\A\/(.*)\/\z/, 1]) =~ $install_name or next @@ -124,8 +121,16 @@ File.foreach "config.status" do |line| universal, val = val, 'universal' if universal when /^arch$/ if universal - val.sub!(/universal/, %q[#{arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || '\&'}]) + platform = val.sub(/universal/, '$(arch)') + end + when /^target_cpu$/ + if universal + val = 'cpu' end + when /^target$/ + val = '"$(target_cpu)-$(target_vendor)-$(target_os)"' + when /^host(?:_(?:os|vendor|cpu|alias))?$/ + val = %["$(#{name.sub(/^host/, 'target')})"] when /^includedir$/ val = '"$(SDKROOT)"'+val if /darwin/ =~ arch end @@ -135,10 +140,10 @@ File.foreach "config.status" do |line| else v_others << v end - case name - when "RUBY_PROGRAM_VERSION" - version = val[/\A"(.*)"\z/, 1] - end + #case name + #when "RUBY_PROGRAM_VERSION" + # version = val[/\A"(.*)"\z/, 1] + #end end # break if /^CEOF/ end @@ -163,45 +168,74 @@ def vars.expand(val, config = self) val.replace(newval) unless newval == val val end -prefix = vars.expand(vars["prefix"] ||= "") -rubyarchdir = vars.expand(vars["rubyarchdir"] ||= "") +prefix = vars.expand(vars["prefix"] ||= +"") +rubyarchdir = vars.expand(vars["rubyarchdir"] ||= +"") relative_archdir = rubyarchdir.rindex(prefix, 0) ? rubyarchdir[prefix.size..-1] : rubyarchdir + puts %[\ -# This file was created by #{mkconfig} when ruby was built. Any -# changes made to this file will be lost the next time ruby is built. +# encoding: ascii-8bit +# frozen-string-literal: false +# +# The module storing Ruby interpreter configurations on building. +# +# This file was created by #{mkconfig} when ruby was built. It contains +# build information for ruby which is used e.g. by mkmf to build +# compatible native extensions. Any changes made to this file will be +# lost the next time ruby is built. module RbConfig RUBY_VERSION.start_with?("#{version[/^[0-9]+\.[0-9]+\./] || version}") or raise "ruby lib version (#{version}) doesn't match executable version (\#{RUBY_VERSION})" ] +print " # Ruby installed directory.\n" print " TOPDIR = File.dirname(__FILE__).chomp!(#{relative_archdir.dump})\n" +print " # DESTDIR on make install.\n" print " DESTDIR = ", (drive ? "TOPDIR && TOPDIR[/\\A[a-z]:/i] || " : ""), "'' unless defined? DESTDIR\n" -print <<'ARCH' if universal +print <<"UNIVERSAL", <<'ARCH' if universal + universal = #{universal} +UNIVERSAL arch_flag = ENV['ARCHFLAGS'] || ((e = ENV['RC_ARCHS']) && e.split.uniq.map {|a| "-arch #{a}"}.join(' ')) arch = arch_flag && arch_flag[/\A\s*-arch\s+(\S+)\s*\z/, 1] + cpu = arch && universal[/(?:\A|\s)#{Regexp.quote(arch)}=(\S+)/, 1] || RUBY_PLATFORM[/\A[^-]*/] ARCH -print " universal = #{universal}\n" if universal +print " # The hash configurations stored.\n" print " CONFIG = {}\n" print " CONFIG[\"DESTDIR\"] = DESTDIR\n" versions = {} -IO.foreach(File.join(srcdir, "version.h")) do |l| +File.foreach(File.join(srcdir, "version.h")) do |l| m = /^\s*#\s*define\s+RUBY_(PATCHLEVEL)\s+(-?\d+)/.match(l) if m versions[m[1]] = m[2] - break + break if versions.size == 4 + next end -end -IO.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| - m = /^\s*#\s*define\s+RUBY_API_VERSION_(MAJOR|MINOR|TEENY)\s+(-?\d+)/.match(l) + m = /^\s*#\s*define\s+RUBY_VERSION_(\w+)\s+(-?\d+)/.match(l) if m versions[m[1]] = m[2] break if versions.size == 4 + next + end + m = /^\s*#\s*define\s+RUBY_VERSION\s+\W?([.\d]+)/.match(l) + if m + versions['MAJOR'], versions['MINOR'], versions['TEENY'] = m[1].split('.') + break if versions.size == 4 + next + end +end +if versions.size != 4 + File.foreach(File.join(srcdir, "include/ruby/version.h")) do |l| + m = /^\s*#\s*define\s+RUBY_API_VERSION_(\w+)\s+(-?\d+)/.match(l) + if m + versions[m[1]] ||= m[2] + break if versions.size == 4 + next + end end end %w[MAJOR MINOR TEENY PATCHLEVEL].each do |v| - print " CONFIG[#{v.dump}] = #{versions[v].dump}\n" + print " CONFIG[#{v.dump}] = #{(versions[v]||vars[v]).dump}\n" end dest = drive ? %r'= "(?!\$[\(\{])(?i:[a-z]:)' : %r'= "(?!\$[\(\{])' @@ -222,14 +256,14 @@ end v_others.compact! if $install_name - if install_name and vars.expand("$(RUBY_INSTALL_NAME)") == $install_name + if install_name and vars.expand(+"$(RUBY_INSTALL_NAME)") == $install_name $install_name = install_name end v_fast << " CONFIG[\"ruby_install_name\"] = \"" + $install_name + "\"\n" v_fast << " CONFIG[\"RUBY_INSTALL_NAME\"] = \"" + $install_name + "\"\n" end if $so_name - if so_name and vars.expand("$(RUBY_SO_NAME)") == $so_name + if so_name and vars.expand(+"$(RUBY_SO_NAME)") == $so_name $so_name = so_name end v_fast << " CONFIG[\"RUBY_SO_NAME\"] = \"" + $so_name + "\"\n" @@ -237,14 +271,64 @@ end print(*v_fast) print(*v_others) -print <<EOS if /darwin/ =~ arch - CONFIG["SDKROOT"] = ENV["SDKROOT"] || "" # don't run xcrun everytime, usually useless. +print <<EOS if $unicode_version + CONFIG["UNICODE_VERSION"] = #{$unicode_version.dump} +EOS +print <<EOS if $unicode_emoji_version + CONFIG["UNICODE_EMOJI_VERSION"] = #{$unicode_emoji_version.dump} +EOS +print prefix.start_with?("/System/") ? <<EOS : <<EOS if /darwin/ =~ arch + if sdkroot = ENV["SDKROOT"] + sdkroot = sdkroot.dup + elsif File.exist?(File.join(CONFIG["prefix"], "include")) || + !(sdkroot = (IO.popen(%w[/usr/bin/xcrun --sdk macosx --show-sdk-path], in: IO::NULL, err: IO::NULL, &:read) rescue nil)) + sdkroot = +"" + else + sdkroot.chomp! + end + CONFIG["SDKROOT"] = sdkroot +EOS + CONFIG["SDKROOT"] = "" EOS print <<EOS + CONFIG["platform"] = #{platform || '"$(arch)"'} CONFIG["archdir"] = "$(rubyarchdir)" CONFIG["topdir"] = File.dirname(__FILE__) + # Almost same with CONFIG. MAKEFILE_CONFIG has other variable + # reference like below. + # + # MAKEFILE_CONFIG["bindir"] = "$(exec_prefix)/bin" + # + # The values of this constant is used for creating Makefile. + # + # require 'rbconfig' + # + # print <<-END_OF_MAKEFILE + # prefix = \#{RbConfig::MAKEFILE_CONFIG['prefix']} + # exec_prefix = \#{RbConfig::MAKEFILE_CONFIG['exec_prefix']} + # bindir = \#{RbConfig::MAKEFILE_CONFIG['bindir']} + # END_OF_MAKEFILE + # + # => prefix = /usr/local + # exec_prefix = $(prefix) + # bindir = $(exec_prefix)/bin MAKEFILE_CONFIG = {} + # + # RbConfig.expand is used for resolving references like above in rbconfig. + # + # require 'rbconfig' + # p RbConfig.expand(RbConfig::MAKEFILE_CONFIG["bindir"]) + # # => "/usr/local/bin" MAKEFILE_CONFIG = {} CONFIG.each{|k,v| MAKEFILE_CONFIG[k] = v.dup} + + # call-seq: + # + # RbConfig.expand(val) -> string + # RbConfig.expand(val, config) -> string + # + # expands variable with given +val+ value. + # + # RbConfig.expand("$(bindir)") # => /home/foobar/all-ruby/ruby19x/bin def RbConfig::expand(val, config = CONFIG) newval = val.gsub(/\\$\\$|\\$\\(([^()]+)\\)|\\$\\{([^{}]+)\\}/) { var = $& @@ -267,6 +351,41 @@ print <<EOS RbConfig::expand(val) end + # call-seq: + # + # RbConfig.fire_update!(key, val) -> array + # RbConfig.fire_update!(key, val, mkconf, conf) -> array + # + # updates +key+ in +mkconf+ with +val+, and all values depending on + # the +key+ in +mkconf+. + # + # RbConfig::MAKEFILE_CONFIG.values_at("CC", "LDSHARED") # => ["gcc", "$(CC) -shared"] + # RbConfig::CONFIG.values_at("CC", "LDSHARED") # => ["gcc", "gcc -shared"] + # RbConfig.fire_update!("CC", "gcc-8") # => ["CC", "LDSHARED"] + # RbConfig::MAKEFILE_CONFIG.values_at("CC", "LDSHARED") # => ["gcc-8", "$(CC) -shared"] + # RbConfig::CONFIG.values_at("CC", "LDSHARED") # => ["gcc-8", "gcc-8 -shared"] + # + # returns updated keys list, or +nil+ if nothing changed. + def RbConfig.fire_update!(key, val, mkconf = MAKEFILE_CONFIG, conf = CONFIG) # :nodoc: + return if mkconf[key] == val + mkconf[key] = val + keys = [key] + deps = [] + begin + re = Regexp.new("\\\\$\\\\((?:%1$s)\\\\)|\\\\$\\\\{(?:%1$s)\\\\}" % keys.join('|')) + deps |= keys + keys.clear + mkconf.each {|k,v| keys << k if re =~ v} + end until keys.empty? + deps.each {|k| conf[k] = mkconf[k].dup} + deps.each {|k| expand(conf[k])} + deps + end + + # call-seq: + # + # RbConfig.ruby -> path + # # returns the absolute pathname of the ruby command. def RbConfig.ruby File.join( @@ -275,24 +394,8 @@ print <<EOS ) end end +# Non-nil if configured for cross compiling. CROSS_COMPILING = nil unless defined? CROSS_COMPILING EOS -$stdout = STDOUT -mode = IO::RDWR|IO::CREAT -mode |= IO::BINARY if defined?(IO::BINARY) -open(rbconfig_rb, mode) do |f| - if $timestamp and f.stat.size == config.size and f.read == config - puts "#{rbconfig_rb} unchanged" - else - puts "#{rbconfig_rb} updated" - f.rewind - f.truncate(0) - f.print(config) - end -end -if String === $timestamp - FileUtils.touch($timestamp) -end - # vi:set sw=2: diff --git a/tool/mkrunnable.rb b/tool/mkrunnable.rb index 60d0889eb4..ef358e2425 100755 --- a/tool/mkrunnable.rb +++ b/tool/mkrunnable.rb @@ -1,8 +1,12 @@ #!./miniruby # -*- coding: us-ascii -*- +# Used by "make runnable" target, to make symbolic links from a build +# directory. + require './rbconfig' require 'fileutils' +require_relative 'lib/path' case ARGV[0] when "-n" @@ -15,75 +19,13 @@ else include FileUtils end -module Mswin - def ln_safe(src, dest, *opt) - cmd = ["mklink", dest.tr("/", "\\"), src.tr("/", "\\")] - cmd[1, 0] = opt - return if system("cmd", "/c", *cmd) - # TODO: use RUNAS or something - puts cmd.join(" ") - end - - def ln_dir_safe(src, dest) - ln_safe(src, dest, "/d") - end -end - -def ln_safe(src, dest) - link = File.readlink(dest) rescue nil - return if link == src - ln_sf(src, dest) -end - -alias ln_dir_safe ln_safe - -if /mingw|mswin/ =~ (CROSS_COMPILING || RUBY_PLATFORM) - extend Mswin -end - -def clean_path(path) - path = "#{path}/".gsub(/(\A|\/)(?:\.\/)+/, '\1').tr_s('/', '/') - nil while path.sub!(/[^\/]+\/\.\.\//, '') - path -end - -def relative_path_from(path, base) - path = clean_path(path) - base = clean_path(base) - path, base = [path, base].map{|s|s.split("/")} - until path.empty? or base.empty? or path[0] != base[0] - path.shift - base.shift - end - path, base = [path, base].map{|s|s.join("/")} - if /(\A|\/)\.\.\// =~ base - File.expand_path(path) - else - base.gsub!(/[^\/]+/, '..') - File.join(base, path) - end -end - -def ln_relative(src, dest) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - ln_safe(relative_path_from(src, parent), dest) -end - -def ln_dir_relative(src, dest) - return if File.identical?(src, dest) - parent = File.dirname(dest) - File.directory?(parent) or mkdir_p(parent) - ln_dir_safe(relative_path_from(src, parent), dest) -end +include Path config = RbConfig::MAKEFILE_CONFIG.merge("prefix" => ".", "exec_prefix" => ".") config.each_value {|s| RbConfig.expand(s, config)} srcdir = config["srcdir"] ||= File.dirname(__FILE__) top_srcdir = config["top_srcdir"] ||= File.dirname(srcdir) extout = ARGV[0] || config["EXTOUT"] -version = config["ruby_version"] arch = config["arch"] bindir = config["bindir"] libdirname = config["libdirname"] @@ -92,19 +34,25 @@ vendordir = config["vendordir"] rubylibdir = config["rubylibdir"] rubyarchdir = config["rubyarchdir"] archdir = "#{extout}/#{arch}" -rubylibs = [vendordir, rubylibdir, rubyarchdir] -[bindir, libdir, archdir].uniq.each do |dir| +exedir = bindir +if libdirname == "archlibdir" + exedir = exedir.sub(%r[/\K(?=[^/]+\z)]) {extout+"/"} +end +[exedir, libdir, archdir].uniq.each do |dir| File.directory?(dir) or mkdir_p(dir) end +unless exedir == bindir + ln_dir_relative(exedir, bindir) +end exeext = config["EXEEXT"] ruby_install_name = config["ruby_install_name"] rubyw_install_name = config["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name -[ruby_install_name, rubyw_install_name, goruby_install_name].map do |ruby| - ruby += exeext +[ruby_install_name, rubyw_install_name, goruby_install_name].each do |ruby| if ruby and !ruby.empty? - ln_relative(ruby, "#{bindir}/#{ruby}") + ruby += exeext + ln_relative(ruby, "#{exedir}/#{ruby}", true) end end so = config["LIBRUBY_SO"] diff --git a/tool/node_name.rb b/tool/node_name.rb index fef7720a5a..dc0584e821 100755 --- a/tool/node_name.rb +++ b/tool/node_name.rb @@ -1,6 +1,8 @@ -#! ./miniruby -while gets - if ~/enum node_type \{/..~/^\};/ - ~/(NODE_.+),/ and puts(" case #{$1}:\n\treturn \"#{$1}\";") - end +#! ./miniruby -n + +# Used when making Ruby to generate node_name.inc. +# See common.mk for details. + +if (t ||= /^enum node_type \{/ =~ $_) and (t = /^\};/ !~ $_) + /(NODE_.+),/ =~ $_ and puts(" case #{$1}:\n\treturn \"#{$1}\";") end diff --git a/tool/notes-github-pr.rb b/tool/notes-github-pr.rb new file mode 100644 index 0000000000..d69d479cdf --- /dev/null +++ b/tool/notes-github-pr.rb @@ -0,0 +1,138 @@ +#!/usr/bin/env ruby +# Add GitHub pull request reference / author info to git notes. + +require 'net/http' +require 'uri' +require 'tmpdir' +require 'json' +require 'yaml' + +# Conversion for people whose GitHub account name and SVN_ACCOUNT_NAME are different. +GITHUB_TO_SVN = { + 'amatsuda' => 'a_matsuda', + 'matzbot' => 'git', + 'jeremyevans' => 'jeremy', + 'znz' => 'kazu', + 'k-tsj' => 'ktsj', + 'nurse' => 'naruse', + 'ioquatix' => 'samuel', + 'suketa' => 'suke', + 'unak' => 'usa', +} + +EMAIL_YML_URL = 'https://raw.githubusercontent.com/ruby/git.ruby-lang.org/refs/heads/master/config/email.yml' +SVN_TO_EMAILS = YAML.safe_load(Net::HTTP.get_response(URI(EMAIL_YML_URL)).tap(&:value).body) + +class GitHub + ENDPOINT = URI.parse('https://api.github.com') + + def initialize(access_token) + @access_token = access_token + end + + # https://developer.github.com/changes/2019-04-11-pulls-branches-for-commit/ + def pulls(owner:, repo:, commit_sha:) + resp = get("/repos/#{owner}/#{repo}/commits/#{commit_sha}/pulls", accept: 'application/vnd.github.groot-preview+json') + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/pulls/#get-a-single-pull-request + def pull_request(owner:, repo:, number:) + resp = get("/repos/#{owner}/#{repo}/pulls/#{number}") + JSON.parse(resp.body) + end + + # https://developer.github.com/v3/users/#get-a-single-user + def user(username:) + resp = get("/users/#{username}") + JSON.parse(resp.body) + end + + private + + def get(path, accept: 'application/vnd.github.v3+json') + Net::HTTP.start(ENDPOINT.host, ENDPOINT.port, use_ssl: ENDPOINT.scheme == 'https') do |http| + headers = { 'Accept': accept, 'Authorization': "bearer #{@access_token}" } + http.get(path, headers).tap(&:value) + end + end +end + +module Git + class << self + def abbrev_ref(refname, repo_path:) + git('rev-parse', '--symbolic', '--abbrev-ref', refname, repo_path: repo_path).strip + end + + def rev_list(arg, first_parent: false, repo_path: nil) + git('rev-list', *[('--first-parent' if first_parent)].compact, arg, repo_path: repo_path).lines.map(&:chomp) + end + + def commit_message(sha) + git('log', '-1', '--pretty=format:%B', sha) + end + + def notes_message(sha) + git('log', '-1', '--pretty=format:%N', sha) + end + + def committer_name(sha) + git('log', '-1', '--pretty=format:%cn', sha) + end + + def committer_email(sha) + git('log', '-1', '--pretty=format:%cE', sha) + end + + private + + def git(*cmd, repo_path: nil) + env = {} + if repo_path + env['GIT_DIR'] = repo_path + end + out = IO.popen(env, ['git', *cmd], &:read) + unless $?.success? + abort "Failed to execute: git #{cmd.join(' ')}\n#{out}" + end + out + end + end +end + +github = GitHub.new(ENV.fetch('GITHUB_TOKEN')) + +repo_path, *rest = ARGV +rest.each_slice(3).map do |oldrev, newrev, _refname| + system('git', 'fetch', 'origin', 'refs/notes/commits:refs/notes/commits', exception: true) + + updated = false + Git.rev_list("#{oldrev}..#{newrev}", first_parent: true).each do |sha| + github.pulls(owner: 'ruby', repo: 'ruby', commit_sha: sha).each do |pull| + number = pull.fetch('number') + url = pull.fetch('html_url') + next unless url.start_with?('https://github.com/ruby/ruby/pull/') + + # "Merged" notes for "Squash and merge" + message = Git.commit_message(sha) + notes = Git.notes_message(sha) + if !message.include?(url) && !message.match(/[ (]##{number}[) ]/) && !notes.include?(url) + system('git', 'notes', 'append', '-m', "Merged: #{url}", sha, exception: true) + updated = true + end + + # "Merged-By" notes for "Rebase and merge" + if Git.committer_name(sha) == 'GitHub' && Git.committer_email(sha) == 'noreply@github.com' + username = github.pull_request(owner: 'ruby', repo: 'ruby', number: number).fetch('merged_by').fetch('login') + email = github.user(username: username).fetch('email') + email ||= SVN_TO_EMAILS[GITHUB_TO_SVN.fetch(username, username)]&.first + system('git', 'notes', 'append', '-m', "Merged-By: #{username}#{(" <#{email}>" if email)}", sha, exception: true) + updated = true + end + end + end + + if updated + system('git', 'push', 'origin', 'refs/notes/commits', exception: true) + end +end diff --git a/tool/notify-slack-commits.rb b/tool/notify-slack-commits.rb new file mode 100644 index 0000000000..73e22b9a03 --- /dev/null +++ b/tool/notify-slack-commits.rb @@ -0,0 +1,87 @@ +#!/usr/bin/env ruby + +require "net/https" +require "open3" +require "json" +require "digest/md5" + +SLACK_WEBHOOK_URLS = [ + ENV.fetch("SLACK_WEBHOOK_URL_ALERTS").chomp, # ruby-lang#alerts + ENV.fetch("SLACK_WEBHOOK_URL_COMMITS").chomp, # ruby-lang#commits + ENV.fetch("SLACK_WEBHOOK_URL_RUBY_JP").chomp, # ruby-jp#ruby-commits +] +GRAVATAR_OVERRIDES = { + "nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars0.githubusercontent.com/u/21976", + "noreply@github.com" => "https://avatars1.githubusercontent.com/u/9919", + "nurse@users.noreply.github.com" => "https://avatars1.githubusercontent.com/u/13423", + "svn-admin@ruby-lang.org" => "https://avatars1.githubusercontent.com/u/29403229", + "svn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars1.githubusercontent.com/u/29403229", + "usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e" => "https://avatars2.githubusercontent.com/u/17790", + "usa@ruby-lang.org" => "https://avatars2.githubusercontent.com/u/17790", + "yui-knk@ruby-lang.org" => "https://avatars0.githubusercontent.com/u/5356517", + "znz@users.noreply.github.com" => "https://avatars3.githubusercontent.com/u/11857", +} + +def escape(s) + s.gsub(/[&<>]/, "&" => "&", "<" => "<", ">" => ">") +end + +ARGV.each_slice(3) do |oldrev, newrev, refname| + out, = Open3.capture2("git", "rev-parse", "--symbolic", "--abbrev-ref", refname) + branch = out.strip + + out, = Open3.capture2("git", "log", "--pretty=format:%H\n%h\n%cn\n%ce\n%ct\n%B", "--abbrev=10", "-z", "#{oldrev}..#{newrev}") + + attachments = [] + out.split("\0").reverse_each do |s| + sha, sha_abbr, committer, committeremail, committertime, body = s.split("\n", 6) + subject, body = body.split("\n", 2) + + # Append notes content to `body` if it's notes + if refname.match(%r[\Arefs/notes/\w+\z]) + # `--diff-filter=AM -M` to exclude rename by git's directory optimization + object = IO.popen(["git", "diff", "--diff-filter=AM", "-M", "--name-only", "#{sha}^..#{sha}"], &:read).chomp + if md = object.match(/\A(?<prefix>\h{2})\/?(?<rest>\h{38})\z/) + body = [body, IO.popen(["git", "notes", "show", md[:prefix] + md[:rest]], &:read)].join + end + end + + gravatar = GRAVATAR_OVERRIDES.fetch(committeremail) do + "https://www.gravatar.com/avatar/#{ Digest::MD5.hexdigest(committeremail.downcase) }" + end + + attachments << { + title: "#{ sha_abbr } (#{ branch }): #{ escape(subject) }", + title_link: "https://github.com/ruby/ruby/commit/#{ sha }", + text: escape((body || "").strip), + footer: committer, + footer_icon: gravatar, + ts: committertime.to_i, + color: '#24282D', + } + end + + # 100 attachments cannot be exceeded. 20 is recommended. https://api.slack.com/docs/message-attachments + attachments.each_slice(20).each do |attachments_group| + payload = { attachments: attachments_group } + + #Net::HTTP.post( + # URI.parse(SLACK_WEBHOOK_URL), + # JSON.generate(payload), + # "Content-Type" => "application/json" + #) + responses = SLACK_WEBHOOK_URLS.map do |url| + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.start do + req = Net::HTTP::Post.new(uri.path) + req.set_form_data(payload: payload.to_json) + http.request(req) + end + end + + results = responses.map { |resp| "#{resp.code} (#{resp.body})" }.join(', ') + puts "#{results} -- #{payload.to_json}" + end +end diff --git a/tool/outdate-bundled-gems.rb b/tool/outdate-bundled-gems.rb new file mode 100755 index 0000000000..47ee80bc89 --- /dev/null +++ b/tool/outdate-bundled-gems.rb @@ -0,0 +1,190 @@ +#!/usr/bin/ruby +require 'fileutils' +require 'rubygems' + +fu = FileUtils::Verbose + +until ARGV.empty? + case ARGV.first + when '--' + ARGV.shift + break + when '-n', '--dry-run', '--dryrun' + ## -n, --dry-run Don't remove + fu = FileUtils::DryRun + when /\A--make=/ + # just to run when `make -n` + when /\A--mflags=(.*)/ + fu = FileUtils::DryRun if /\A-\S*n/ =~ $1 + when /\A--gem[-_]platform=(.*)/im + ## --gem-platform=PLATFORM Platform in RubyGems style + gem_platform = $1 + ruby_platform = nil + when /\A--ruby[-_]platform=(.*)/im + ## --ruby-platform=PLATFORM Platform in Ruby style + ruby_platform = $1 + gem_platform = nil + when /\A--ruby[-_]version=(.*)/im + ## --ruby-version=VERSION Ruby version to keep + ruby_version = $1 + when /\A--only=(?:(curdir|srcdir)|all)\z/im + ## --only=(curdir|srcdir|all) Specify directory to remove gems from + only = $1&.downcase + when /\A--all\z/im + ## --all Remove all gems not only bundled gems + all = true + when /\A--help\z/im + ## --help Print this message + puts "Usage: #$0 [options] [srcdir]" + File.foreach(__FILE__) do |line| + line.sub!(/^ *## /, "") or next + break if line.chomp!.empty? + opt, desc = line.split(/ {2,}/, 2) + printf " %-28s %s\n", opt, desc + end + exit + when /\A-/ + raise "#{$0}: unknown option: #{ARGV.first}" + else + break + end + ## + ARGV.shift +end + +gem_platform ||= Gem::Platform.new(ruby_platform).to_s if ruby_platform + +class Removal + attr_reader :base + + def initialize(base = nil) + @base = (File.join(base, "/") if base) + @remove = {} + end + + def prefixed(name) + @base ? File.join(@base, name) : name + end + + def stripped(name) + if @base && name.start_with?(@base) + name[@base.size..-1] + else + name + end + end + + def slash(name) + name.sub(%r[[^/]\K\z], '/') + end + + def exist?(name) + !@remove.fetch(name) {|k| @remove[k] = !File.exist?(prefixed(name))} + end + def directory?(name) + !@remove.fetch(slash(name)) {|k| @remove[k] = !File.directory?(prefixed(name))} + end + + def unlink(name) + @remove[stripped(name)] = :rm_f + end + def rmdir(name) + @remove[slash(stripped(name))] = :rm_rf + end + + def glob(pattern, *rest) + Dir.glob(prefixed(pattern), *rest) {|n| + yield stripped(n) + } + end + + def sorted + @remove.sort_by {|k, | [-k.count("/"), k]} + end + + def each_file + sorted.each {|k, v| yield prefixed(k) if v == :rm_f} + end + + def each_directory + sorted.each {|k, v| yield prefixed(k) if v == :rm_rf} + end +end + +srcdir = Removal.new(ARGV.shift) +curdir = !srcdir.base || File.identical?(srcdir.base, ".") ? srcdir : Removal.new + +bundled = File.readlines("#{srcdir.base}gems/bundled_gems"). + grep(/^(\w[^\#\s]+)\s+[^\#\s]+(?:\s+[^\#\s]+\s+([^\#\s]+))?/) {$~.captures}.to_h rescue nil + +srcdir.glob(".bundle/gems/*/") do |dir| + base = File.basename(dir) + next if !all && bundled && !bundled.key?(base[/\A.+(?=-)/]) + unless srcdir.exist?("gems/#{base}.gem") + srcdir.rmdir(dir) + end +end + +srcdir.glob(".bundle/.timestamp/*.revision") do |file| + unless bundled&.fetch(File.basename(file, ".revision"), nil) + srcdir.unlink(file) + end +end + +srcdir.glob(".bundle/specifications/*.gemspec") do |spec| + unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}/") + srcdir.unlink(spec) + end +end + +curdir.glob(".bundle/specifications/*.gemspec") do |spec| + unless srcdir.directory?(".bundle/gems/#{File.basename(spec, '.gemspec')}") + curdir.unlink(spec) + end +end + +curdir.glob(".bundle/gems/*/") do |dir| + base = File.basename(dir) + unless curdir.exist?(".bundle/specifications/#{base}.gemspec") or + curdir.exist?("#{dir}/.bundled.#{base}.gemspec") + curdir.rmdir(dir) + end +end + +curdir.glob(".bundle/{extensions,.timestamp}/*/") do |dir| + unless gem_platform and File.fnmatch?(gem_platform, File.basename(dir)) + curdir.rmdir(dir) + end +end + +if gem_platform + curdir.glob(".bundle/{extensions,.timestamp}/#{gem_platform}/*/") do |dir| + unless ruby_version and File.fnmatch?(ruby_version, File.basename(dir, '-static')) + curdir.rmdir(dir) + end + end +end + +if ruby_version + curdir.glob(".bundle/extensions/#{gem_platform || '*'}/#{ruby_version}/*/") do |dir| + unless curdir.exist?(".bundle/specifications/#{File.basename(dir)}.gemspec") + curdir.rmdir(dir) + end + end + + curdir.glob(".bundle/.timestamp/#{gem_platform || '*'}/#{ruby_version}/.*.time") do |stamp| + dir = stamp[%r[/\.([^/]+)\.time\z], 1].gsub('.-.', '/')[%r[\A[^/]+/[^/]+]] + unless curdir.directory?(File.join(".bundle", dir)) + curdir.unlink(stamp) + end + end +end + +unless only == "curdir" + srcdir.each_file {|f| fu.rm_f(f)} + srcdir.each_directory {|d| fu.rm_rf(d)} +end +unless only == "srcdir" or curdir.equal?(srcdir) + curdir.each_file {|f| fu.rm_f(f)} + curdir.each_directory {|d| fu.rm_rf(d)} +end diff --git a/tool/parse.rb b/tool/parse.rb index 6243d7aa8e..93ae3e43cb 100644 --- a/tool/parse.rb +++ b/tool/parse.rb @@ -1,3 +1,6 @@ +# Used as part of the "make parse" Makefile target. +# See common.mk for details. + $file = ARGV[0] $str = ARGF.read.sub(/^__END__.*\z/m, '') puts '# ' + '-' * 70 diff --git a/tool/prereq.status b/tool/prereq.status new file mode 100644 index 0000000000..6aca615e90 --- /dev/null +++ b/tool/prereq.status @@ -0,0 +1,45 @@ +s,@EXTOUT@,tmp,g +s,@ruby_version@,0.0.0,g +s,@NULLCMD@,:,g +s,@ARCH_FLAG@,,g +s,@ASMEXT@,S,g +s,@BASERUBY@,ruby,g +s,@BOOTSTRAPRUBY@,$(BASERUBY),g +s,@CC@,false,g +s,@CFLAGS@,,g +s,@CHDIR@,cd,g +s,@CONFIGURE@,configure,g +s,@CP@,cp,g +s,@CPPFLAGS@,,g +s,@CXXFLAGS@,,g +s,@DLDFLAGS@,,g +s,@DTRACE_EXT@,dmyh,g +s,@EXEEXT@,,g +s,@HAVE_BASERUBY@,yes,g +s,@IFCHANGE@,tool/ifchange,g +s,@LDFLAGS@,,g +s,@LIBEXT@,a,g +s,@LIBRUBY@,libruby.a,g +s,@LIBRUBY_A@,libruby.a,g +s,@MINIRUBY@,$(BASERUBY),g +s,@MKDIR_P@,mkdir -p,g +s,@OBJEXT@,o,g +s,@PATH_SEPARATOR@,:,g +s,@PWD@,.,g +s,@RM@,rm -f,g +s,@RMALL@,rm -fr,g +s,@RMDIR@,rmdir,g +s,@RMDIRS@,$(RMDIR) -p,g +s,@RUBY@,$(BASERUBY),g +s,@RUNRUBY@,$(MINIRUBY),g +s,@arch@,noarch,g +s,@bindir@,,g +s,@configure_args@,,g +s,@ruby_install_name@,,g +s,@rubyarchdir@,,g +s,@rubylibprefix@,,g +s,@srcdir@,.,g + +s/@[A-Za-z][A-Za-z0-9_]*@//g +s/{\$([^(){}]*)}//g +s/^!/#!/ diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 1a17f093a8..874c3ef1d9 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -1,5 +1,10 @@ #!./miniruby +# Used by the "make install" target to install Ruby. +# See common.mk for more details. + +ENV["SDKROOT"] ||= "" if /darwin/ =~ RUBY_PLATFORM + begin load "./rbconfig.rb" rescue LoadError @@ -17,16 +22,17 @@ require 'fileutils' require 'shellwords' require 'optparse' require 'optparse/shellwords' -require 'ostruct' require 'rubygems' begin require "zlib" rescue LoadError $" << "zlib.rb" end +require_relative 'lib/path' +INDENT = " "*36 STDOUT.sync = true -File.umask(077) +File.umask(022) def parse_args(argv = ARGV) $mantype = 'doc' @@ -35,22 +41,28 @@ def parse_args(argv = ARGV) $make = 'make' $mflags = [] $install = [] + $installed = {} $installed_list = nil + $exclude = [] $dryrun = false $rdocdir = nil + $htmldir = nil $data_mode = 0644 $prog_mode = 0755 $dir_mode = nil $script_mode = nil $strip = false + $debug_symbols = nil $cmdtype = (if File::ALT_SEPARATOR == '\\' File.exist?("rubystub.exe") ? 'exe' : 'cmd' end) mflags = [] + gnumake = false opt = OptionParser.new opt.on('-n', '--dry-run') {$dryrun = true} opt.on('--dest-dir=DIR') {|dir| $destdir = dir} opt.on('--extout=DIR') {|dir| $extout = (dir unless dir.empty?)} + opt.on('--ext-build-dir=DIR') {|v| $ext_build_dir = v } opt.on('--make=COMMAND') {|make| $make = make} opt.on('--mantype=MAN') {|man| $mantype = man} opt.on('--make-flags=FLAGS', '--mflags', Shellwords) do |v| @@ -62,6 +74,9 @@ def parse_args(argv = ARGV) opt.on('-i', '--install=TYPE', $install_procs.keys) do |ins| $install << ins end + opt.on('-x', '--exclude=TYPE', $install_procs.keys) do |exc| + $exclude << exc + end opt.on('--data-mode=OCTAL-MODE', OptionParser::OctalInteger) do |mode| $data_mode = mode end @@ -76,14 +91,31 @@ def parse_args(argv = ARGV) end opt.on('--installed-list [FILENAME]') {|name| $installed_list = name} opt.on('--rdoc-output [DIR]') {|dir| $rdocdir = dir} + opt.on('--html-output [DIR]') {|dir| $htmldir = dir} opt.on('--cmd-type=TYPE', %w[cmd plain]) {|cmd| $cmdtype = (cmd unless cmd == 'plain')} opt.on('--[no-]strip') {|strip| $strip = strip} + opt.on('--gnumake') {gnumake = true} + opt.on('--debug-symbols=SUFFIX', /\w+/) {|name| $debug_symbols = ".#{name}"} + + unless $install_procs.empty? + w = (w = ENV["COLUMNS"] and (w = w.to_i) > 80) ? w - 30 : 50 + opt.on("\n""Types for --install and --exclude:") + mesg = +" " + $install_procs.each_key do |t| + if mesg.size + t.size > w + opt.on(mesg) + mesg = +" " + end + mesg << " " << t.to_s + end + opt.on(mesg) + end opt.order!(argv) do |v| case v when /\AINSTALL[-_]([-\w]+)=(.*)/ argv.unshift("--#{$1.tr('_', '-')}=#{$2}") - when /\A\w[-\w+]*=\z/ + when /\A\w[-\w]*=/ mflags << v when /\A\w[-\w+]*\z/ $install << v.intern @@ -100,6 +132,7 @@ def parse_args(argv = ARGV) $make, *rest = Shellwords.shellwords($make) $mflags.unshift(*rest) unless rest.empty? $mflags.unshift(*mflags) + $mflags.reject! {|v| /\A-[OW]/ =~ v} if gnumake def $mflags.set?(flag) grep(/\A-(?!-).*#{flag.chr}/i) { return true } @@ -117,6 +150,7 @@ def parse_args(argv = ARGV) end $destdir ||= $mflags.defined?("DESTDIR") + $destdir = File.expand_path($destdir) unless $destdir.empty? if $extout ||= $mflags.defined?("EXTOUT") RbConfig.expand($extout) end @@ -125,19 +159,35 @@ def parse_args(argv = ARGV) if $installed_list ||= $mflags.defined?('INSTALLED_LIST') RbConfig.expand($installed_list, RbConfig::CONFIG) - $installed_list = open($installed_list, "ab") + $installed_list = File.open($installed_list, "ab") $installed_list.sync = true end $rdocdir ||= $mflags.defined?('RDOCOUT') + $htmldir ||= $mflags.defined?('HTMLOUT') $dir_mode ||= $prog_mode | 0700 $script_mode ||= $prog_mode + if $ext_build_dir.nil? + raise OptionParser::MissingArgument.new("--ext-build-dir=DIR") + end +end + +Compressors = {".gz"=>"gzip", ".bz2"=>"bzip2"} +def Compressors.for(type) + ext = File.extname(type) + if compress = fetch(ext, nil) + [type.chomp(ext), ext, compress] + else + [type, *find {|_, z| system(z, in: IO::NULL, out: IO::NULL)}] + end end $install_procs = Hash.new {[]} def install?(*types, &block) - $install_procs[:all] <<= block + unless types.delete(:nodefault) + $install_procs[:all] <<= block + end types.each do |type| $install_procs[type] <<= block end @@ -159,9 +209,12 @@ def install(src, dest, options = {}) options = options.clone strip = options.delete(:strip) options[:preserve] = true + srcs = Array(src).select {|s| !$installed[$made_dirs[dest] ? File.join(dest, s) : dest]} + return if srcs.empty? + src = srcs if Array === src d = with_destdir(dest) - super(src, d, options) - srcs = Array(src) + super(src, d, **options) + srcs.each {|s| $installed[$made_dirs[dest] ? File.join(dest, s) : dest] = true} if strip d = srcs.map {|s| File.join(d, File.basename(s))} if $made_dirs[dest] strip_file(d) @@ -178,15 +231,20 @@ def ln_sf(src, dest) end $made_dirs = {} + +def dir_creating(dir) + $made_dirs.fetch(dir) do + $made_dirs[dir] = true + $installed_list.puts(File.join(dir, "")) if $installed_list + yield if defined?(yield) + end +end + def makedirs(dirs) dirs = fu_list(dirs) dirs.collect! do |dir| realdir = with_destdir(dir) - realdir unless $made_dirs.fetch(dir) do - $made_dirs[dir] = true - $installed_list.puts(File.join(dir, "")) if $installed_list - File.directory?(realdir) - end + realdir unless dir_creating(dir) {File.directory?(realdir)} end.compact! super(dirs, :mode => $dir_mode) unless dirs.empty? end @@ -242,7 +300,7 @@ def install_recursive(srcdir, dest, options = {}) elsif stat.symlink? # skip else - files << [src, d, false] if File.fnmatch?(glob, f) and !skip[f] + files << [src, d, false] if File.fnmatch?(glob, f, File::FNM_EXTGLOB) and !skip[f] end end paths.insert(0, *files) @@ -250,7 +308,8 @@ def install_recursive(srcdir, dest, options = {}) end for src, d, dir in found if dir - makedirs(d) + next + # makedirs(d) else makedirs(d[/.*(?=\/)/m]) if block_given? @@ -263,11 +322,11 @@ def install_recursive(srcdir, dest, options = {}) end def open_for_install(path, mode) - data = open(realpath = with_destdir(path), "rb") {|f| f.read} rescue nil + data = File.binread(realpath = with_destdir(path)) rescue nil newdata = yield unless $dryrun unless newdata == data - open(realpath, "wb", mode) {|f| f.write newdata} + File.open(realpath, "wb", mode) {|f| f.write newdata} end File.chmod(mode, realpath) end @@ -281,9 +340,8 @@ def with_destdir(dir) end def without_destdir(dir) - return dir if !$destdir or $destdir.empty? or !dir.start_with?($destdir) - dir = dir.sub(/\A\w:/, '') if File::PATH_SEPARATOR == ';' - dir[$destdir.size..-1] + return dir if !$destdir or $destdir.empty? + dir.start_with?($destdir) ? dir[$destdir.size..-1] : dir end def prepare(mesg, basedir, subdirs=nil) @@ -299,7 +357,7 @@ def prepare(mesg, basedir, subdirs=nil) else dirs = [basedir, *subdirs.collect {|dir| File.join(basedir, dir)}] end - printf("installing %-18s %s%s\n", "#{mesg}:", basedir, + printf("%-*s%s%s\n", INDENT.size, "installing #{mesg}:", basedir, (subdirs ? " (#{subdirs.join(', ')})" : "")) makedirs(dirs) end @@ -319,47 +377,619 @@ rubyw_install_name = CONFIG["rubyw_install_name"] goruby_install_name = "go" + ruby_install_name bindir = CONFIG["bindir", true] +if CONFIG["libdirname"] == "archlibdir" + archbindir = bindir.sub(%r[/\K(?=[^/]+\z)]) {CONFIG["config_target"] + "/"} +end libdir = CONFIG[CONFIG.fetch("libdirname", "libdir"), true] rubyhdrdir = CONFIG["rubyhdrdir", true] archhdrdir = CONFIG["rubyarchhdrdir"] || (rubyhdrdir + "/" + CONFIG['arch']) rubylibdir = CONFIG["rubylibdir", true] archlibdir = CONFIG["rubyarchdir", true] -sitelibdir = CONFIG["sitelibdir"] -sitearchlibdir = CONFIG["sitearchdir"] -vendorlibdir = CONFIG["vendorlibdir"] -vendorarchlibdir = CONFIG["vendorarchdir"] +if CONFIG["sitedir"] + sitelibdir = CONFIG["sitelibdir"] + sitearchlibdir = CONFIG["sitearchdir"] +end +if CONFIG["vendordir"] + vendorlibdir = CONFIG["vendorlibdir"] + vendorarchlibdir = CONFIG["vendorarchdir"] +end mandir = CONFIG["mandir", true] docdir = CONFIG["docdir", true] -configure_args = Shellwords.shellwords(CONFIG["configure_args"]) enable_shared = CONFIG["ENABLE_SHARED"] == 'yes' dll = CONFIG["LIBRUBY_SO", enable_shared] lib = CONFIG["LIBRUBY", true] arc = CONFIG["LIBRUBY_A", true] -config_h = File.read(CONFIG["EXTOUT"]+"/include/"+CONFIG["arch"]+"/ruby/config.h") -load_relative = config_h[/^\s*#\s*define\s+LOAD_RELATIVE\s+(\d+)/, 1].to_i.nonzero? +load_relative = CONFIG["LIBRUBY_RELATIVE"] == 'yes' + +rdoc_noinst = %w[created.rid] + +prolog_script = <<EOS +bindir="#{load_relative ? '${0%/*}' : bindir.gsub(/\"/, '\\\\"')}" +EOS +if !load_relative and libpathenv = CONFIG["LIBPATHENV"] + pathsep = File::PATH_SEPARATOR + prolog_script << <<EOS +libdir="#{libdir.gsub(/\"/, '\\\\"')}" +export #{libpathenv}="$libdir${#{libpathenv}:+#{pathsep}$#{libpathenv}}" +EOS +end +prolog_script << %Q[exec "$bindir/#{ruby_install_name}" "-x" "$0" "$@"\n] +PROLOG_SCRIPT = {} +PROLOG_SCRIPT["exe"] = "#!#{bindir}/#{ruby_install_name}" +PROLOG_SCRIPT["cmd"] = <<EOS +:""||{ ""=> %q<-*- ruby -*- +@"%~dp0#{ruby_install_name}" -x "%~f0" %* +@exit /b %ERRORLEVEL% +};{ #\n#{prolog_script.gsub(/(?=\n)/, ' #')}>,\n} +EOS +PROLOG_SCRIPT.default = (load_relative || /\s/ =~ bindir) ? + <<EOS : PROLOG_SCRIPT["exe"] +#!/bin/sh +# -*- ruby -*- +_=_\\ +=begin +#{prolog_script.chomp} +=end +EOS + +installer = Struct.new(:ruby_shebang, :ruby_bin, :ruby_install_name, :stub, :trans) do + def transform(name) + RbConfig.expand(trans[name]) + end +end + +$script_installer = Class.new(installer) do + ruby_shebang = File.join(bindir, ruby_install_name) + if File::ALT_SEPARATOR + ruby_bin = ruby_shebang.tr(File::SEPARATOR, File::ALT_SEPARATOR) + end + if trans = CONFIG["program_transform_name"] + exp = [] + trans.gsub!(/\$\$/, '$') + trans.scan(%r[\G[\s;]*(/(?:\\.|[^/])*+/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*+)\3((?:(?!\3)(?:\\.|.))*+)\3([gi]*)]) do + |addr, cmd, sep, pat, rep, opt| + addr &&= Regexp.new(addr[/\A\/(.*)\/\z/, 1]) + case cmd + when 's' + next if pat == '^' and rep.empty? + exp << [addr, (opt.include?('g') ? :gsub! : :sub!), + Regexp.new(pat, opt.include?('i')), rep.gsub(/&/){'\&'}] + when 'y' + exp << [addr, :tr!, Regexp.quote(pat), rep] + end + end + trans = proc do |base| + exp.each {|addr, opt, pat, rep| base.__send__(opt, pat, rep) if !addr or addr =~ base} + base + end + elsif /ruby/ =~ ruby_install_name + trans = proc {|base| ruby_install_name.sub(/ruby/, base)} + else + trans = proc {|base| base} + end + + def prolog(shebang) + shebang.sub!(/\r$/, '') + script = PROLOG_SCRIPT[$cmdtype] + shebang.sub!(/\A(\#!.*?ruby\b)?/) do + if script.end_with?("\n") + script + ($1 || "#!ruby\n") + else + $1 ? script : "#{script}\n" + end + end + shebang + end + + def install(src, cmd) + cmd = cmd.sub(/[^\/]*\z/m) {|n| transform(n)} + + shebang, body = File.open(src, "rb") do |f| + next f.gets, f.read + end + shebang or raise "empty file - #{src}" + shebang = prolog(shebang) + body.gsub!(/\r$/, '') + + cmd << ".#{$cmdtype}" if $cmdtype + open_for_install(cmd, $script_mode) do + case $cmdtype + when "exe" + stub + shebang + body + else + shebang + body + end + end + end + + def self.get_rubystub + stubfile = "rubystub.exe" + stub = File.open(stubfile, "rb") {|f| f.read} << "\n" + rescue => e + abort "No #{stubfile}: #{e}" + else + stub + end + + def stub + super or self.stub = self.class.get_rubystub + end + + break new(ruby_shebang, ruby_bin, ruby_install_name, nil, trans) +end + +module RbInstall + def self.no_write(options = nil) + u = File.umask(0022) + if $dryrun + fu = ::Object.class_eval do + fu = remove_const(:FileUtils) + const_set(:FileUtils, fu::NoWrite) + fu + end + # RubyGems 3.0.0 or later supports `dir_mode`, but it uses + # `File` method to apply it, not `FileUtils`. + dir_mode = options.delete(:dir_mode) if options + end + yield + ensure + options[:dir_mode] = dir_mode if dir_mode + if fu + ::Object.class_eval do + remove_const(:FileUtils) + const_set(:FileUtils, fu) + end + end + File.umask(u) + end + + module Specs + class FileCollector + def self.for(srcdir, type, gemspec) + relative_base = (File.dirname(gemspec) if gemspec.include?("/")) + const_get(type.capitalize).new(gemspec, srcdir, relative_base) + end + + attr_reader :gemspec, :srcdir, :relative_base + def initialize(gemspec, srcdir, relative_base) + @gemspec = gemspec + @srcdir = srcdir + @relative_base = relative_base + end + + def collect + requirable_features.sort + end + + private + + def features_from_makefile(makefile_path) + makefile = File.read(makefile_path) + + name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + return [] if name.nil? || name.empty? + + feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = feature.sub("$(TARGET)", name) + + target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? + + Array(feature) + end + + class Ext < self + def requirable_features + # install ext only when it's configured + return [] unless File.exist?(makefile_path) + + ruby_features + ext_features + end + + private + + def ruby_features + Dir.glob("**/*.rb", base: "#{makefile_dir}/lib") + end + + def ext_features + features_from_makefile(makefile_path) + end + + def makefile_path + if File.exist?("#{makefile_dir}/Makefile") + "#{makefile_dir}/Makefile" + else + # for out-of-place build + "#{$ext_build_dir}/#{relative_base}/Makefile" + end + end + + def makefile_dir + "#{root}/#{relative_base}" + end + + def root + File.expand_path($ext_build_dir, srcdir) + end + end + + class Lib < self + def requirable_features + ruby_features + ext_features + end + + private + + def ruby_features + gemname = File.basename(gemspec, ".gemspec") + base = relative_base || gemname + # for lib/net/net-smtp.gemspec + if m = /.*(?=-(.*)\z)/.match(gemname) + base = File.join(base, *m.to_a.select {|n| !base.include?(n)}) + end + files = Dir.glob("#{base}{.rb,/**/*.rb}", base: root) + if !relative_base and files.empty? # no files at the toplevel + # pseudo gem like ruby2_keywords + files << "#{gemname}.rb" + end + + case gemname + when "net-http" + files << "net/https.rb" + when "optparse" + files << "optionparser.rb" + end + + files + end + + def ext_features + loaded_gemspec = load_gemspec("#{root}/#{gemspec}") + extension = loaded_gemspec.extensions.first + return [] unless extension + + extconf = File.expand_path(extension, srcdir) + ext_build_dir = File.dirname(extconf) + makefile_path = "#{ext_build_dir}/Makefile" + return [] unless File.exist?(makefile_path) + + features_from_makefile(makefile_path) + end + + def root + "#{srcdir}/lib" + end + end + + class UnpackedGem < self + def collect + base = @srcdir or return [] + Dir.glob("**/*", File::FNM_DOTMATCH, base: base).select do |n| + next if n == "." + next if File.fnmatch?("*.gemspec", n, File::FNM_DOTMATCH|File::FNM_PATHNAME) + !File.directory?(File.join(base, n)) + end + end + end + end + end + + class DirPackage + attr_reader :spec + + attr_accessor :dir_mode + attr_accessor :prog_mode + attr_accessor :data_mode + + def initialize(spec, dir_map = nil) + @spec = spec + @src_dir = File.dirname(@spec.loaded_from) + @dir_map = dir_map + end + + def extract_files(destination_dir, pattern = "*") + return if @src_dir == destination_dir + File.chmod(0700, destination_dir) unless $dryrun + mode = pattern == File.join(spec.bindir, '*') ? prog_mode : data_mode + destdir = without_destdir(destination_dir) + if @dir_map + (dir_map = @dir_map.map {|k, v| Regexp.quote(k) unless k == v}).compact! + dir_map = %r{\A(?:#{dir_map.join('|')})(?=/)} + end + spec.files.each do |f| + next unless File.fnmatch(pattern, f) + src = File.join(@src_dir, dir_map =~ f ? "#{@dir_map[$&]}#{$'}" : f) + dest = File.join(destdir, f) + makedirs(dest[/.*(?=\/)/m]) + install src, dest, :mode => mode + end + File.chmod(dir_mode, destination_dir) unless $dryrun + end + end + + class UnpackedInstaller < Gem::Installer + # This method is mostly copied from old version of Gem::Installer#install + def install_with_default_gem + verify_gem_home + + # The name and require_paths must be verified first, since it could contain + # ruby code that would be eval'ed in #ensure_loadable_spec + verify_spec + + ensure_loadable_spec + + if options[:install_as_default] + Gem.ensure_default_gem_subdirectories gem_home + else + Gem.ensure_gem_subdirectories gem_home + end + + return true if @force + + ensure_dependencies_met unless @ignore_dependencies + + run_pre_install_hooks + + # Set loaded_from to ensure extension_dir is correct + if @options[:install_as_default] + spec.loaded_from = default_spec_file + else + spec.loaded_from = spec_file + end + + # Completely remove any previous gem files + FileUtils.rm_rf gem_dir + FileUtils.rm_rf spec.extension_dir + + dir_mode = options[:dir_mode] + FileUtils.mkdir_p gem_dir, mode: dir_mode && 0o755 + + if @options[:install_as_default] + extract_bin + write_default_spec + else + extract_files + + build_extensions + write_build_info_file + run_post_build_hooks + end + + generate_bin + generate_plugins + + unless @options[:install_as_default] + write_spec + write_cache_file + end + + File.chmod(dir_mode, gem_dir) if dir_mode + + say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? + + Gem::Specification.add_spec(spec) unless @install_dir + + load_plugin + + run_post_install_hooks + + spec + rescue Errno::EACCES => e + # Permission denied - /path/to/foo + raise Gem::FilePermissionError, e.message.split(" - ").last + end + + def write_cache_file + end + + def shebang(bin_file_name) + path = File.join(gem_dir, spec.bindir, bin_file_name) + first_line = File.open(path, "rb") {|file| file.gets} + $script_installer.prolog(first_line).chomp + end + + def app_script_text(bin_file_name) + # move shell script part after comments generated by RubyGems. + super.sub(/\A + (\#!\/bin\/sh\n\#.*-\*-\s*ruby\s*-\*-.*\n) + ((?:.*\n)*?\#!.*ruby.*\n) + \#\n + ((?:\#.*\n)+)/x, '\1\3\2') + end + + def check_executable_overwrite(filename) + return if @wrappers and same_bin_script?(filename, @bin_dir) + super + end + + def same_bin_script?(filename, bindir) + path = File.join(bindir, formatted_program_filename(filename)) + begin + return true if File.binread(path) == app_script_text(filename) + rescue + end + false + end + + def write_spec + super unless $dryrun + $installed_list.puts(without_destdir(spec_file)) if $installed_list + end + + def write_default_spec + super unless $dryrun + $installed_list.puts(without_destdir(default_spec_file)) if $installed_list + end + + def install + spec.post_install_message = nil + dir_creating(without_destdir(gem_dir)) + RbInstall.no_write(options) { install_with_default_gem } + end + + # Now build-ext builds all extensions including bundled gems. + def build_extensions + end + + def generate_bin_script(filename, bindir) + return if same_bin_script?(filename, bindir) + name = formatted_program_filename(filename) + unless $dryrun + super + script = File.join(bindir, name) + File.chmod($script_mode, script) + File.unlink("#{script}.lock") rescue nil + end + $installed_list.puts(File.join(without_destdir(bindir), name)) if $installed_list + end + + def verify_gem_home # :nodoc: + end + + def ensure_writable_dir(dir) + $made_dirs.fetch(d = without_destdir(dir)) do + $made_dirs[d] = true + super unless $dryrun + $installed_list.puts(d+"/") if $installed_list + end + end + + def load_plugin + # Suppress warnings for constant re-assignment + verbose, $VERBOSE = $VERBOSE, nil + super + ensure + $VERBOSE = verbose + end + end +end + +def load_gemspec(file, base = nil, files: nil) + file = File.realpath(file) + code = File.read(file, encoding: "utf-8:-") + + code.gsub!(/^ *#.*/, "") + spec_files = files ? files.map(&:dump).join(", ") : "" + code.gsub!(/(?:`git[^\`]*`|%x\[git[^\]]*\])\.split(\([^\)]*\))?/m) do + "[" + spec_files + "]" + end \ + or + code.gsub!(/IO\.popen\(.*git.*?\)/) do + "[" + spec_files + "] || itself" + end + + spec = eval(code, binding, file) + # for out-of-place build + collected_files = files ? spec.files.concat(files).uniq : spec.files + spec.files = collected_files.map do |f| + if !File.exist?(File.join(base || ".", f)) && f.end_with?(".rb") + "lib/#{f}" + else + f + end + end + unless Gem::Specification === spec + raise TypeError, "[#{file}] isn't a Gem::Specification (#{spec.class} instead)." + end + spec.loaded_from = base ? File.join(base, File.basename(file)) : file + spec.date = RUBY_RELEASE_DATE + + spec +end + +def install_default_gem(dir, srcdir, bindir) + gem_dir = Gem.default_dir + install_dir = with_destdir(gem_dir) + prepare "default gems from #{dir}", gem_dir + RbInstall.no_write do + # Record making directories + makedirs(Gem.ensure_default_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) + end + + options = { + :install_dir => with_destdir(gem_dir), + :bin_dir => with_destdir(bindir), + :ignore_dependencies => true, + :dir_mode => $dir_mode, + :data_mode => $data_mode, + :prog_mode => $script_mode, + :wrappers => true, + :format_executable => true, + :install_as_default => true, + } + default_spec_dir = Gem.default_specifications_dir + + base = "#{srcdir}/#{dir}" + gems = Dir.glob("**/*.gemspec", base: base).map {|src| + files = RbInstall::Specs::FileCollector.for(srcdir, dir, src).collect + if files.empty? + next + end + load_gemspec("#{base}/#{src}", files: files) + } + gems.compact.sort_by(&:name).each do |gemspec| + old_gemspecs = Dir[File.join(with_destdir(default_spec_dir), "#{gemspec.name}-*.gemspec")] + if old_gemspecs.size > 0 + old_gemspecs.each {|spec| rm spec } + end + + full_name = "#{gemspec.name}-#{gemspec.version}" + + gemspec.loaded_from = File.join srcdir, gemspec.spec_name + + package = RbInstall::DirPackage.new gemspec, {gemspec.bindir => 'libexec'} + ins = RbInstall::UnpackedInstaller.new(package, options) + puts "#{INDENT}#{gemspec.name} #{gemspec.version}" + ins.install + end +end + +def mdoc_file?(mdoc) + /^\.Nm / =~ File.read(mdoc, 1024) +end + +# :startdoc: install?(:local, :arch, :bin, :'bin-arch') do - prepare "binary commands", bindir + prepare "binary commands", (dest = archbindir || bindir) - install ruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + def (bins = []).add(name) + push(name) + name + end + + install bins.add(ruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip if rubyw_install_name and !rubyw_install_name.empty? - install rubyw_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + install bins.add(rubyw_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip + end + # emcc produces ruby and ruby.wasm, the first is a JavaScript file of runtime support + # to load and execute the second .wasm file. Both are required to execute ruby + if RUBY_PLATFORM =~ /emscripten/ and File.exist? ruby_install_name+".wasm" + install bins.add(ruby_install_name+".wasm"), dest, :mode => $prog_mode, :strip => $strip end if File.exist? goruby_install_name+exeext - install goruby_install_name+exeext, bindir, :mode => $prog_mode, :strip => $strip + install bins.add(goruby_install_name+exeext), dest, :mode => $prog_mode, :strip => $strip end if enable_shared and dll != lib - install dll, bindir, :mode => $prog_mode, :strip => $strip + install bins.add(dll), dest, :mode => $prog_mode, :strip => $strip + end + if archbindir + prepare "binary command links", bindir + relpath = Path.relative(archbindir, bindir) + bins.each do |f| + ln_sf(File.join(relpath, f), File.join(bindir, f)) + end end end -install?(:local, :arch, :lib) do +install?(:local, :arch, :lib, :'lib-arch') do prepare "base libraries", libdir install lib, libdir, :mode => $prog_mode, :strip => $strip unless lib == arc - install arc, libdir, :mode => $data_mode + install arc, libdir, :mode => $data_mode unless CONFIG["INSTALL_STATIC_LIBRARY"] == "no" if dll == lib and dll != arc - for link in CONFIG["LIBRUBY_ALIASES"].split + for link in CONFIG["LIBRUBY_ALIASES"].split - [File.basename(dll)] ln_sf(dll, File.join(libdir, link)) end end @@ -378,6 +1008,11 @@ install?(:local, :arch, :data) do if pc and File.file?(pc) and File.size?(pc) prepare "pkgconfig data", pkgconfigdir = File.join(libdir, "pkgconfig") install pc, pkgconfigdir, :mode => $data_mode + if (pkgconfig_base = CONFIG["libdir", true]) != libdir + prepare "pkgconfig data link", File.join(pkgconfig_base, "pkgconfig") + ln_sf(File.join("..", Path.relative(pkgconfigdir, pkgconfig_base), pc), + File.join(pkgconfig_base, "pkgconfig", pc)) + end end end @@ -387,18 +1022,30 @@ install?(:ext, :arch, :'ext-arch') do install_recursive("#{$extout}/#{CONFIG['arch']}", archlibdir, :no_install => noinst, :mode => $prog_mode, :strip => $strip) prepare "extension objects", sitearchlibdir prepare "extension objects", vendorarchlibdir + if extso = File.read("exts.mk")[/^EXTSO[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] and + !(extso = extso.gsub(/\\\n/, '').split).empty? + libpathenv = CONFIG["LIBPATHENV"] + dest = CONFIG[!libpathenv || libpathenv == "PATH" ? "bindir" : "libdir"] + prepare "external libraries", dest + for file in extso + install file, dest, :mode => $prog_mode + end + end end -install?(:ext, :arch, :hdr, :'arch-hdr') do + +install?(:ext, :arch, :hdr, :'arch-hdr', :'hdr-arch') do prepare "extension headers", archhdrdir install_recursive("#{$extout}/include/#{CONFIG['arch']}", archhdrdir, :glob => "*.h", :mode => $data_mode) end + install?(:ext, :comm, :'ext-comm') do prepare "extension scripts", rubylibdir install_recursive("#{$extout}/common", rubylibdir, :mode => $data_mode) prepare "extension scripts", sitelibdir prepare "extension scripts", vendorlibdir end -install?(:ext, :comm, :hdr, :'comm-hdr') do + +install?(:ext, :comm, :hdr, :'comm-hdr', :'hdr-comm') do hdrdir = rubyhdrdir + "/ruby" prepare "extension headers", hdrdir install_recursive("#{$extout}/include/ruby", hdrdir, :glob => "*.h", :mode => $data_mode) @@ -408,104 +1055,33 @@ install?(:doc, :rdoc) do if $rdocdir ridatadir = File.join(CONFIG['ridir'], CONFIG['ruby_version'], "system") prepare "rdoc", ridatadir - install_recursive($rdocdir, ridatadir, :mode => $data_mode) + install_recursive($rdocdir, ridatadir, :no_install => rdoc_noinst, :mode => $data_mode) + end +end + +install?(:doc, :html) do + if $htmldir + prepare "html-docs", docdir + install_recursive($htmldir, docdir+"/html", :no_install => rdoc_noinst, :mode => $data_mode) end end + install?(:doc, :capi) do prepare "capi-docs", docdir install_recursive "doc/capi", docdir+"/capi", :mode => $data_mode end -if load_relative - PROLOG_SCRIPT = <<EOS -#!/bin/sh\n# -*- ruby -*- -bindir="${0%/*}" -EOS - if CONFIG["LIBRUBY_RELATIVE"] != 'yes' and libpathenv = CONFIG["LIBPATHENV"] - pathsep = File::PATH_SEPARATOR - PROLOG_SCRIPT << <<EOS -prefix="${bindir%/bin}" -export #{libpathenv}="$prefix/lib${#{libpathenv}:+#{pathsep}$#{libpathenv}}" -EOS - end - PROLOG_SCRIPT << %Q[exec "$bindir/#{ruby_install_name}" -x "$0" "$@"\n] -else - PROLOG_SCRIPT = nil -end - install?(:local, :comm, :bin, :'bin-comm') do prepare "command scripts", bindir - ruby_shebang = File.join(bindir, ruby_install_name) - if File::ALT_SEPARATOR - ruby_bin = ruby_shebang.tr(File::SEPARATOR, File::ALT_SEPARATOR) - if $cmdtype == 'exe' - stub = File.open("rubystub.exe", "rb") {|f| f.read} << "\n" rescue nil - end - end - if trans = CONFIG["program_transform_name"] - exp = [] - trans.gsub!(/\$\$/, '$') - trans.scan(%r[\G[\s;]*(/(?:\\.|[^/])*/)?([sy])(\\?\W)((?:(?!\3)(?:\\.|.))*)\3((?:(?!\3)(?:\\.|.))*)\3([gi]*)]) do - |addr, cmd, sep, pat, rep, opt| - addr &&= Regexp.new(addr[/\A\/(.*)\/\z/, 1]) - case cmd - when 's' - next if pat == '^' and rep.empty? - exp << [addr, (opt.include?('g') ? :gsub! : :sub!), - Regexp.new(pat, opt.include?('i')), rep.gsub(/&/){'\&'}] - when 'y' - exp << [addr, :tr!, Regexp.quote(pat), rep] - end - end - trans = proc do |base| - exp.each {|addr, opt, pat, rep| base.__send__(opt, pat, rep) if !addr or addr =~ base} - base - end - elsif /ruby/ =~ ruby_install_name - trans = proc {|base| ruby_install_name.sub(/ruby/, base)} - else - trans = proc {|base| base} - end - prebatch = ':""||{ ""=> %q<-*- ruby -*-'"\n" - postbatch = PROLOG_SCRIPT ? "};{\n#{PROLOG_SCRIPT.sub(/\A(?:#.*\n)*/, '')}" : '' - postbatch << ">,\n}\n" - postbatch.gsub!(/(?=\n)/, ' #') install_recursive(File.join(srcdir, "bin"), bindir, :maxdepth => 1) do |src, cmd| - cmd = cmd.sub(/[^\/]*\z/m) {|n| RbConfig.expand(trans[n])} - - shebang, body = open(src, "rb") do |f| - next f.gets, f.read - end - shebang or raise "empty file - #{src}" - if PROLOG_SCRIPT and !$cmdtype - shebang.sub!(/\A(\#!.*?ruby\b)?/) {PROLOG_SCRIPT + ($1 || "#!ruby\n")} - else - shebang.sub!(/\A(\#!.*?ruby\b)?/) {"#!" + ruby_shebang + ($1 ? "" : "\n")} - end - shebang.sub!(/\r$/, '') - body.gsub!(/\r$/, '') - - cmd << ".#{$cmdtype}" if $cmdtype - open_for_install(cmd, $script_mode) do - case $cmdtype - when "exe" - stub + shebang + body - when "cmd" - prebatch + <<"/EOH" << postbatch << shebang << body -@"%~dp0#{ruby_install_name}" -x "%~f0" %* -@exit /b %ERRORLEVEL% -/EOH - else - shebang + body - end - end + $script_installer.install(src, cmd) end end install?(:local, :comm, :lib) do prepare "library scripts", rubylibdir - noinst = %w[README* *.txt *.rdoc *.gemspec] + noinst = %w[*.txt *.rdoc *.gemspec] install_recursive(File.join(srcdir, "lib"), rubylibdir, :no_install => noinst, :mode => $data_mode) end @@ -517,209 +1093,116 @@ install?(:local, :comm, :hdr, :'comm-hdr') do noinst << "win32.h" end noinst = nil if noinst.empty? - install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.h", :mode => $data_mode) + install_recursive(File.join(srcdir, "include"), rubyhdrdir, :no_install => noinst, :glob => "*.{h,hpp}", :mode => $data_mode) end install?(:local, :comm, :man) do mdocs = Dir["#{srcdir}/man/*.[1-9]"] prepare "manpages", mandir, ([] | mdocs.collect {|mdoc| mdoc[/\d+$/]}).sort.collect {|sec| "man#{sec}"} - mandir = File.join(mandir, "man") + mantype, suffix, compress = Compressors.for($mantype) has_goruby = File.exist?(goruby_install_name+exeext) - require File.join(srcdir, "tool/mdoc2man.rb") if $mantype != "doc" + require File.join(srcdir, "tool/mdoc2man.rb") if /\Adoc\b/ !~ mantype mdocs.each do |mdoc| - next unless File.file?(mdoc) and open(mdoc){|fh| fh.read(1) == '.'} + next unless File.file?(mdoc) and File.read(mdoc, 1) == '.' base = File.basename(mdoc) if base == "goruby.1" next unless has_goruby end - destdir = mandir + (section = mdoc[/\d+$/]) - destname = ruby_install_name.sub(/ruby/, base.chomp(".#{section}")) + destdir = File.join(mandir, "man" + (section = mdoc[/\d+$/])) + destname = $script_installer.transform(base.chomp(".#{section}")) destfile = File.join(destdir, "#{destname}.#{section}") - if $mantype == "doc" - install mdoc, destfile, :mode => $data_mode + if /\Adoc\b/ =~ mantype or !mdoc_file?(mdoc) + if compress + begin + w = IO.popen(compress, "rb", in: mdoc, &:read) + rescue + else + destfile << suffix + end + end + if w + open_for_install(destfile, $data_mode) {w} + else + install mdoc, destfile, :mode => $data_mode + end else class << (w = []) alias print push end - open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} + File.open(mdoc) {|r| Mdoc2Man.mdoc2man(r, w)} w = w.join("") - case $mantype - when /\.(?:(gz)|bz2)\z/ - suffix = $& - compress = $1 ? "gzip" : "bzip2" - require 'tmpdir' - Dir.mktmpdir("man") {|d| - dest = File.join(d, File.basename(destfile)) - File.open(dest, "wb") {|f| f.write w} - if system(compress, dest) - w = File.open(dest+suffix, "rb") {|f| f.read} - destfile << suffix + if compress + begin + w = IO.popen(compress, "r+b") do |f| + Thread.start {f.write w; f.close_write} + f.read end - } + rescue + else + destfile << suffix + end end open_for_install(destfile, $data_mode) {w} end end end -module RbInstall - module Specs - class FileCollector - def initialize(base_dir) - @base_dir = base_dir - end - - def collect - (ruby_libraries + built_libraries).sort - end - - private - def type - /\/(ext|lib)?\/.*?\z/ =~ @base_dir - $1 - end - - def ruby_libraries - case type - when "ext" - prefix = "#{$extout}/common/" - base = "#{prefix}#{relative_base}" - when "lib" - base = @base_dir - prefix = base.sub(/lib\/.*?\z/, "") + "lib/" - end - - Dir.glob("#{base}{.rb,/**/*.rb}").collect do |ruby_source| - remove_prefix(prefix, ruby_source) - end - end - - def built_libraries - case type - when "ext" - prefix = "#{$extout}/#{CONFIG['arch']}/" - base = "#{prefix}#{relative_base}" - dlext = CONFIG['DLEXT'] - Dir.glob("#{base}{.#{dlext},/**/*.#{dlext}}").collect do |built_library| - remove_prefix(prefix, built_library) - end - when "lib" - [] - end - end - - def relative_base - /\/#{Regexp.escape(type)}\/(.*?)\z/ =~ @base_dir - $1 - end - - def remove_prefix(prefix, string) - string.sub(/\A#{Regexp.escape(prefix)}/, "") - end - end - - class Reader < Struct.new(:src) - def gemspec - @gemspec ||= begin - spec = Gem::Specification.load(src) || raise("invalid spec in #{src}") - file_collector = FileCollector.new(File.dirname(src)) - spec.files = file_collector.collect - spec - end - end - - def spec_source - @gemspec.to_ruby - end +install?(:dbg, :nodefault) do + prepare "debugger commands", bindir + prepare "debugger scripts", rubylibdir + conf = MAKEFILE_CONFIG.merge({"prefix"=>"${prefix#/}"}) + Dir.glob(File.join(srcdir, "template/ruby-*db.in")) do |src| + cmd = $script_installer.transform(File.basename(src, ".in")) + open_for_install(File.join(bindir, cmd), $script_mode) { + RbConfig.expand(File.read(src), conf) + } + end + Dir.glob(File.join(srcdir, "misc/lldb_*")) do |src| + if File.directory?(src) + install_recursive src, File.join(rubylibdir, File.basename(src)) + else + install src, rubylibdir end end - - class UnpackedInstaller < Gem::Installer - module DirPackage - def extract_files(destination_dir, pattern = "*") - path = File.dirname(@gem.path) - return if path == destination_dir - File.chmod(0700, destination_dir) - mode = pattern == "bin/*" ? $script_mode : $data_mode - install_recursive(path, without_destdir(destination_dir), - :glob => pattern, - :no_install => "*.gemspec", - :mode => mode) - File.chmod($dir_mode, destination_dir) + install File.join(srcdir, ".gdbinit"), File.join(rubylibdir, "gdbinit") + if $debug_symbols + { + ruby_install_name => archbindir || bindir, + rubyw_install_name => archbindir || bindir, + goruby_install_name => archbindir || bindir, + dll => libdir, + }.each do |src, dest| + next if src.empty? + src += $debug_symbols + if File.directory?(src) + install_recursive src, File.join(dest, src) end end - - def initialize(spec, *options) - super(spec.loaded_from, *options) - @package.extend(DirPackage).spec = spec - end - - def write_cache_file - end end end -class Gem::Installer - install = instance_method(:install) - define_method(:install) do - spec.post_install_message = nil - install.bind(self).call - end - - generate_bin_script = instance_method(:generate_bin_script) - define_method(:generate_bin_script) do |filename, bindir| - generate_bin_script.bind(self).call(filename, bindir) - File.chmod($script_mode, File.join(bindir, formatted_program_filename(filename))) - end +install?(:ext, :comm, :gem, :'default-gems', :'default-gems-comm') do + install_default_gem('lib', srcdir, bindir) end -# :startdoc: - -install?(:ext, :comm, :gem) do - gem_dir = Gem.default_dir - directories = Gem.ensure_gem_subdirectories(gem_dir, :mode => $dir_mode) - prepare "default gems", gem_dir, directories - - spec_dir = File.join(gem_dir, directories.grep(/^spec/)[0]) - default_spec_dir = "#{spec_dir}/default" - makedirs(default_spec_dir) - - gems = {} - - Dir.glob(srcdir+"/{lib,ext}/**/*.gemspec").each do |src| - specgen = RbInstall::Specs::Reader.new(src) - gems[specgen.gemspec.name] ||= specgen - end - - gems.sort.each do |name, specgen| - gemspec = specgen.gemspec - full_name = "#{gemspec.name}-#{gemspec.version}" - - puts "#{" "*30}#{gemspec.name} #{gemspec.version}" - gemspec_path = File.join(default_spec_dir, "#{full_name}.gemspec") - open_for_install(gemspec_path, $data_mode) do - specgen.spec_source - end - - unless gemspec.executables.empty? then - bin_dir = File.join(gem_dir, 'gems', full_name, 'bin') - makedirs(bin_dir) - - execs = gemspec.executables.map {|exec| File.join(srcdir, 'bin', exec)} - install(execs, bin_dir, :mode => $script_mode) - end - end +install?(:ext, :arch, :gem, :'default-gems', :'default-gems-arch') do + install_default_gem('ext', srcdir, bindir) end -install?(:ext, :comm, :gem) do +install?(:ext, :comm, :gem, :'bundled-gems') do gem_dir = Gem.default_dir - directories = Gem.ensure_gem_subdirectories(gem_dir, :mode => $dir_mode) - prepare "bundle gems", gem_dir, directories install_dir = with_destdir(gem_dir) + prepare "bundled gems", gem_dir + RbInstall.no_write do + # Record making directories + makedirs(Gem.ensure_gem_subdirectories(install_dir, $dir_mode).map {|d| File.join(gem_dir, d)}) + end + installed_gems = {} + skipped = {} options = { :install_dir => install_dir, :bin_dir => with_destdir(bindir), @@ -727,35 +1210,92 @@ install?(:ext, :comm, :gem) do :ignore_dependencies => true, :dir_mode => $dir_mode, :data_mode => $data_mode, - :prog_mode => $prog_mode, + :prog_mode => $script_mode, :wrappers => true, :format_executable => true, } - Gem::Specification.each_spec([srcdir+'/gems/*']) do |spec| - ins = RbInstall::UnpackedInstaller.new(spec, options) - puts "#{" "*30}#{spec.name} #{spec.version}" + + extensions_dir = Gem::StubSpecification.gemspec_stub("", gem_dir, gem_dir).extensions_dir + specifications_dir = File.join(gem_dir, "specifications") + build_dir = Gem::StubSpecification.gemspec_stub("", ".bundle", ".bundle").extensions_dir + + # We are about to build extensions, and want to configure extensions with the + # newly installed ruby. + Gem.instance_variable_set(:@ruby, with_destdir(File.join(bindir, ruby_install_name))) + # Prevent fake.rb propagation. It conflicts with the natural mkmf configs of + # the newly installed ruby. + ENV.delete('RUBYOPT') + + collector = RbInstall::Specs::FileCollector::UnpackedGem + File.foreach("#{srcdir}/gems/bundled_gems") do |name| + next if /^\s*(?:#|$)/ =~ name + next unless /^(\S+)\s+(\S+).*/ =~ name + gem = $1 + gem_name = "#$1-#$2" + path = [ + # gemspec that removed duplicated dependencies of bundled gems + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem}.gemspec", + # gemspec for C ext gems, It has the original dependencies + # ex .bundle/gems/debug-1.7.1/debug-1.7.1.gemspec + "#{srcdir}/.bundle/gems/#{gem_name}/#{gem_name}.gemspec", + # original gemspec generated by rubygems + "#{srcdir}/.bundle/specifications/#{gem_name}.gemspec" + ].find { |gemspec| File.exist?(gemspec) } + if path.nil? + skipped[gem_name] = "gemspec not found" + next + end + base = "#{srcdir}/.bundle/gems/#{gem_name}" + files = collector.new(path, base, nil).collect + files.delete("#{gem}.gemspec") + files.delete("#{gem_name}.gemspec") + spec = load_gemspec(path, base, files: files) + unless spec.platform == Gem::Platform::RUBY + skipped[gem_name] = "not ruby platform (#{spec.platform})" + next + end + unless spec.full_name == gem_name + skipped[gem_name] = "full name unmatch #{spec.full_name}" + next + end + # Skip install C ext bundled gem if it is build failed or not found + if !spec.extensions.empty? && !File.exist?("#{build_dir}/#{gem_name}/gem.build_complete") + skipped[gem_name] = "extensions not found or build failed #{spec.full_name}" + next + end + spec.extension_dir = "#{extensions_dir}/#{spec.full_name}" + + package = RbInstall::DirPackage.new spec + ins = RbInstall::UnpackedInstaller.new(package, options) + puts "#{INDENT}#{spec.name} #{spec.version}" ins.install + install_recursive("#{build_dir}/#{gem_name}", "#{extensions_dir}/#{gem_name}") do |src, dest| + # puts "#{INDENT} #{dest[extensions_dir.size+gem_name.size+2..-1]}" + install src, dest, :mode => (File.executable?(src) ? $prog_mode : $data_mode) + end installed_gems[spec.full_name] = true end installed_gems, gems = Dir.glob(srcdir+'/gems/*.gem').partition {|gem| installed_gems.key?(File.basename(gem, '.gem'))} unless installed_gems.empty? + prepare "bundled gem cache", gem_dir+"/cache" install installed_gems, gem_dir+"/cache" end - next if gems.empty? - if defined?(Zlib) - Gem.instance_variable_set(:@ruby, with_destdir(File.join(bindir, ruby_install_name))) + unless gems.empty? + skipped.default = "not found in bundled_gems" + puts "skipped bundled gems:" gems.each do |gem| - Gem.install(gem, Gem::Requirement.default, options) - gemname = File.basename(gem) - puts "#{" "*30}#{gemname}" - end - # fix directory permissions - # TODO: Gem.install should accept :dir_mode option or something - File.chmod($dir_mode, *Dir.glob(install_dir+"/**/")) - # fix .gemspec permissions - File.chmod($data_mode, *Dir.glob(install_dir+"/specifications/*.gemspec")) - else - puts "skip installing bundle gems because of lacking zlib" + gem = File.basename(gem) + printf " %-31s %s\n", gem, skipped[gem.chomp(".gem")] + end + end +end + +install?('modular-gc') do + if modular_gc_dir = CONFIG['modular_gc_dir'] and !modular_gc_dir.empty? + dlext = CONFIG['DLEXT', true] + modular_gc_dir = File.expand_path(modular_gc_dir, CONFIG['prefix']) + prepare "modular GC library", modular_gc_dir + install Dir.glob("gc/*/librubygc.*.#{dlext}"), modular_gc_dir end end @@ -766,8 +1306,7 @@ include FileUtils::NoWrite if $dryrun @fileutils_output = STDOUT @fileutils_label = '' -all = $install.delete(:all) -$install << :local << :ext if $install.empty? +$install << :all if $install.empty? installs = $install.map do |inst| if !(procs = $install_procs[inst]) || procs.empty? next warn("unknown install target - #{inst}") @@ -775,8 +1314,7 @@ installs = $install.map do |inst| procs end installs.flatten! -installs.uniq! -installs |= $install_procs[:all] if all +installs -= $exclude.map {|exc| $install_procs[exc]}.flatten installs.each do |block| dir = Dir.pwd begin @@ -785,5 +1323,9 @@ installs.each do |block| Dir.chdir(dir) end end +unless installs.empty? or $destdir.empty? + require_relative 'lib/colorize' + puts "Installed under #{Colorize.new.info($destdir)}" +end # vi:set sw=2: diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests new file mode 100644 index 0000000000..4bcb5707a5 --- /dev/null +++ b/tool/rbs_skip_tests @@ -0,0 +1,57 @@ +# Running tests of RBS gem may fail because of various reasons. +# You can skip tests of RBS gem using this file, instead of pushing a new commit to `ruby/rbs` repository. +# +# The most frequently seen reason is the incompatibilities introduced to the unreleased version, including +# +# * Strict argument type check is introduced +# * A required method parameter is added +# * A method/class is removed +# +# Feel free to skip the tests with this file for that case. +# +# Syntax: +# +# $(test-case-name) ` ` $(optional comment) # Skipping single test case +# $(test-class-name) ` ` $(optional comment) # Skipping a test class +# + +## Failed tests because of testing environment + +test_collection_install(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__config__stdlib_source(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__dependency_no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__no_bundled(RBS::CliTest) running tests without Bundler +test_collection_install__mutex_m__rbs_dependency_and__gem_dependency(RBS::CliTest) running tests without Bundler +test_collection_install_frozen(RBS::CliTest) running tests without Bundler +test_collection_install_gemspec(RBS::CliTest) running tests without Bundler +test_collection_update(RBS::CliTest) running tests without Bundler + +NetSingletonTest depending on external resources +NetInstanceTest depending on external resources +TestHTTPRequest depending on external resources +TestSingletonNetHTTPResponse depending on external resources +TestInstanceNetHTTPResponse depending on external resources + +test_TOPDIR(RbConfigSingletonTest) `TOPDIR` is `nil` during CI while RBS type is declared as `String` + +# Failing because ObjectSpace.count_nodes has been removed +test_count_nodes(ObjectSpaceTest) + +## Unknown failures + +# NoMethodError: undefined method 'inspect' for an instance of RBS::UnitTest::Convertibles::ToInt +test_compile(RegexpSingletonTest) +test_linear_time?(RegexpSingletonTest) +test_new(RegexpSingletonTest) + +## Failed tests caused by unreleased version of Ruby +test_source_location(MethodInstanceTest) +test_source_location(ProcInstanceTest) +test_source_location(UnboundMethodInstanceTest) + +# Errno::ENOENT: No such file or directory - bundle +test_collection_install__pathname_set(RBS::CliTest) +test_collection_install__set_pathname__manifest(RBS::CliTest) diff --git a/tool/rbs_skip_tests_windows b/tool/rbs_skip_tests_windows new file mode 100644 index 0000000000..db12c69419 --- /dev/null +++ b/tool/rbs_skip_tests_windows @@ -0,0 +1,111 @@ +ARGFTest Failing on Windows + +RactorSingletonTest Hangs up on Windows +RactorInstanceTest Hangs up on Windows + +# NotImplementedError: fileno() function is unimplemented on this machine +test_fileno(DirInstanceTest) +test_fchdir(DirSingletonTest) +test_for_fd(DirSingletonTest) + +# ArgumentError: user root doesn't exist +test_home(DirSingletonTest) + +# NameError: uninitialized constant Etc::CS_PATH +test_confstr(EtcSingletonTest) + +# NameError: uninitialized constant Etc::SC_ARG_MAX +test_sysconf(EtcSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-udw6rx/chmod +test_chmod(FileInstanceTest) +test_chmod(FileInstanceTest) +test_truncate(FileInstanceTest) + +# Errno::EISDIR: Is a directory @ rb_sysopen - C:/a/ruby/ruby/src/gems/src/rbs/test/stdlib +test_directory?(FileSingletonTest) + +# NotImplementedError: lutime() function is unimplemented on this machine +test_lutime(FileSingletonTest) + +# NotImplementedError: mkfifo() function is unimplemented on this machine +test_mkfifo(FileSingletonTest) + +# Returns `nil` on Windows +test_getgrgid(EtcSingletonTest) +test_getgrnam(EtcSingletonTest) +test_getpwnam(EtcSingletonTest) +test_getpwuid(EtcSingletonTest) + +# Returns `false` +test_setgid?(FileSingletonTest) +test_setuid?(FileSingletonTest) +test_sticky?(FileSingletonTest) + +test_world_readable?(FileSingletonTest) # Returns `420` +test_world_readable?(FileStatInstanceTest) # Returns `420` +test_world_writable?(FileSingletonTest) # Returns `nil` +test_dev_major(FileStatInstanceTest) # Returns `nil` +test_dev_minor(FileStatInstanceTest) # Returns `nil` +test_rdev_major(FileStatInstanceTest) # Returns `nil` +test_rdev_minor(FileStatInstanceTest) # Returns `nil` + +# ArgumentError: wrong number of arguments (given -403772944, expected 0+) +test_curry(MethodInstanceTest) + +# ArgumentError: no output encoding given +test_tolocale(KconvSingletonTest) + +# Errno::EINVAL: Invalid argument - : +test_system(KernelInstanceTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_load(OpenSSLConfigSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - +test_parse(OpenSSLConfigSingletonTest) +test_parse_config(OpenSSLConfigSingletonTest) + +# OpenSSL::ConfigError: BIO_new_file: no such file +test_each(OpenSSLConfigTest) +test_lookup_and_set(OpenSSLConfigTest) +test_sections(OpenSSLConfigTest) + +# OpenSSL::SSL::SSLError: SSL_connect returned=1 errno=0 peeraddr=185.199.108.153:443 state=error: certificate verify failed (unable to get local issuer certificate) +test_URI_open(OpenURISingletonTest) + +# ArgumentError: both textmode and binmode specified +test_binwrite(PathnameInstanceTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-delete-test20250813-10156-mb3e9i +test_delete(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/rbs-pathname-binwrite-test20250813-10156-sh8145 +test_open(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_truncate(PathnameInstanceTest) +# Errno::EACCES: Permission denied @ rb_file_s_truncate - C:/a/_temp/rbs-pathname-truncate-test20250813-10156-dqqiw3 +test_unlink(PathnameInstanceTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /etc/resolv.conf +test_parse_resolv_conf(ResolvDNSConfigSingletonTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvInstanceTest) +# Resolv::ResolvError: no name for 127.0.0.1 +test_getname(ResolvSingletonTest) + +# ArgumentError: unsupported signal 'SIGUSR2' +test_trap(SignalSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-mgr4tx +test_create(TempfileSingletonTest) + +# Errno::ENOENT: No such file or directory @ rb_sysopen - /tmp/README.md20250813-10156-hp9nzu +test_initialize(TempfileSingletonTest) +test_new(TempfileSingletonTest) + +# Errno::EACCES: Permission denied @ apply2files - C:/a/_temp/d20250813-10156-f8z9pn/test.gz +test_open(ZlibGzipReaderSingletonTest) + +# Errno::EACCES: Permission denied @ rb_file_s_rename +# D:/a/ruby/ruby/src/lib/rubygems/util/atomic_file_writer.rb:42:in 'File.rename' +test_write_binary(GemSingletonTest) diff --git a/tool/rbuninstall.rb b/tool/rbuninstall.rb index 1a11766790..60f5241a4f 100755 --- a/tool/rbuninstall.rb +++ b/tool/rbuninstall.rb @@ -1,4 +1,8 @@ #! /usr/bin/ruby -nl + +# Used by the "make uninstall" target to uninstall Ruby. +# See common.mk for more details. + BEGIN { $dryrun = false $tty = STDOUT.tty? @@ -17,16 +21,35 @@ BEGIN { end $dirs = [] $files = [] + COLUMNS = $tty && (ENV["COLUMNS"]&.to_i || begin require 'io/console/size'; rescue; else IO.console_size&.at(1); end)&.then do |n| + n-1 if n > 1 + end + if COLUMNS + $column = 0 + def message(str = nil) + $stdout.print "\b \b" * $column + if str + if str.size > COLUMNS + str = "..." + str[(-COLUMNS+3)..-1] + end + $stdout.print str + end + $stdout.flush + $column = str&.size || 0 + end + else + alias message puts + end } list = ($_.chomp!('/') ? $dirs : $files) -$_ = File.join($destdir, $_) if $destdir list << $_ END { status = true - $\ = ors = (!$dryrun and $tty) ? "\e[K\r" : "\n" + $\ = nil $files.each do |file| - print "rm #{file}" + message "rm #{file}" unless $dryrun + file = File.join($destdir, file) if $destdir begin File.unlink(file) rescue Errno::ENOENT @@ -40,28 +63,37 @@ END { $dirs.each do |dir| unlink[dir] = true end + nonempty = {} while dir = $dirs.pop - print "rmdir #{dir}" + dir = File.dirname(dir) while File.basename(dir) == '.' + message "rmdir #{dir}" unless $dryrun + realdir = $destdir ? File.join($destdir, dir) : dir begin begin unlink.delete(dir) - Dir.rmdir(dir) + Dir.rmdir(realdir) rescue Errno::ENOTDIR - raise unless File.symlink?(dir) - File.unlink(dir) + raise unless File.symlink?(realdir) + File.unlink(realdir) end - rescue Errno::ENOENT, Errno::ENOTEMPTY + rescue Errno::ENOTEMPTY + nonempty[dir] = true + rescue Errno::ENOENT rescue status = false puts $! else + nonempty.delete(dir) parent = File.dirname(dir) $dirs.push(parent) unless parent == dir or unlink[parent] end end end - $\ = nil - print ors.chomp + message + unless nonempty.empty? + puts "Non empty director#{nonempty.size == 1 ? 'y' : 'ies'}:" + nonempty.each_key {|dir| print " #{dir}\n"} + end exit(status) } diff --git a/tool/rdoc-srcdir b/tool/rdoc-srcdir new file mode 100755 index 0000000000..ecc49b4b2c --- /dev/null +++ b/tool/rdoc-srcdir @@ -0,0 +1,30 @@ +#!ruby -W0 + +%w[tsort rdoc].each do |lib| + path = Dir.glob("#{File.dirname(__dir__)}/.bundle/gems/#{lib}-*").first + $LOAD_PATH.unshift("#{path}/lib") +end +require 'rdoc/rdoc' + +# Make only the output directory relative to the invoked directory. +invoked = Dir.pwd + +# Load options and parse files from srcdir. +Dir.chdir(File.dirname(__dir__)) + +options = RDoc::Options.load_options +options.title = options.title.sub(/Ruby \K.*version/) { + File.read("include/ruby/version.h") + .scan(/^ *# *define +RUBY_API_VERSION_(MAJOR|MINOR) +(\d+)/) + .sort # "MAJOR" < "MINOR", fortunately + .to_h.values.join(".") +} +options.parse ARGV + ["#{invoked}/rbconfig.rb"] + +options.singleton_class.define_method(:finish) do + super() + @op_dir = File.expand_path(@op_dir, invoked) +end + +# Do not hide errors when generating documents of Ruby itself. +RDoc::RDoc.new.document options diff --git a/tool/redmine-backporter.rb b/tool/redmine-backporter.rb index aa9991155c..95a9688cb2 100755 --- a/tool/redmine-backporter.rb +++ b/tool/redmine-backporter.rb @@ -9,13 +9,8 @@ require 'strscan' require 'optparse' require 'abbrev' require 'pp' -begin - require 'readline' -rescue LoadError - module Readline; end -end - -VERSION = '0.0.1' +require 'shellwords' +require 'reline' opts = OptionParser.new target_version = nil @@ -23,10 +18,9 @@ repo_path = nil api_key = nil ssl_verify = true opts.on('-k REDMINE_API_KEY', '--key=REDMINE_API_KEY', 'specify your REDMINE_API_KEY') {|v| api_key = v} -opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 2.1)') {|v| target_version = v} +opts.on('-t TARGET_VERSION', '--target=TARGET_VARSION', /\A\d(?:\.\d)+\z/, 'specify target version (ex: 3.1)') {|v| target_version = v} opts.on('-r RUBY_REPO_PATH', '--repository=RUBY_REPO_PATH', 'specify repository path') {|v| repo_path = v} opts.on('--[no-]ssl-verify', TrueClass, 'use / not use SSL verify') {|v| ssl_verify = v} -opts.version = VERSION opts.parse!(ARGV) http_options = {use_ssl: true} @@ -34,17 +28,17 @@ http_options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify $openuri_options = {} $openuri_options[:ssl_verify_mode] = OpenSSL::SSL::VERIFY_NONE unless ssl_verify -TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (raise 'need to specify TARGET_VERSION') +TARGET_VERSION = target_version || ENV['TARGET_VERSION'] || (puts opts.help; raise 'need to specify TARGET_VERSION') RUBY_REPO_PATH = repo_path || ENV['RUBY_REPO_PATH'] BACKPORT_CF_KEY = 'cf_5' STATUS_CLOSE = 5 -REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (raise 'need to specify REDMINE_API_KEY') +REDMINE_API_KEY = api_key || ENV['REDMINE_API_KEY'] || (puts opts.help; raise 'need to specify REDMINE_API_KEY') REDMINE_BASE = 'https://bugs.ruby-lang.org' @query = { 'f[]' => BACKPORT_CF_KEY, "op[#{BACKPORT_CF_KEY}]" => '~', - "v[#{BACKPORT_CF_KEY}][]" => "#{TARGET_VERSION}: REQUIRED", + "v[#{BACKPORT_CF_KEY}][]" => "\"#{TARGET_VERSION}: REQUIRED\"", 'limit' => 40, 'status_id' => STATUS_CLOSE, 'sort' => 'updated_on' @@ -69,74 +63,34 @@ COLORS = { } class String - def color(fore=nil, back=nil, bold: false, underscore: false) + def color(fore=nil, back=nil, opts={}, bold: false, underscore: false) seq = "" - if bold - seq << "\e[1m" + if bold || opts[:bold] + seq = seq + "\e[1m" end - if underscore - seq << "\e[2m" + if underscore || opts[:underscore] + seq = seq + "\e[2m" end if fore c = COLORS[fore] raise "unknown foreground color #{fore}" unless c - seq << "\e[#{c}m" + seq = seq + "\e[#{c}m" end if back c = COLORS[back] raise "unknown background color #{back}" unless c - seq << "\e[#{c + 10}m" + seq = seq + "\e[#{c + 10}m" end if seq.empty? self else - seq << self << "\e[0m" - end - end -end - -def wcwidth(wc) - return 8 if wc == "\t" - n = wc.ord - if n < 0x20 - 0 - elsif n < 0x80 - 1 - else - 2 - end -end - -def fold(str, col) - i = 0 - size = str.size - len = 0 - while i < size - case c = str[i] - when "\r", "\n" - len = 0 - else - d = wcwidth(c) - len += d - if len == col - str.insert(i+1, "\n") - len = 0 - i += 2 - next - elsif len > col - str.insert(i, "\n") - len = d - i += 2 - next - end + seq = seq + self + "\e[0m" end - i += 1 end - str end class StringScanner - # lx: limit of x (colmns of screen) + # lx: limit of x (columns of screen) # ly: limit of y (rows of screen) def getrows(lx, ly) cp1 = charpos @@ -201,86 +155,13 @@ def more(sio) end end -class << Readline - def readline(prompt = '') - console = IO.console - console.binmode - ly, lx = console.winsize - if /mswin|mingw/ =~ RUBY_PLATFORM or /^(?:vt\d\d\d|xterm)/i =~ ENV["TERM"] - cls = "\r\e[2K" - else - cls = "\r" << (" " * lx) - end - cls << "\r" << prompt - console.print prompt - console.flush - line = '' - while 1 - case c = console.getch - when "\r", "\n" - puts - HISTORY << line - return line - when "\C-?", "\b" # DEL/BS - print "\b \b" if line.chop! - when "\C-u" - print cls - line.clear - when "\C-d" - return nil if line.empty? - line << c - when "\C-p" - HISTORY.pos -= 1 - line = HISTORY.current - print cls - print line - when "\C-n" - HISTORY.pos += 1 - line = HISTORY.current - print cls - print line - else - if c >= " " - print c - line << c - end - end - end - end - - HISTORY = [] - def HISTORY.<<(val) - HISTORY.push(val) - @pos = self.size - self - end - def HISTORY.pos - @pos ||= 0 - end - def HISTORY.pos=(val) - @pos = val - if @pos < 0 - @pos = -1 - elsif @pos >= self.size - @pos = self.size - end - end - def HISTORY.current - @pos ||= 0 - if @pos < 0 || @pos >= self.size - '' - else - self[@pos] - end - end -end unless defined?(Readline.readline) - -def mergeinfo - `svn propget svn:mergeinfo #{RUBY_REPO_PATH}` +def find_git_log(pattern) + `git #{RUBY_REPO_PATH ? "-C #{RUBY_REPO_PATH.shellescape}" : ""} log --grep="#{pattern}"` end -def find_svn_log(pattern) - `svn log --xml --stop-on-copy --search="#{pattern}" #{RUBY_REPO_PATH}` +def has_commit(commit, branch) + base = RUBY_REPO_PATH ? ["-C", RUBY_REPO_PATH.shellescape] : nil + system("git", *base, "merge-base", "--is-ancestor", commit, branch) end def show_last_journal(http, uri) @@ -299,20 +180,23 @@ def show_last_journal(http, uri) puts x["notes"] end +def merger_path + RUBY_PLATFORM =~ /mswin|mingw/ ? 'merger' : File.expand_path('../merger.rb', __FILE__) +end + def backport_command_string unless @changesets.respond_to?(:validated) @changesets = @changesets.select do |c| - begin - uri = URI("#{REDMINE_BASE}/projects/ruby-trunk/repository/revisions/#{c}") - uri.read($openuri_options) - true - rescue - false - end + next false if c.match(/\A\d{1,6}\z/) # skip SVN revision + + # check if the Git revision is included in master + has_commit(c, "origin/master") + end.sort_by do |changeset| + Integer(IO.popen(%W[git show -s --format=%ct #{changeset}], &:read)) end @changesets.define_singleton_method(:validated){true} end - " backport --ticket=#{@issue} #{@changesets.join(',')}" + "#{merger_path} --ticket=#{@issue} #{@changesets.join(',')}" end def status_char(obj) @@ -325,33 +209,39 @@ def status_char(obj) end console = IO.console -row, col = console.winsize +row, = console.winsize @query['limit'] = row - 2 -puts "Backporter #{VERSION}".color(bold: true) + " for #{TARGET_VERSION}" +puts "Redmine Backporter".color(bold: true) + " for Ruby #{TARGET_VERSION}" class CommandSyntaxError < RuntimeError; end commands = { "ls" => proc{|args| raise CommandSyntaxError unless /\A(\d+)?\z/ =~ args - uri = URI(REDMINE_BASE+'/projects/ruby-trunk/issues.json?'+URI.encode_www_form(@query.dup.merge('page' => ($1 ? $1.to_i : 1)))) + uri = URI(REDMINE_BASE+'/projects/ruby-master/issues.json?'+URI.encode_www_form(@query.dup.merge('page' => ($1 ? $1.to_i : 1)))) # puts uri res = JSON(uri.read($openuri_options)) @issues = issues = res["issues"] from = res["offset"] + 1 total = res["total_count"] + closed = issues.count { |x, _| x["status"]["name"] == "Closed" } to = from + issues.size - 1 - puts "#{from}-#{to} / #{total}" + puts "#{from}-#{to} / #{total} (closed: #{closed})" issues.each_with_index do |x, i| - id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]]) + id = "##{x["id"]}".color(*PRIORITIES[x["priority"]["name"]], bold: x["status"]["name"] == "Closed") puts "#{'%2d' % i} #{id} #{x["priority"]["name"][0]} #{status_char(x["status"])} #{x["subject"][0,80]}" end }, "show" => proc{|args| - raise CommandSyntaxError unless /\A(\d+)\z/ =~ args - id = $1.to_i - id = @issues[id]["id"] if @issues && id < @issues.size - @issue = id + if /\A(\d+)\z/ =~ args + id = $1.to_i + id = @issues[id]["id"] if @issues && id < @issues.size + @issue = id + elsif @issue + id = @issue + else + raise CommandSyntaxError + end uri = "#{REDMINE_BASE}/issues/#{id}" uri = URI(uri+".json?include=children,attachments,relations,changesets,journals") res = JSON(uri.read($openuri_options)) @@ -359,8 +249,14 @@ commands = { unless i["changesets"] abort "You don't have view_changesets permission" end + unless i["custom_fields"] + puts "The specified ticket \##{@issue} seems to be a feature ticket" + @issue = nil + next + end id = "##{i["id"]}".color(*PRIORITIES[i["priority"]["name"]]) sio = StringIO.new + sio.set_encoding("utf-8") sio.puts <<eom #{i["subject"].color(bold: true, underscore: true)} #{i["project"]["name"]} [#{i["tracker"]["name"]} #{id}] #{i["status"]["name"]} (#{i["created_on"]}) @@ -370,7 +266,7 @@ eom i["custom_fields"].each do |x| sio.puts "%-10s: %s" % [x["name"], x["value"]] end - #res["attachements"].each do |x| + #res["attachments"].each do |x| #end sio.puts i["description"] sio.puts @@ -397,17 +293,33 @@ eom "rel" => proc{|args| # this feature requires custom redmine which allows add_related_issue API - raise CommandSyntaxError unless /\A(\d+)\z/ =~ args + case args + when /\A\h{7,40}\z/ # Git + rev = args + uri = URI("#{REDMINE_BASE}/projects/ruby-master/repository/git/revisions/#{rev}/issues.json") + else + raise CommandSyntaxError + end unless @issue puts "ticket not selected" next end - rev = $1.to_i - uri = URI("#{REDMINE_BASE}/projects/ruby-trunk/repository/revisions/#{rev}/issues.json") + Net::HTTP.start(uri.host, uri.port, http_options) do |http| res = http.post(uri.path, "issue_id=#@issue", 'X-Redmine-API-Key' => REDMINE_API_KEY) + begin + res.value + rescue + if $!.respond_to?(:response) && $!.response.is_a?(Net::HTTPConflict) + $stderr.puts "the revision has already related to the ticket" + else + $stderr.puts "#{$!.class}: #{$!.message}\n\ndeployed redmine doesn't have https://github.com/ruby/bugs.ruby-lang.org/commit/01fbba60d68cb916ddbccc8a8710e68c5217171d\nask naruse or hsbt" + end + next + end puts res.body + @changesets << rev class << @changesets remove_method(:validated) rescue nil end @@ -416,7 +328,7 @@ eom "backport" => proc{|args| # this feature implies backport command which wraps tool/merger.rb - raise CommandSyntexError unless args.empty? + raise CommandSyntaxError unless args.empty? unless @issue puts "ticket not selected" next @@ -425,9 +337,10 @@ eom }, "done" => proc{|args| - raise CommandSyntaxError unless /\A(\d+)?(?:\s*-- +(.*))?\z/ =~ args - notes = $2 + raise CommandSyntaxError unless /\A(\d+)?(?: *by (\h+))?(?:\s*-- +(.*))?\z/ =~ args + notes = $3 notes.strip! if notes + rev = $2 if $1 i = $1.to_i i = @issues[i]["id"] if @issues && i < @issues.size @@ -438,11 +351,17 @@ eom next end - log = find_svn_log("##@issue]") - if log && /revision="(?<rev>\d+)/ =~ log - str = log[/merge revision\(s\) ([^:]+)(?=:)/] - str.insert(5, "d") - str = "ruby_#{TARGET_VERSION.tr('.','_')} r#{rev} #{str}." + if rev && has_commit(rev, "ruby_#{TARGET_VERSION.tr('.','_')}") + notes = "ruby_#{TARGET_VERSION.tr('.','_')} commit:#{rev}." + elsif rev.nil? && (log = find_git_log("##@issue]")) && !(revs = log.scan(/^commit (\h{40})$/).flatten).empty? + commits = revs.map { |rev| "commit:#{rev}" }.join(", ") + if merged_revs = log[/merge revision\(s\) ([^:]+)(?=:)/] + merged_revs.sub!(/\Amerge/, 'merged') + merged_revs.gsub!(/\h{8,40}/, 'commit:\0') + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits} #{merged_revs}." + else + str = "ruby_#{TARGET_VERSION.tr('.','_')} #{commits}." + end if notes str << "\n" str << notes @@ -459,7 +378,7 @@ eom res = http.get(uri.path) data = JSON(res.body) h = data["issue"]["custom_fields"].find{|x|x["id"]==5} - if h and val = h["value"] + if h and val = h["value"] and val != "" case val[/(?:\A|, )#{Regexp.quote TARGET_VERSION}: ([^,]+)/, 1] when 'REQUIRED', 'UNKNOWN', 'DONTNEED', 'WONTFIX' val[$~.offset(1)[0]...$~.offset(1)[1]] = 'DONE' @@ -472,7 +391,7 @@ eom raise "unknown status '#$1'" end else - val = '#{TARGET_VERSION}: DONE' + val = "#{TARGET_VERSION}: DONE" end data = { "issue" => { "custom_fields" => [ {"id"=>5, "value" => val} ] } } @@ -556,7 +475,7 @@ list = Abbrev.abbrev(commands.keys) @changesets = nil while true begin - l = Readline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " + l = Reline.readline "#{('#' + @issue.to_s).color(bold: true) if @issue}> " rescue Interrupt break end @@ -567,10 +486,10 @@ while true args = cmd cmd = "show" end + cmd = list[cmd] if commands[cmd].is_a? String - cmd = list[cmd] + cmd = list[commands[cmd]] end - cmd = list[cmd] begin if cmd commands[cmd].call(args) diff --git a/tool/release.sh b/tool/release.sh index 4c1d7ccd59..d467d8f24b 100755 --- a/tool/release.sh +++ b/tool/release.sh @@ -1,38 +1,27 @@ -#!/bin/sh +#!/bin/bash +# Bash version 3.2+ is required for regexp +# Usage: +# tool/release.sh 3.0.0 +# tool/release.sh 3.0.0-rc1 -RUBYDIR=/home/ftp/pub/ruby -EXTS='.tar.gz .tar.bz2 .tar.xz .zip' +EXTS='.tar.gz .tar.xz .zip' +if [[ -n $AWS_ACCESS_KEY_ID ]]; then + AWS_CLI_OPTS="" +else + AWS_CLI_OPTS="--profile ruby" +fi -releases=`ls ruby-*|grep -o 'ruby-[0-9]\.[0-9]\.[0-9]\(-\(preview\|rc\|p\)[0-9]\{1,4\}\)\?'|uniq` +ver=$1 +if [[ $ver =~ ^([1-9]\.[0-9])\.([0-9]|[1-9][0-9]|0-(preview[1-9]|rc[1-9]))$ ]]; then + : +else + echo $ver is not valid release version + exit 1 +fi -# check files -for r in $releases -do - echo "checking files for $r..." - for ext in $EXTS - do - if ! [ -f $r$ext ];then - echo "ERROR: $r$ext not found" - exit 1 - fi - done - echo "files are ok" -done - -# version directory -for r in $releases -do - xy=`echo $r|grep -o '[0-9]\.[0-9]'` - preview=`echo $r|grep -o -- '-\(preview\|rc\)'` - dir="${RUBYDIR}/$xy" - echo "$dir" - mkdir -p $dir - for ext in $EXTS - do - cp $r$ext $dir/$r$ext - ln -sf $xy/$r$ext ${RUBYDIR}/$r$ext - if [ x$preview = x ];then - ln -sf $xy/$r$ext ${RUBYDIR}/ruby-$xy-stable$ext - fi - done +short=${BASH_REMATCH[1]} +echo $ver +echo $short +for ext in $EXTS; do + aws $AWS_CLI_OPTS s3 cp s3://ftp.r-l.o/pub/tmp/ruby-$ver-draft$ext s3://ftp.r-l.o/pub/ruby/$short/ruby-$ver$ext done diff --git a/tool/releng/gen-mail.rb b/tool/releng/gen-mail.rb new file mode 100755 index 0000000000..17fa499d69 --- /dev/null +++ b/tool/releng/gen-mail.rb @@ -0,0 +1,55 @@ +#!/usr/bin/env ruby +require "open-uri" +require "yaml" + +lang = ARGV.shift +unless lang + abort "usage: #$1 {en,ja,release.md} | pbcopy" +end + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end +confirm_w_r_l_o_wd + +releases = YAML.safe_load_file('_data/releases.yml', permitted_classes: [Date]) + +case lang +when "en", "ja" + url = "https://hackmd.io/@naruse/ruby-relnote-#{lang}/download" + src = URI(url).read +else # the path of the Release note in markdown is given + src = File.read(lang) +end +src.gsub!(/[ \t]+$/, "") +src.sub!(/(?<!\n)\z/, "\n") +src.sub!(/^breaks: false\n/, '') + +if /^\{% assign release = site.data.releases \| where: "version", "([^"]+)" \| first %\}/ =~ src + version = $1 +else + abort %[#{url} doesn't include `{% assign release = site.data.releases | where: "version", "<version>" | first %}`] +end + +release = releases.find{|rel|rel['version'] == version} +unless release + abort "#{version} is not found in '_data/releases.yml'" +end + +src.gsub!(/^{% assign .*\n/, '') +src.gsub!(/\{\{(.*?)\}\}/) do + var = $1.strip + case var + when /\Arelease\.(.*)/ + val = release.dig(*$1.split('.')) + raise "invalid variable '#{var}'" unless val + else + raise "unknown variable '#{var}'" + end + val +end +puts src diff --git a/tool/releng/gen-release-note.rb b/tool/releng/gen-release-note.rb new file mode 100755 index 0000000000..5afd11b796 --- /dev/null +++ b/tool/releng/gen-release-note.rb @@ -0,0 +1,36 @@ +#!/usr/bin/env ruby +require 'open-uri' +require 'time' +require 'yaml' + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end +confirm_w_r_l_o_wd + +%w[ + https://hackmd.io/@naruse/ruby-relnote-en/download + https://hackmd.io/@naruse/ruby-relnote-ja/download +].each do |url| + src = URI(url).read + src.gsub!(/[ \t]+$/, "") + src.sub!(/\s+\z/, "\n") + src.sub!(/^breaks: false\n/, '') + if /^\{% assign release = site.data.releases \| where: "version", "([^"]+)" \| first %\}/ =~ src + version = $1 + else + abort %[#{url} doesn't include `{% assign release = site.data.releases | where: "version", "<version>" | first %}`] + end + puts "#{url} -> #{version}" + + + # Write release note article + path = Time.parse(src[/^date: (.*)/, 1]). + strftime("./#{src[/^lang: (\w+)/, 1]}/news/_posts/%Y-%m-%d-ruby-#{version.tr('.', '-')}-released.md") + puts path + File.write(path, src) +end diff --git a/tool/releng/update-www-meta.rb b/tool/releng/update-www-meta.rb new file mode 100755 index 0000000000..0dd5b25631 --- /dev/null +++ b/tool/releng/update-www-meta.rb @@ -0,0 +1,200 @@ +#!/usr/bin/env ruby +require "open-uri" +require "yaml" +require_relative "../ruby-version" + +class Tarball + attr_reader :version, :size, :sha1, :sha256, :sha512 + + def initialize(version, url, size, sha1, sha256, sha512) + @url = url + @size = size + @sha1 = sha1 + @sha256 = sha256 + @sha512 = sha512 + @version = version + @xy = version[/\A\d+\.\d+/] + end + + def gz?; @url.end_with?('.gz'); end + def zip?; @url.end_with?('.zip'); end + def xz?; @url.end_with?('.xz'); end + + def ext; @url[/(?:zip|tar\.(?:gz|xz))\z/]; end + + def to_md + <<eom +* <https://cache.ruby-lang.org/pub/ruby/#{@xy}/ruby-#{@version}.#{ext}> + + SIZE: #{@size} bytes + SHA1: #{@sha1} + SHA256: #{@sha256} + SHA512: #{@sha512} +eom + end + + # * /home/naruse/obj/ruby-trunk/tmp/ruby-2.6.0-preview3.tar.gz + # SIZE: 17116009 bytes + # SHA1: 21f62c369661a2ab1b521fd2fa8191a4273e12a1 + # SHA256: 97cea8aa63dfa250ba6902b658a7aa066daf817b22f82b7ee28f44aec7c2e394 + # SHA512: 1e2042324821bb4e110af7067f52891606dcfc71e640c194ab1c117f0b941550e0b3ac36ad3511214ac80c536b9e5cfaf8789eec74cf56971a832ea8fc4e6d94 + def self.parse(wwwdir, version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + teeny = Integer($3) + + uri = "https://cache.ruby-lang.org/pub/tmp/ruby-info-#{version}-draft.yml" + info = YAML.load(URI(uri).read) + if info.size != 1 + raise "unexpected info.yml '#{uri}'" + end + tarballs = [] + info[0]["size"].each_key do |ext| + url = info[0]["url"][ext] + size = info[0]["size"][ext] + sha1 = info[0]["sha1"][ext] + sha256 = info[0]["sha256"][ext] + sha512 = info[0]["sha512"][ext] + tarball = Tarball.new(version, url, size, sha1, sha256, sha512) + tarballs << tarball + end + + if teeny == 0 + # show diff shortstat + tag = RubyVersion.tag(version) + prev_tag = RubyVersion.tag(RubyVersion.previous(version)) + rubydir = File.expand_path(File.join(__FILE__, '../../../')) + puts %`git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + stat = `git -C #{rubydir} diff --shortstat #{prev_tag}..#{tag}` + files_changed, insertions, deletions = stat.scan(/\d+/) + end + + xy = version[/\A\d+\.\d+/] + #puts "## Download\n\n" + #tarballs.each do |tarball| + # puts tarball.to_md + #end + update_branches_yml(version, xy, wwwdir) + update_downloads_yml(version, xy, wwwdir) + update_releases_yml(version, xy, tarballs, wwwdir, files_changed, insertions, deletions) + end + + def self.update_branches_yml(ver, xy, wwwdir) + filename = "_data/branches.yml" + data = File.read(File.join(wwwdir, filename)) + if data.include?("\n- name: #{xy}\n") + data.sub!(/\n- name: #{Regexp.escape(xy)}\n(?: .*\n)*/) do |node| + unless ver.include?("-") + # assume this is X.Y.0 release + node.sub!(/^ status: preview\n/, " status: normal maintenance\n") + node.sub!(/^ date:\n/, " date: #{Time.now.year}-12-25\n") + end + node + end + else + if ver.include?("-") + status = "preview" + year = nil + else + status = "normal maintenance" + year = Time.now.year + end + entry = <<eom +- name: #{xy} + status: #{status} + date:#{ year && " #{year}-12-25" } + eol_date: + +eom + data.sub!(/(?=^- name)/, entry) + end + File.write(File.join(wwwdir, filename), data) + end + + def self.update_downloads_yml(ver, xy, wwwdir) + filename = "_data/downloads.yml" + data = File.read(File.join(wwwdir, filename)) + + if /^preview:\n\n(?: .*\n)* - #{Regexp.escape(xy)}\./ =~ data + if ver.include?("-") + data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + else + data.sub!(/^ - #{Regexp.escape(xy)}\..*\n/, "") + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + else + unless data.sub!(/^ - #{Regexp.escape(xy)}\..*/, " - #{ver}") + if ver.include?("-") + data.sub!(/(?<=^preview:\n\n)/, " - #{ver}\n") + else + data.sub!(/(?<=^stable:\n\n)/, " - #{ver}\n") + end + end + end + File.write(File.join(wwwdir, filename), data) + end + + def self.update_releases_yml(ver, xy, ary, wwwdir, files_changed, insertions, deletions) + filename = "_data/releases.yml" + data = File.read(File.join(wwwdir, filename)) + + date = Time.now.utc # use utc to use previous day in midnight + entry = <<eom +- version: #{ver} + tag: #{RubyVersion.tag(ver)} + date: #{date.strftime("%Y-%m-%d")} + post: /en/news/#{date.strftime("%Y/%m/%d")}/ruby-#{ver.tr('.', '-')}-released/ + stats: + files_changed: #{files_changed} + insertions: #{insertions} + deletions: #{deletions} + url: + gz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.gz + zip: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.zip + xz: https://cache.ruby-lang.org/pub/ruby/#{xy}/ruby-#{ver}.tar.xz + size: + gz: #{ary.find{|x|x.gz? }.size} + zip: #{ary.find{|x|x.zip?}.size} + xz: #{ary.find{|x|x.xz? }.size} + sha1: + gz: #{ary.find{|x|x.gz? }.sha1} + zip: #{ary.find{|x|x.zip?}.sha1} + xz: #{ary.find{|x|x.xz? }.sha1} + sha256: + gz: #{ary.find{|x|x.gz? }.sha256} + zip: #{ary.find{|x|x.zip?}.sha256} + xz: #{ary.find{|x|x.xz? }.sha256} + sha512: + gz: #{ary.find{|x|x.gz? }.sha512} + zip: #{ary.find{|x|x.zip?}.sha512} + xz: #{ary.find{|x|x.xz? }.sha512} +eom + + if data.include?("\n- version: #{ver}\n") + elsif data.sub!(/\n# #{Regexp.escape(xy)} series\n/, "\\&\n#{entry}") + else + data.sub!(/^$/, "\n# #{xy} series\n\n#{entry}") + end + File.write(File.join(wwwdir, filename), data) + end +end + +# Confirm current directory is www.ruby-lang.org's working directory +def confirm_w_r_l_o_wd + File.foreach('.git/config') do |line| + return true if line.include?('git@github.com:ruby/www.ruby-lang.org.git') + end + abort "Run this script in www.ruby-lang.org's working directory" +end + +def main + if ARGV.size != 1 + abort "usage: #$1 <version>" + end + confirm_w_r_l_o_wd + version = ARGV.shift + Tarball.parse(Dir.pwd, version) +end + +main diff --git a/tool/rmdirs b/tool/rmdirs index 6dcf984df6..76c4a39cb1 100755 --- a/tool/rmdirs +++ b/tool/rmdirs @@ -1,4 +1,7 @@ #!/bin/sh + +# Script used by configure to delete directories recursively. + for dir do while rmdir "$dir" >/dev/null 2>&1 && parent=`expr "$dir" : '\(.*\)/[^/][^/]*'`; do diff --git a/tool/ruby-version.rb b/tool/ruby-version.rb new file mode 100755 index 0000000000..3bbec576e1 --- /dev/null +++ b/tool/ruby-version.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby + +module RubyVersion + def self.tag(version) + major_version = Integer(version.split('.', 2)[0]) + if major_version >= 4 + "v#{version}" + else + "v#{version.tr('.-', '_')}" + end + end + + # Return the previous version to be used for release diff links. + # For a ".0" version, it returns the previous ".0" version. + # For a non-".0" version, it returns the previous teeny version. + def self.previous(version) + unless /\A(\d+)\.(\d+)\.(\d+)(?:-(?:preview|rc)\d+)?\z/ =~ version + raise "unexpected version string '#{version}'" + end + major = Integer($1) + minor = Integer($2) + teeny = Integer($3) + + if teeny != 0 + "#{major}.#{minor}.#{teeny-1}" + elsif minor != 0 # && teeny == 0 + "#{major}.#{minor-1}.#{teeny}" + else # minor == 0 && teeny == 0 + case major + when 3 + "2.7.0" + when 4 + "3.4.0" + else + raise "it doesn't know what is the previous version of '#{version}'" + end + end + end +end + +if __FILE__ == $0 + case ARGV[0] + when "tag" + print RubyVersion.tag(ARGV[1]) + when "previous" + print RubyVersion.previous(ARGV[1]) + when "previous-tag" + print RubyVersion.tag(RubyVersion.previous(ARGV[1])) + else + "#{$0}: unexpected command #{ARGV[0].inspect}" + end +end diff --git a/tool/ruby_vm/controllers/application_controller.rb b/tool/ruby_vm/controllers/application_controller.rb new file mode 100644 index 0000000000..f6c0e39600 --- /dev/null +++ b/tool/ruby_vm/controllers/application_controller.rb @@ -0,0 +1,25 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/dumper' +require_relative '../models/instructions' +require_relative '../models/typemap' +require_relative '../loaders/vm_opts_h' + +class ApplicationController + def generate i, destdir, basedir + path = Pathname.new i + dst = destdir ? Pathname.new(destdir).join(i) : Pathname.new(i) + base = basedir ? Pathname.new(basedir) : Pathname.pwd + dumper = RubyVM::Dumper.new dst, base.expand_path + return [path, dumper] + end +end diff --git a/tool/ruby_vm/helpers/c_escape.rb b/tool/ruby_vm/helpers/c_escape.rb new file mode 100644 index 0000000000..628cb0428b --- /dev/null +++ b/tool/ruby_vm/helpers/c_escape.rb @@ -0,0 +1,130 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require 'securerandom' + +module RubyVM::CEscape + module_function + + # generate comment, with escaps. + def commentify str + unless str = str.dump[/\A"\K.*(?="\z)/] + raise Encoding::CompatibilityError, "must be ASCII-compatible (#{str.encoding})" + end + return "/* #{str.gsub('*/', '*\\/').gsub('/*', '/\\*')} */" + end + + # Mimic gensym of CL. + def gensym prefix = 'gensym_' + return as_tr_cpp "#{prefix}#{SecureRandom.uuid}" + end + + # Mimic AS_TR_CPP() of autoconf. + def as_tr_cpp name + q = name.b + q.gsub! %r/[^a-zA-Z0-9_]/m, '_' + q.gsub! %r/_+/, '_' + return q + end + + # Section 6.10.4 of ISO/IEC 9899:1999 specifies that the file name used for + # #line directive shall be a "character string literal". So this is needed. + # + # I'm not sure how many chars are allowed here, though. The standard + # specifies 4095 chars at most, after string concatenation (section 5.2.4.1). + # But it is easy to have a path that is longer than that. + # + # Here we ignore the standard. Just create single string literal of any + # needed length. + def rstring2cstr str + # I believe this is the fastest implementation done in pure-ruby. + # Constants cached, gsub skips block evaluation, string literal optimized. + buf = str.b + buf.gsub! %r/./nm, RString2CStr + return %'"#{buf}"' + end + + RString2CStr = { + "\x00"=> "\\0", "\x01"=> "\\x1", "\x02"=> "\\x2", "\x03"=> "\\x3", + "\x04"=> "\\x4", "\x05"=> "\\x5", "\x06"=> "\\x6", "\a"=> "\\a", + "\b"=> "\\b", "\t"=> "\\t", "\n"=> "\\n", "\v"=> "\\v", + "\f"=> "\\f", "\r"=> "\\r", "\x0E"=> "\\xe", "\x0F"=> "\\xf", + "\x10"=>"\\x10", "\x11"=>"\\x11", "\x12"=>"\\x12", "\x13"=>"\\x13", + "\x14"=>"\\x14", "\x15"=>"\\x15", "\x16"=>"\\x16", "\x17"=>"\\x17", + "\x18"=>"\\x18", "\x19"=>"\\x19", "\x1A"=>"\\x1a", "\e"=>"\\x1b", + "\x1C"=>"\\x1c", "\x1D"=>"\\x1d", "\x1E"=>"\\x1e", "\x1F"=>"\\x1f", + " "=> " ", "!"=> "!", "\""=> "\\\"", "#"=> "#", + "$"=> "$", "%"=> "%", "&"=> "&", "'"=> "'", + "("=> "(", ")"=> ")", "*"=> "*", "+"=> "+", + ","=> ",", "-"=> "-", "."=> ".", "/"=> "/", + "0"=> "0", "1"=> "1", "2"=> "2", "3"=> "3", + "4"=> "4", "5"=> "5", "6"=> "6", "7"=> "7", + "8"=> "8", "9"=> "9", ":"=> ":", ";"=> ";", + "<"=> "<", "="=> "=", ">"=> ">", "?"=> "?", + "@"=> "@", "A"=> "A", "B"=> "B", "C"=> "C", + "D"=> "D", "E"=> "E", "F"=> "F", "G"=> "G", + "H"=> "H", "I"=> "I", "J"=> "J", "K"=> "K", + "L"=> "L", "M"=> "M", "N"=> "N", "O"=> "O", + "P"=> "P", "Q"=> "Q", "R"=> "R", "S"=> "S", + "T"=> "T", "U"=> "U", "V"=> "V", "W"=> "W", + "X"=> "X", "Y"=> "Y", "Z"=> "Z", "["=> "[", + "\\"=> "\\\\", "]"=> "]", "^"=> "^", "_"=> "_", + "`"=> "`", "a"=> "a", "b"=> "b", "c"=> "c", + "d"=> "d", "e"=> "e", "f"=> "f", "g"=> "g", + "h"=> "h", "i"=> "i", "j"=> "j", "k"=> "k", + "l"=> "l", "m"=> "m", "n"=> "n", "o"=> "o", + "p"=> "p", "q"=> "q", "r"=> "r", "s"=> "s", + "t"=> "t", "u"=> "u", "v"=> "v", "w"=> "w", + "x"=> "x", "y"=> "y", "z"=> "z", "{"=> "{", + "|"=> "|", "}"=> "}", "~"=> "~", "\x7F"=>"\\x7f", + "\x80"=>"\\x80", "\x81"=>"\\x81", "\x82"=>"\\x82", "\x83"=>"\\x83", + "\x84"=>"\\x84", "\x85"=>"\\x85", "\x86"=>"\\x86", "\x87"=>"\\x87", + "\x88"=>"\\x88", "\x89"=>"\\x89", "\x8A"=>"\\x8a", "\x8B"=>"\\x8b", + "\x8C"=>"\\x8c", "\x8D"=>"\\x8d", "\x8E"=>"\\x8e", "\x8F"=>"\\x8f", + "\x90"=>"\\x90", "\x91"=>"\\x91", "\x92"=>"\\x92", "\x93"=>"\\x93", + "\x94"=>"\\x94", "\x95"=>"\\x95", "\x96"=>"\\x96", "\x97"=>"\\x97", + "\x98"=>"\\x98", "\x99"=>"\\x99", "\x9A"=>"\\x9a", "\x9B"=>"\\x9b", + "\x9C"=>"\\x9c", "\x9D"=>"\\x9d", "\x9E"=>"\\x9e", "\x9F"=>"\\x9f", + "\xA0"=>"\\xa0", "\xA1"=>"\\xa1", "\xA2"=>"\\xa2", "\xA3"=>"\\xa3", + "\xA4"=>"\\xa4", "\xA5"=>"\\xa5", "\xA6"=>"\\xa6", "\xA7"=>"\\xa7", + "\xA8"=>"\\xa8", "\xA9"=>"\\xa9", "\xAA"=>"\\xaa", "\xAB"=>"\\xab", + "\xAC"=>"\\xac", "\xAD"=>"\\xad", "\xAE"=>"\\xae", "\xAF"=>"\\xaf", + "\xB0"=>"\\xb0", "\xB1"=>"\\xb1", "\xB2"=>"\\xb2", "\xB3"=>"\\xb3", + "\xB4"=>"\\xb4", "\xB5"=>"\\xb5", "\xB6"=>"\\xb6", "\xB7"=>"\\xb7", + "\xB8"=>"\\xb8", "\xB9"=>"\\xb9", "\xBA"=>"\\xba", "\xBB"=>"\\xbb", + "\xBC"=>"\\xbc", "\xBD"=>"\\xbd", "\xBE"=>"\\xbe", "\xBF"=>"\\xbf", + "\xC0"=>"\\xc0", "\xC1"=>"\\xc1", "\xC2"=>"\\xc2", "\xC3"=>"\\xc3", + "\xC4"=>"\\xc4", "\xC5"=>"\\xc5", "\xC6"=>"\\xc6", "\xC7"=>"\\xc7", + "\xC8"=>"\\xc8", "\xC9"=>"\\xc9", "\xCA"=>"\\xca", "\xCB"=>"\\xcb", + "\xCC"=>"\\xcc", "\xCD"=>"\\xcd", "\xCE"=>"\\xce", "\xCF"=>"\\xcf", + "\xD0"=>"\\xd0", "\xD1"=>"\\xd1", "\xD2"=>"\\xd2", "\xD3"=>"\\xd3", + "\xD4"=>"\\xd4", "\xD5"=>"\\xd5", "\xD6"=>"\\xd6", "\xD7"=>"\\xd7", + "\xD8"=>"\\xd8", "\xD9"=>"\\xd9", "\xDA"=>"\\xda", "\xDB"=>"\\xdb", + "\xDC"=>"\\xdc", "\xDD"=>"\\xdd", "\xDE"=>"\\xde", "\xDF"=>"\\xdf", + "\xE0"=>"\\xe0", "\xE1"=>"\\xe1", "\xE2"=>"\\xe2", "\xE3"=>"\\xe3", + "\xE4"=>"\\xe4", "\xE5"=>"\\xe5", "\xE6"=>"\\xe6", "\xE7"=>"\\xe7", + "\xE8"=>"\\xe8", "\xE9"=>"\\xe9", "\xEA"=>"\\xea", "\xEB"=>"\\xeb", + "\xEC"=>"\\xec", "\xED"=>"\\xed", "\xEE"=>"\\xee", "\xEF"=>"\\xef", + "\xF0"=>"\\xf0", "\xF1"=>"\\xf1", "\xF2"=>"\\xf2", "\xF3"=>"\\xf3", + "\xF4"=>"\\xf4", "\xF5"=>"\\xf5", "\xF6"=>"\\xf6", "\xF7"=>"\\xf7", + "\xF8"=>"\\xf8", "\xF9"=>"\\xf9", "\xFA"=>"\\xfa", "\xFB"=>"\\xfb", + "\xFC"=>"\\xfc", "\xFD"=>"\\xfd", "\xFE"=>"\\xfe", "\xFF"=>"\\xff", + }.freeze + private_constant :RString2CStr +end + +unless defined? ''.b + class String + def b + return dup.force_encoding 'binary' + end + end +end diff --git a/tool/ruby_vm/helpers/dumper.rb b/tool/ruby_vm/helpers/dumper.rb new file mode 100644 index 0000000000..f0758dd44f --- /dev/null +++ b/tool/ruby_vm/helpers/dumper.rb @@ -0,0 +1,109 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require 'pathname' +require 'erb' +require_relative 'c_escape' + +class RubyVM::Dumper + include RubyVM::CEscape + private + + def new_binding + # This `eval 'binding'` does not return the current binding + # but creates one on top of it. + return eval 'binding' + end + + def new_erb spec + path = Pathname.new(__FILE__) + path = (path.relative_path_from(Pathname.pwd) rescue path).dirname + path += '../views' + path += Pathname.pwd.join(spec).expand_path.to_s.sub("#{@base}/", '') + src = path.expand_path.read mode: 'rt:utf-8:utf-8' + rescue Errno::ENOENT + raise "don't know how to generate #{path}" + else + erb = ERB.new(src, trim_mode: '%-') + erb.filename = path.to_path + return erb + end + + def finderb spec + return @erb.fetch spec do |k| + erb = new_erb k + @erb[k] = erb + end + end + + def replace_pragma_line str, lineno + if /#(\s*)pragma RubyVM reset source\n/ =~ str then + return "##{$1}line #{lineno + 2} #{@file}\n" + else + return str + end + end + + def local_variable_set bnd, var, val + eval '__locals__ ||= {}', bnd + locals = eval '__locals__', bnd + locals[var] = val + eval "#{var} = __locals__[:#{var}]", bnd + test = eval "#{var}", bnd + raise unless test == val + end + + public + + def do_render source, locals + erb = finderb source + bnd = @empty.dup + locals.each_pair do |k, v| + local_variable_set bnd, k, v + end + return erb.result bnd + end + + def replace_pragma str + return str \ + . each_line \ + . with_index \ + . map {|i, j| replace_pragma_line i, j } \ + . join + end + + def initialize dst, base + @erb = {} + @empty = new_binding + @file = cstr dst.to_path + @base = base + end + + def render partial, opts = { :locals => {} } + return do_render "_#{partial}.erb", opts[:locals] + end + + def generate template + str = do_render "#{template}.erb", {} + return replace_pragma str + end + + private + + # view helpers + + alias cstr rstring2cstr + alias comm commentify + + def render_c_expr expr + render 'c_expr', locals: { expr: expr, } + end +end diff --git a/tool/ruby_vm/helpers/scanner.rb b/tool/ruby_vm/helpers/scanner.rb new file mode 100644 index 0000000000..8998abb2d3 --- /dev/null +++ b/tool/ruby_vm/helpers/scanner.rb @@ -0,0 +1,52 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require 'pathname' + +# Poor man's StringScanner. +# Sadly https://bugs.ruby-lang.org/issues/8343 is not backported to 2.0. We +# have to do it by hand. +class RubyVM::Scanner + attr_reader :__FILE__ + attr_reader :__LINE__ + + def initialize path + src = Pathname.new(__FILE__) + src = (src.relative_path_from(Pathname.pwd) rescue src).dirname + src += path + @__LINE__ = 1 + @__FILE__ = src.to_path + @str = src.read mode: 'rt:utf-8:utf-8' + @pos = 0 + end + + def eos? + return @pos >= @str.length + end + + def scan re + ret = @__LINE__ + @last_match = @str.match re, @pos + return unless @last_match + @__LINE__ += @last_match.to_s.count "\n" + @pos = @last_match.end 0 + return ret + end + + def scan! re + scan re or raise sprintf "parse error at %s:%d near:\n %s...", \ + @__FILE__, @__LINE__, @str[@pos, 32] + end + + def [] key + return @last_match[key] + end +end diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb new file mode 100644 index 0000000000..d45d0ba83c --- /dev/null +++ b/tool/ruby_vm/loaders/insns_def.rb @@ -0,0 +1,99 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/scanner' +require_relative './vm_opts_h' + +json = [] +scanner = RubyVM::Scanner.new '../../../insns.def' +path = scanner.__FILE__ +grammar = %r' + (?<comment> /[*] [^*]* [*]+ (?: [^*/] [^*]* [*]+ )* / ){0} + (?<keyword> typedef | extern | static | auto | register | + struct | union | enum ){0} + (?<C> (?: \g<block> | [^{}]+ )* ){0} + (?<block> \{ \g<ws>* \g<C> \g<ws>* \} ){0} + (?<ws> \g<comment> | \s ){0} + (?<ident> [_a-zA-Z] [0-9_a-zA-Z]* ){0} + (?<type> (?: \g<keyword> \g<ws>+ )* \g<ident> ){0} + (?<arg> \g<type> \g<ws>+ \g<ident> ){0} + (?<argv> (?# empty ) | + void | + (?: \.\.\. | \g<arg>) (?: \g<ws>* , \g<ws>* \g<arg> \g<ws>* )* ){0} + (?<pragma> \g<ws>* // \s* attr \g<ws>+ + (?<pragma:type> \g<type> ) \g<ws>+ + (?<pragma:name> \g<ident> ) \g<ws>* + = \g<ws>* + (?<pragma:expr> .+?; ) \g<ws>* ){0} + (?<insn> DEFINE_INSN(_IF\((?<insn:if>\w+)\))? \g<ws>+ + (?<insn:name> \g<ident> ) \g<ws>* + [(] \g<ws>* (?<insn:opes> \g<argv> ) \g<ws>* [)] \g<ws>* + [(] \g<ws>* (?<insn:pops> \g<argv> ) \g<ws>* [)] \g<ws>* + [(] \g<ws>* (?<insn:rets> \g<argv> ) \g<ws>* [)] \g<ws>* ){0} +'x + +until scanner.eos? do + next if scanner.scan(/\G#{grammar}\g<ws>+/o) + split = lambda {|v| + case v when /\Avoid\z/ then + [] + else + v.split(/, */) + end + } + + l1 = scanner.scan!(/\G#{grammar}\g<insn>/o) + name = scanner["insn:name"] + opt = scanner["insn:if"] + ope = split.(scanner["insn:opes"]) + pop = split.(scanner["insn:pops"]) + ret = split.(scanner["insn:rets"]) + if ope.include?("...") + raise sprintf("parse error at %s:%d:%s: operands cannot be variadic", + scanner.__FILE__, scanner.__LINE__, name) + end + + attrs = [] + while l2 = scanner.scan(/\G#{grammar}\g<pragma>/o) do + attrs << { + location: [path, l2], + name: scanner["pragma:name"], + type: scanner["pragma:type"], + expr: scanner["pragma:expr"], + } + end + + l3 = scanner.scan!(/\G#{grammar}\g<block>/o) + if opt.nil? || RubyVM::VmOptsH[opt] + json << { + name: name, + location: [path, l1], + signature: { + name: name, + ope: ope, + pop: pop, + ret: ret, + }, + attributes: attrs, + expr: { + location: [path, l3], + expr: scanner["block"], + }, + } + end +end + +RubyVM::InsnsDef = json + +if __FILE__ == $0 then + require 'json' + JSON.dump RubyVM::InsnsDef, STDOUT +end diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb new file mode 100644 index 0000000000..0750f1823a --- /dev/null +++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb @@ -0,0 +1,33 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/scanner' + +json = [] +scanner = RubyVM::Scanner.new '../../../defs/opt_insn_unif.def' +path = scanner.__FILE__ +until scanner.eos? do + next if scanner.scan(/\G ^ (?: \#.* )? \n /x) + break if scanner.scan(/\G ^ __END__ $ /x) + + pos = scanner.scan!(/\G (?<series> (?: [\ \t]* \w+ )+ ) \n /mx) + json << { + location: [path, pos], + signature: scanner["series"].strip.split + } +end + +RubyVM::OptInsnUnifDef = json + +if __FILE__ == $0 then + require 'json' + JSON.dump RubyVM::OptInsnUnifDef, STDOUT +end diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb new file mode 100644 index 0000000000..e08509a433 --- /dev/null +++ b/tool/ruby_vm/loaders/opt_operand_def.rb @@ -0,0 +1,55 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/scanner' + +json = [] +scanner = RubyVM::Scanner.new '../../../defs/opt_operand.def' +path = scanner.__FILE__ +grammar = %r/ + (?<comment> \# .+? \n ){0} + (?<ws> \g<comment> | \s ){0} + (?<insn> \w+ ){0} + (?<paren> \( (?: \g<paren> | [^()]+)* \) ){0} + (?<expr> (?: \g<paren> | [^(),\ \n] )+ ){0} + (?<remain> \g<expr> ){0} + (?<arg> \g<expr> ){0} + (?<extra> , \g<ws>* \g<remain> ){0} + (?<args> \g<arg> \g<extra>* ){0} + (?<decl> \g<insn> \g<ws>+ \g<args> \n ){0} +/mx + +until scanner.eos? do + break if scanner.scan(/\G ^ __END__ $ /x) + next if scanner.scan(/\G#{grammar} \g<ws>+ /ox) + + line = scanner.scan!(/\G#{grammar} \g<decl> /mox) + insn = scanner["insn"] + args = scanner["args"] + ary = [] + until args.strip.empty? do + md = /\G#{grammar} \g<args> /mox.match(args) + ary << md["arg"] + args = md["remain"] + break unless args + end + json << { + location: [path, line], + signature: [insn, ary] + } +end + +RubyVM::OptOperandDef = json + +if __FILE__ == $0 then + require 'json' + JSON.dump RubyVM::OptOperandDef, STDOUT +end diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb new file mode 100644 index 0000000000..d626ea0296 --- /dev/null +++ b/tool/ruby_vm/loaders/vm_opts_h.rb @@ -0,0 +1,36 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/scanner' + +json = {} +scanner = RubyVM::Scanner.new '../../../vm_opts.h' +grammar = %r/ + (?<ws> \u0020 ){0} + (?<key> \w+ ){0} + (?<value> 0|1 ){0} + (?<define> \G \#define \g<ws>+ OPT_\g<key> \g<ws>+ \g<value> \g<ws>*\n ) +/mx + +until scanner.eos? do + if scanner.scan grammar then + json[scanner['key']] = ! scanner['value'].to_i.zero? # not nonzero? + else + scanner.scan(/\G.*\n/) + end +end + +RubyVM::VmOptsH = json + +if __FILE__ == $0 then + require 'json' + JSON.dump RubyVM::VmOptsH, STDOUT +end diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb new file mode 100644 index 0000000000..177b701b92 --- /dev/null +++ b/tool/ruby_vm/models/attribute.rb @@ -0,0 +1,58 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative 'c_expr' + +class RubyVM::Attribute + include RubyVM::CEscape + attr_reader :insn, :key, :type, :expr + + def initialize opts = {} + @insn = opts[:insn] + @key = opts[:name] + @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] + @type = opts[:type] + @ope_decls = @insn.operands.map do |operand| + decl = operand[:decl] + if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA' + decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci') + end + decl + end + end + + def name + as_tr_cpp "attr #{@key} @ #{@insn.name}" + end + + def pretty_name + "attr #{type} #{key} @ #{insn.pretty_name}" + end + + def declaration + if @ope_decls.empty? + argv = "void" + else + argv = @ope_decls.join(', ') + end + sprintf '%s %s(%s)', @type, name, argv + end + + def definition + if @ope_decls.empty? + argv = "void" + else + argv = @ope_decls.map {|decl| "MAYBE_UNUSED(#{decl})" }.join(",\n ") + argv = "\n #{argv}\n" if @ope_decls.size > 1 + end + sprintf "%s\n%s(%s)", @type, name, argv + end +end diff --git a/tool/ruby_vm/models/bare_instruction.rb b/tool/ruby_vm/models/bare_instruction.rb new file mode 100644 index 0000000000..f87dd74179 --- /dev/null +++ b/tool/ruby_vm/models/bare_instruction.rb @@ -0,0 +1,236 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../loaders/insns_def' +require_relative 'c_expr' +require_relative 'typemap' +require_relative 'attribute' + +class RubyVM::BareInstruction + attr_reader :template, :name, :operands, :pops, :rets, :decls, :expr + + def initialize opts = {} + @template = opts[:template] + @name = opts[:name] + @loc = opts[:location] + @sig = opts[:signature] + @expr = RubyVM::CExpr.new opts[:expr] + @operands = typesplit @sig[:ope] + @pops = typesplit @sig[:pop].reject {|i| i == '...' } + @rets = typesplit @sig[:ret].reject {|i| i == '...' } + @attrs = opts[:attributes].map {|i| + RubyVM::Attribute.new i.merge(:insn => self) + }.each_with_object({}) {|a, h| + h[a.key] = a + } + @attrs_orig = @attrs.dup + check_attribute_consistency + predefine_attributes + end + + def pretty_name + n = @sig[:name] + o = @sig[:ope].map{|i| i[/\S+$/] }.join ', ' + p = @sig[:pop].map{|i| i[/\S+$/] }.join ', ' + r = @sig[:ret].map{|i| i[/\S+$/] }.join ', ' + return sprintf "%s(%s)(%s)(%s)", n, o, p, r + end + + def bin + return "BIN(#{name})" + end + + def call_attribute name + return sprintf 'attr_%s_%s(%s)', name, @name, \ + @operands.map {|i| i[:name] }.compact.join(', ') + end + + def has_attribute? k + @attrs_orig.has_key? k + end + + def attributes + return @attrs \ + . sort_by {|k, _| k } \ + . map {|_, v| v } + end + + def width + return 1 + operands.size + end + + def declarations + return @variables \ + . values \ + . group_by {|h| h[:type] } \ + . sort_by {|t, v| t } \ + . map {|t, v| [t, v.map {|i| i[:name] }.sort ] } \ + . map {|t, v| + sprintf("MAYBE_UNUSED(%s) %s", t, v.join(', ')) + } + end + + def preamble + # preamble makes sense for operand unifications + return [] + end + + def sc? + # sc stands for stack caching. + return false + end + + def cast_to_VALUE var, expr = var[:name] + RubyVM::Typemap.typecast_to_VALUE var[:type], expr + end + + def cast_from_VALUE var, expr = var[:name] + RubyVM::Typemap.typecast_from_VALUE var[:type], expr + end + + def operands_info + operands.map {|o| + c, _ = RubyVM::Typemap.fetch o[:type] + next c + }.join + end + + def handles_sp? + /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr + end + + def handle_canary stmt + # Stack canary is basically a good thing that we want to add, however: + # + # - When the instruction returns variadic number of return values, + # it is not easy to tell where is the stack top. We can't but + # skip it. + # + # - When the instruction body is empty (like putobject), we can + # say for 100% sure that canary is a waste of time. + # + # So we skip canary for those cases. + return '' if @sig[:ret].any? {|i| i == '...' } + return '' if @expr.blank? + return " #{stmt};\n" + end + + def inspect + sprintf "#<%s %s@%s:%d>", self.class.name, @name, @loc[0], @loc[1] + end + + def has_ope? var + return @operands.any? {|i| i[:name] == var[:name] } + end + + def has_pop? var + return @pops.any? {|i| i[:name] == var[:name] } + end + + def use_call_data? + @use_call_data ||= + @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } + end + + def zjit_profile? + @attrs.fetch('zjit_profile').expr.expr != 'false;' + end + + private + + def check_attribute_consistency + if has_attribute?('sp_inc') \ + && use_call_data? \ + && !has_attribute?('comptime_sp_inc') + # As the call cache caches information that can only be obtained at + # runtime, we do not need it when compiling from AST to bytecode. This + # attribute defines an expression that computes the stack pointer + # increase based on just the call info to avoid reserving space for the + # call cache at compile time. In the expression, all call data operands + # are mapped to their call info counterpart. Additionally, all mentions + # of `cd` in the operand name are replaced with `ci`. + raise "Please define attribute `comptime_sp_inc` for `#{@name}`" + end + end + + def generate_attribute t, k, v + @attrs[k] ||= RubyVM::Attribute.new \ + insn: self, \ + name: k, \ + type: t, \ + location: [], \ + expr: v.to_s + ';' + return @attrs[k] ||= attr + end + + def predefine_attributes + # Beware: order matters here because some attribute depends another. + generate_attribute 'const char*', 'name', "insn_name(#{bin})" + generate_attribute 'enum ruby_vminsn_type', 'bin', bin + generate_attribute 'rb_num_t', 'open', operands.size + generate_attribute 'rb_num_t', 'popn', pops.size + generate_attribute 'rb_num_t', 'retn', rets.size + generate_attribute 'rb_num_t', 'width', width + generate_attribute 'rb_snum_t', 'sp_inc', rets.size - pops.size + generate_attribute 'bool', 'handles_sp', default_definition_of_handles_sp + generate_attribute 'bool', 'leaf', default_definition_of_leaf + generate_attribute 'bool', 'zjit_profile', false + end + + def default_definition_of_handles_sp + # Insn with ISEQ should yield it; can handle sp. + return operands.any? {|o| o[:type] == 'ISEQ' } + end + + def default_definition_of_leaf + # Insn that handles SP can never be a leaf. + if not has_attribute? 'handles_sp' then + return ! default_definition_of_handles_sp + elsif handles_sp? then + return "! #{call_attribute 'handles_sp'}" + else + return true + end + end + + def typesplit a + @variables ||= {} + a.map do |decl| + md = %r' + (?<comment> /[*] [^*]* [*]+ (?: [^*/] [^*]* [*]+ )* / ){0} + (?<ws> \g<comment> | \s ){0} + (?<ident> [_a-zA-Z] [0-9_a-zA-Z]* ){0} + (?<type> (?: \g<ident> \g<ws>+ )* \g<ident> ){0} + (?<var> \g<ident> ){0} + \G \g<ws>* \g<type> \g<ws>+ \g<var> + 'x.match(decl) + @variables[md['var']] ||= { + decl: decl, + type: md['type'], + name: md['var'], + } + end + end + + @instances = RubyVM::InsnsDef.map {|h| + new h.merge(:template => h) + } + + def self.find(name) + @instances.find do |insn| + insn.name == name + end or raise IndexError, "instruction not found: #{name}" + end + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/models/c_expr.rb b/tool/ruby_vm/models/c_expr.rb new file mode 100644 index 0000000000..095ff4f1d9 --- /dev/null +++ b/tool/ruby_vm/models/c_expr.rb @@ -0,0 +1,44 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/c_escape.rb' + +class RubyVM::CExpr + include RubyVM::CEscape + + attr_reader :__FILE__, :__LINE__, :expr + + def initialize opts = {} + @__FILE__ = opts[:location][0] + @__LINE__ = opts[:location][1] + @expr = opts[:expr] + end + + # blank, in sense of C program. + RE = %r'\A{\g<s>*}\z|\A(?<s>\s|/[*][^*]*[*]+([^*/][^*]*[*]+)*/)*\z' + if RUBY_VERSION > '2.4' then + def blank? + RE.match? @expr + end + else + def blank? + RE =~ @expr + end + end + + def inspect + if @__LINE__ + sprintf "#<%s:%d %s>", @__FILE__, @__LINE__, @expr + else + sprintf "#<%s %s>", @__FILE__, @expr + end + end +end diff --git a/tool/ruby_vm/models/instructions.rb b/tool/ruby_vm/models/instructions.rb new file mode 100644 index 0000000000..7be7064b14 --- /dev/null +++ b/tool/ruby_vm/models/instructions.rb @@ -0,0 +1,23 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative 'bare_instruction' +require_relative 'operands_unification' +require_relative 'instructions_unification' +require_relative 'trace_instruction' +require_relative 'zjit_instruction' + +RubyVM::Instructions = RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all + + RubyVM::TraceInstruction.all + + RubyVM::ZJITInstruction.all +RubyVM::Instructions.freeze diff --git a/tool/ruby_vm/models/instructions_unification.rb b/tool/ruby_vm/models/instructions_unification.rb new file mode 100644 index 0000000000..5c798e6d54 --- /dev/null +++ b/tool/ruby_vm/models/instructions_unification.rb @@ -0,0 +1,42 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/c_escape' +require_relative '../loaders/opt_insn_unif_def' +require_relative 'bare_instruction' + +class RubyVM::InstructionsUnification + include RubyVM::CEscape + + attr_reader :name + + def initialize opts = {} + @location = opts[:location] + @name = namegen opts[:signature] + @series = opts[:signature].map do |i| + RubyVM::BareInstruction.find(i) # Misshit is fatal + end + end + + private + + def namegen signature + as_tr_cpp ['UNIFIED', *signature].join('_') + end + + @instances = RubyVM::OptInsnUnifDef.map do |h| + new h + end + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/models/operands_unification.rb b/tool/ruby_vm/models/operands_unification.rb new file mode 100644 index 0000000000..ce118648ca --- /dev/null +++ b/tool/ruby_vm/models/operands_unification.rb @@ -0,0 +1,141 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/c_escape' +require_relative '../loaders/opt_operand_def' +require_relative 'bare_instruction' + +class RubyVM::OperandsUnification < RubyVM::BareInstruction + include RubyVM::CEscape + + attr_reader :preamble, :original, :spec + + def initialize opts = {} + name = opts[:signature][0] + @original = RubyVM::BareInstruction.find(name) + template = @original.template + parts = compose opts[:location], opts[:signature], template[:signature] + json = template.dup + json[:location] = opts[:location] + json[:signature] = parts[:signature] + json[:name] = parts[:name] + @preamble = parts[:preamble] + @spec = parts[:spec] + super json.merge(:template => template) + @konsts = parts[:vars] + @konsts.each do |v| + @variables[v[:name]] ||= v + end + end + + def operand_shift_of var + before = @original.operands.find_index var + after = @operands.find_index var + raise "no #{var} for #{@name}" unless before and after + return before - after + end + + def condition ptr + # :FIXME: I'm not sure if this method should be in model? + exprs = @spec.each_with_index.map do |(var, val), i| + case val when '*' then + next nil + else + type = @original.operands[i][:type] + expr = RubyVM::Typemap.typecast_to_VALUE type, val + next "#{ptr}[#{i}] == #{expr}" + end + end + exprs.compact! + if exprs.size == 1 then + return exprs[0] + else + exprs.map! {|i| "(#{i})" } + return exprs.join ' && ' + end + end + + def has_ope? var + super or @konsts.any? {|i| i[:name] == var[:name] } + end + + private + + def namegen signature + insn, argv = *signature + wcary = argv.map do |i| + case i when '*' then + 'WC' + else + i + end + end + as_tr_cpp [insn, *wcary].join(', ') + end + + def compose location, spec, template + name = namegen spec + *, argv = *spec + opes = @original.operands + if opes.size != argv.size + raise sprintf("operand size mismatch for %s (%s's: %d, given: %d)", + name, template[:name], opes.size, argv.size) + else + src = [] + mod = [] + spec = [] + vars = [] + argv.each_index do |i| + j = argv[i] + k = opes[i] + spec[i] = [k, j] + case j when '*' then + # operand is from iseq + mod << k[:decl] + else + # operand is inside C + vars << k + src << { + location: location, + expr: " const #{k[:decl]} = #{j};" + } + end + end + src.map! {|i| RubyVM::CExpr.new i } + return { + name: name, + signature: { + name: name, + ope: mod, + pop: template[:pop], + ret: template[:ret], + }, + preamble: src, + vars: vars, + spec: spec + } + end + end + + @instances = RubyVM::OptOperandDef.map do |h| + new h + end + + def self.all + @instances + end + + def self.each_group + all.group_by(&:original).each_pair do |k, v| + yield k, v + end + end +end diff --git a/tool/ruby_vm/models/trace_instruction.rb b/tool/ruby_vm/models/trace_instruction.rb new file mode 100644 index 0000000000..6a3ad53c44 --- /dev/null +++ b/tool/ruby_vm/models/trace_instruction.rb @@ -0,0 +1,70 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require_relative '../helpers/c_escape' +require_relative 'bare_instruction' + +class RubyVM::TraceInstruction + include RubyVM::CEscape + + attr_reader :name + + def initialize orig + @orig = orig + @name = as_tr_cpp "trace @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute? *; + return false + end + + private + + @instances = (RubyVM::BareInstruction.all + + RubyVM::OperandsUnification.all + + RubyVM::InstructionsUnification.all).map {|i| new(i) } + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb new file mode 100644 index 0000000000..68ef5a41a5 --- /dev/null +++ b/tool/ruby_vm/models/typemap.rb @@ -0,0 +1,62 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +RubyVM::Typemap = { + "..." => %w[. TS_VARIABLE], + "CALL_DATA" => %w[C TS_CALLDATA], + "CDHASH" => %w[H TS_CDHASH], + "IC" => %w[K TS_IC], + "IVC" => %w[A TS_IVC], + "ICVARC" => %w[J TS_ICVARC], + "ID" => %w[I TS_ID], + "ISE" => %w[T TS_ISE], + "ISEQ" => %w[S TS_ISEQ], + "OFFSET" => %w[O TS_OFFSET], + "VALUE" => %w[V TS_VALUE], + "lindex_t" => %w[L TS_LINDEX], + "rb_insn_func_t" => %w[F TS_FUNCPTR], + "rb_num_t" => %w[N TS_NUM], + "RB_BUILTIN" => %w[R TS_BUILTIN], +} + +# :FIXME: should this method be here? +class << RubyVM::Typemap + def typecast_from_VALUE type, val + # see also iseq_set_sequence() + case type + when '...' + raise "cast not possible: #{val}" + when 'VALUE' then + return val + when 'rb_num_t', 'lindex_t' then + return "NUM2LONG(#{val})" + when 'ID' then + return "SYM2ID(#{val})" + else + return "(#{type})(#{val})" + end + end + + def typecast_to_VALUE type, val + case type + when 'VALUE' then + return val + when 'ISEQ', 'rb_insn_func_t' then + return "(VALUE)(#{val})" + when 'rb_num_t', 'lindex_t' + "LONG2NUM(#{val})" + when 'ID' then + return "ID2SYM(#{val})" + else + raise ":FIXME: TBW for #{type}" + end + end +end diff --git a/tool/ruby_vm/models/zjit_instruction.rb b/tool/ruby_vm/models/zjit_instruction.rb new file mode 100644 index 0000000000..04764e4c61 --- /dev/null +++ b/tool/ruby_vm/models/zjit_instruction.rb @@ -0,0 +1,56 @@ +require_relative '../helpers/c_escape' +require_relative 'bare_instruction' + +# Profile YARV instructions to optimize code generated by ZJIT +class RubyVM::ZJITInstruction + include RubyVM::CEscape + + attr_reader :name + + def initialize(orig) + @orig = orig + @name = as_tr_cpp "zjit @ #{@orig.name}" + end + + def pretty_name + return sprintf "%s(...)(...)(...)", @name + end + + def jump_destination + return @orig.name + end + + def bin + return sprintf "BIN(%s)", @name + end + + def width + return @orig.width + end + + def operands_info + return @orig.operands_info + end + + def rets + return ['...'] + end + + def pops + return ['...'] + end + + def attributes + return [] + end + + def has_attribute?(*) + return false + end + + @instances = RubyVM::BareInstruction.all.filter(&:zjit_profile?).map {|i| new(i) } + + def self.all + @instances + end +end diff --git a/tool/ruby_vm/scripts/converter.rb b/tool/ruby_vm/scripts/converter.rb new file mode 100644 index 0000000000..4e7c28d67b --- /dev/null +++ b/tool/ruby_vm/scripts/converter.rb @@ -0,0 +1,29 @@ +# This script was needed only once when I converted the old insns.def. +# Consider historical. +# +# ruby converter.rb insns.def | sponge insns.def + +BEGIN { $str = ARGF.read } +END { puts $str } + +# deal with spaces +$str.gsub! %r/\r\n|\r|\n|\z/, "\n" +$str.gsub! %r/([^\t\n]*)\t/ do + x = $1 + y = 8 - x.length % 8 + next x + ' ' * y +end +$str.gsub! %r/\s+$/, "\n" + +# deal with comments +$str.gsub! %r/@c.*?@e/m, '' +$str.gsub! %r/@j.*?\*\//m, '*/' +$str.gsub! %r/\n(\s*\n)+/, "\n\n" +$str.gsub! %r/\/\*\*?\s*\n\s*/, "/* " +$str.gsub! %r/\n\s+\*\//, " */" +$str.gsub! %r/^(?!.*\/\*.+\*\/$)(.+?)\s*\*\//, "\\1\n */" + +# deal with sp_inc +$str.gsub! %r/ \/\/ inc -= (.*)/, ' // inc += -\\1' +$str.gsub! %r/\s+\/\/ inc \+= (.*)/, "\n// attr rb_snum_t sp_inc = \\1;" +$str.gsub! %r/;;$/, ";" diff --git a/tool/ruby_vm/scripts/insns2vm.rb b/tool/ruby_vm/scripts/insns2vm.rb new file mode 100644 index 0000000000..ad8603b1a8 --- /dev/null +++ b/tool/ruby_vm/scripts/insns2vm.rb @@ -0,0 +1,100 @@ +# -*- Ruby -*- +# -*- frozen_string_literal: true; -*- +# -*- warn_indent: true; -*- +# +# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +# +# This file is a part of the programming language Ruby. Permission is hereby +# granted, to either redistribute and/or modify this file, provided that the +# conditions mentioned in the file COPYING are met. Consult the file for +# details. + +require 'optparse' +require_relative '../controllers/application_controller.rb' + +module RubyVM::Insns2VM + def self.router argv + options = { destdir: nil, basedir: nil } + targets = generate_parser(options).parse argv + return targets.map do |i| + next ApplicationController.new.generate i, options[:destdir], options[:basedir] + end + end + + def self.generate_parser(options) + OptionParser.new do |this| + this.on "-I", "--srcdir=DIR", <<-'end' + Historically this option has been passed to the script. This is + supposedly because at the beginning the script was placed + outside of the ruby source tree. Decades passed since the merge + of YARV, now I can safely assume this feature is obsolescent. + Just ignore the passed value here. + end + + this.on "-L", "--vpath=SPEC", <<-'end' + Likewise, this option is no longer supported. + end + + this.on "--path-separator=SEP", /\A(?:\W\z|\.(\W).+)/, <<-'end' + Old script says this option is a "separator for vpath". I am + confident we no longer need this option. + end + + this.on "-Dname", "--enable=name[,name...]", Array, <<-'end' + This option was used to override VM option that is defined in + vm_opts.h. Now it is officially unsupported because vm_opts.h to + remain mismatched with this option must break things. Just edit + vm_opts.h directly. + end + + this.on "-Uname", "--disable=name[,name...]", Array, <<-'end' + This option was used to override VM option that is defined in + vm_opts.h. Now it is officially unsupported because vm_opts.h to + remain mismatched with this option must break things. Just edit + vm_opts.h directly. + end + + this.on "-i", "--insnsdef=FILE", "--instructions-def", <<-'end' + This option was used to specify alternative path to insns.def. For + the same reason to ignore -I, we no longer support this. + end + + this.on "-o", "--opt-operanddef=FILE", "--opt-operand-def", <<-'end' + This option was used to specify alternative path to opt_operand.def. + For the same reason to ignore -I, we no longer support this. + end + + this.on "-u", "--opt-insnunifdef=FILE", "--opt-insn-unif-def", <<-'end' + This option was used to specify alternative path to + opt_insn_unif.def. For the same reason to ignore -I, we no + longer support this. + end + + this.on "-C", "--[no-]use-const", <<-'end' + We use const whenever possible now so this option is ignored. + The author believes that C compilers can constant-fold. + end + + this.on "-d", "--destdir", "--output-directory=DIR", <<-'begin' do |dir| + THIS IS THE ONLY OPTION THAT WORKS today. Change destination + directory from the current working directory to the given path. + begin + raise "directory was not found in '#{dir}'" unless Dir.exist?(dir) + options[:destdir] = dir + end + + this.on "--basedir=DIR", <<-'begin' do |dir| + Change the base directory from the current working directory + to the given path. Used for searching the source template. + begin + raise "directory was not found in '#{dir}'" unless Dir.exist?(dir) + options[:basedir] = dir + end + + this.on "-V", "--[no-]verbose", <<-'end' + Please let us ignore this and be modest. + end + end + end + private_class_method :generate_parser +end diff --git a/tool/ruby_vm/views/_attributes.erb b/tool/ruby_vm/views/_attributes.erb new file mode 100644 index 0000000000..89a89817af --- /dev/null +++ b/tool/ruby_vm/views/_attributes.erb @@ -0,0 +1,35 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# +#ifndef RUBY_VM_EXEC_H +/* can't #include "vm_exec.h" here... */ +typedef long OFFSET; +typedef unsigned long lindex_t; +typedef VALUE GENTRY; +typedef rb_iseq_t *ISEQ; +#endif + +% attrs = RubyVM::Instructions.map(&:attributes).flatten +% +% attrs.each do |a| +PUREFUNC(MAYBE_UNUSED(static <%= a.declaration %>)); +% end +% +% attrs.each do |a| + +/* <%= a.pretty_name %> */ +<%= a.definition %> +{ +% str = render_c_expr a.expr +% case str when /\A#/ then + return +<%= str -%> +% else + return <%= str -%> +% end +} +% end diff --git a/tool/ruby_vm/views/_c_expr.erb b/tool/ruby_vm/views/_c_expr.erb new file mode 100644 index 0000000000..4e1b0ec883 --- /dev/null +++ b/tool/ruby_vm/views/_c_expr.erb @@ -0,0 +1,17 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +% if expr.blank? +% # empty +% elsif ! expr.__LINE__ +<%= expr.expr %> +% else +#line <%= expr.__LINE__ %> <%=cstr expr.__FILE__ %> +<%= expr.expr %> +#pragma RubyVM reset source +% end diff --git a/tool/ruby_vm/views/_comptime_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb new file mode 100644 index 0000000000..8bb28db1c1 --- /dev/null +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -0,0 +1,71 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%# +% +% stack_increase = proc do |i| +% if i.has_attribute?('sp_inc') +% '-127' +% else +% sprintf("%4d", i.rets.size - i.pops.size) +% end +% end +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); +PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); + +rb_snum_t +comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) +{ + static const signed char t[] = { +% insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(8) do |row| + <%= row.map(&stack_increase).join(', ') -%>, +% end +#endif + }; + signed char c = t[insn]; + + ASSERT_VM_INSTRUCTION_SIZE(t); + if (c != -127) { + return c; + } + else switch(insn) { + default: + UNREACHABLE; +% RubyVM::Instructions.each do |i| +% next unless i.has_attribute?('sp_inc') +% attr_function = +% if i.has_attribute?('comptime_sp_inc') +% "attr_comptime_sp_inc_#{i.name}" +% else +% "attr_sp_inc_#{i.name}" +% end + case <%= i.bin %>: + return <%= attr_function %>(<%= + i.operands.map.with_index do |v, j| + if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc') + v = v.dup + v[:type] = 'CALL_INFO' + end + i.cast_from_VALUE v, "opes[#{j}]" + end.join(", ") + %>); +% end + } +} + +int +comptime_insn_stack_increase(int depth, int insn, const VALUE *opes) +{ + enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn; + return depth + (int)comptime_insn_stack_increase_dispatch(itype, opes); +} diff --git a/tool/ruby_vm/views/_copyright.erb b/tool/ruby_vm/views/_copyright.erb new file mode 100644 index 0000000000..2154146f0d --- /dev/null +++ b/tool/ruby_vm/views/_copyright.erb @@ -0,0 +1,31 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%; +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +%; +%# Below is the licensing term for the generated output, not this erb file. +/* This is an auto-generated file and is a part of the programming language + * Ruby. The person who created a program to generate this file (``I'' + * hereafter) would like to refrain from defining licensing of this generated + * source code. + * + * This file consists of many small parts of codes copyrighted by each author, + * not only the ``I'' person. Those original authors agree with some + * open-source license. I believe that the license we agree is the condition + * mentioned in the file COPYING. It states "4. You may modify and include + * the part of the software into any other software ...". But the problem is, + * the license never makes it clear if such modified parts still remain in the + * same license, or not. The fact that we agree with the source code's + * licensing terms does not automatically define that of generated ones. This + * is the reason why this file is under an unclear situation. All what I know + * is that above provision guarantees this file to exist. + * + * Please let me hesitate to declare something about this nuanced contract. I + * am not in the position to take over other authors' license to merge into my + * one. Changing them to (say) GPLv3 is not doable by myself. Perhaps someday + * it might turn out to be okay to say this file is under a license. I wish + * the situation would become more clear in the future. */ diff --git a/tool/ruby_vm/views/_insn_entry.erb b/tool/ruby_vm/views/_insn_entry.erb new file mode 100644 index 0000000000..6ec33461c4 --- /dev/null +++ b/tool/ruby_vm/views/_insn_entry.erb @@ -0,0 +1,75 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +% body = render_c_expr(insn.expr).gsub(/^#/, '# ') + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + /* ### Declare that we have just entered into an instruction. ### */ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + DEBUG_ENTER_INSN(<%=cstr insn.name %>); + + /* ### Declare and assign variables. ### */ +% insn.preamble.each do |konst| +<%= render_c_expr konst -%> +% end +% +% insn.operands.each_with_index do |ope, i| + <%= ope[:decl] %> = (<%= ope[:type] %>)GET_OPERAND(<%= i + 1 %>); +% end +# define INSN_ATTR(x) <%= insn.call_attribute(' ## x ## ') %> + const bool MAYBE_UNUSED(leaf) = INSN_ATTR(leaf); +% insn.pops.reverse_each.with_index.reverse_each do |pop, i| + <%= pop[:decl] %> = <%= insn.cast_from_VALUE pop, "TOPN(#{i})"%>; +% end +% +% insn.rets.each do |ret| +% next if insn.has_ope?(ret) or insn.has_pop?(ret) + <%= ret[:decl] %>; +% end + + /* ### Instruction preambles. ### */ + ADD_PC(INSN_ATTR(width)); +% if insn.handles_sp? + POPN(INSN_ATTR(popn)); +% end +<%= insn.handle_canary "SETUP_CANARY(leaf)" -%> + COLLECT_USAGE_INSN(INSN_ATTR(bin)); +% insn.operands.each_with_index do |ope, i| + COLLECT_USAGE_OPERAND(INSN_ATTR(bin), <%= i %>, <%= ope[:name] %>); +% end +% unless body.empty? + + /* ### Here we do the instruction body. ### */ +%# NAME_OF_CURRENT_INSN is used in vm_exec.h +# define NAME_OF_CURRENT_INSN <%= insn.name %> +<%= body -%> +# undef NAME_OF_CURRENT_INSN +% end + + /* ### Instruction trailers. ### */ + CHECK_VM_STACK_OVERFLOW_FOR_INSN(VM_REG_CFP, INSN_ATTR(retn)); +<%= insn.handle_canary "CHECK_CANARY(leaf, INSN_ATTR(bin))" -%> +% if insn.handles_sp? +% insn.rets.reverse_each do |ret| + PUSH(<%= insn.cast_to_VALUE ret %>); +% end +% else + INC_SP(INSN_ATTR(sp_inc)); +% insn.rets.reverse_each.with_index do |ret, i| + TOPN(<%= i %>) = <%= insn.cast_to_VALUE ret %>; + VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_NONE)); + VM_ASSERT(!RB_TYPE_P(TOPN(<%= i %>), T_MOVED)); +% end +% end +# undef INSN_ATTR + + /* ### Leave the instruction. ### */ + END_INSN(<%= insn.name %>); +} diff --git a/tool/ruby_vm/views/_insn_leaf_info.erb b/tool/ruby_vm/views/_insn_leaf_info.erb new file mode 100644 index 0000000000..f30366ffda --- /dev/null +++ b/tool/ruby_vm/views/_insn_leaf_info.erb @@ -0,0 +1,18 @@ +MAYBE_UNUSED(static bool insn_leaf(int insn, const VALUE *opes)); +static bool +insn_leaf(int insn, const VALUE *opes) +{ + switch (insn) { +% RubyVM::Instructions.each do |insn| +% next if insn.is_a?(RubyVM::TraceInstruction) || insn.is_a?(RubyVM::ZJITInstruction) + case <%= insn.bin %>: + return attr_leaf_<%= insn.name %>(<%= + insn.operands.map.with_index do |ope, i| + "(#{ope[:type]})opes[#{i}]" + end.join(', ') + %>); +% end + default: + return false; + } +} diff --git a/tool/ruby_vm/views/_insn_len_info.erb b/tool/ruby_vm/views/_insn_len_info.erb new file mode 100644 index 0000000000..b29a405918 --- /dev/null +++ b/tool/ruby_vm/views/_insn_len_info.erb @@ -0,0 +1,36 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +CONSTFUNC(MAYBE_UNUSED(static int insn_len(VALUE insn))); + +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const uint8_t rb_vm_insn_len_info[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const uint8_t rb_vm_insn_len_info[] = { +% insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(23) do |row| + <%= row.map(&:width).join(', ') -%>, +% end +#endif +}; + +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_len_info); +#endif + +int +insn_len(VALUE i) +{ + return rb_vm_insn_len_info[i]; +} diff --git a/tool/ruby_vm/views/_insn_name_info.erb b/tool/ruby_vm/views/_insn_name_info.erb new file mode 100644 index 0000000000..2862908631 --- /dev/null +++ b/tool/ruby_vm/views/_insn_name_info.erb @@ -0,0 +1,59 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% next_offset = 0 +% name_offset = proc do |i| +% offset = sprintf("%4d", next_offset) +% next_offset += i.name.length + 1 # insn.name + \0 +% offset +% end +% +CONSTFUNC(MAYBE_UNUSED(static const char *insn_name(VALUE insn))); + +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const int rb_vm_max_insn_name_size; +extern const char rb_vm_insn_name_base[]; +extern const unsigned short rb_vm_insn_name_offset[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +%# "trace_" is longer than "zjit_", so USE_ZJIT doesn't impact the max name size. +const int rb_vm_max_insn_name_size = <%= RubyVM::Instructions.map { |i| i.name.size }.max %>; + +const char rb_vm_insn_name_base[] = +% insns.each do |i| + <%= cstr i.name %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each do |i| + <%= cstr i.name %> "\0" +% end +#endif + ; + +const unsigned short rb_vm_insn_name_offset[] = { +% insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&name_offset).join(', ') %>, +% end +#endif +}; + +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_name_offset); +#endif + +const char * +insn_name(VALUE i) +{ + return &rb_vm_insn_name_base[rb_vm_insn_name_offset[i]]; +} diff --git a/tool/ruby_vm/views/_insn_operand_info.erb b/tool/ruby_vm/views/_insn_operand_info.erb new file mode 100644 index 0000000000..410869fcd3 --- /dev/null +++ b/tool/ruby_vm/views/_insn_operand_info.erb @@ -0,0 +1,69 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +% operands_info = proc { |i| sprintf("%-6s", cstr(i.operands_info)) } +% +% next_offset = 0 +% op_offset = proc do |i| +% offset = sprintf("%3d", next_offset) +% next_offset += i.operands_info.length + 1 # insn.operands_info + \0 +% offset +% end +% +CONSTFUNC(MAYBE_UNUSED(static const char *insn_op_types(VALUE insn))); +CONSTFUNC(MAYBE_UNUSED(static int insn_op_type(VALUE insn, long pos))); + +RUBY_SYMBOL_EXPORT_BEGIN /* for debuggers */ +extern const char rb_vm_insn_op_base[]; +extern const unsigned short rb_vm_insn_op_offset[VM_INSTRUCTION_SIZE]; +RUBY_SYMBOL_EXPORT_END + +#ifdef RUBY_VM_INSNS_INFO +const char rb_vm_insn_op_base[] = +% insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" +% end +#if USE_ZJIT +% zjit_insns.each_slice(5) do |row| + <%= row.map(&operands_info).join(' "\0" ') %> "\0" +% end +#endif + ; + +const unsigned short rb_vm_insn_op_offset[] = { +% insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, +% end +#if USE_ZJIT +% zjit_insns.each_slice(12) do |row| + <%= row.map(&op_offset).join(', ') %>, +% end +#endif +}; + +ASSERT_VM_INSTRUCTION_SIZE(rb_vm_insn_op_offset); +#endif + +const char * +insn_op_types(VALUE i) +{ + return &rb_vm_insn_op_base[rb_vm_insn_op_offset[i]]; +} + +int +insn_op_type(VALUE i, long j) +{ + if (j >= insn_len(i)) { + return 0; + } + else { + return insn_op_types(i)[j]; + } +} diff --git a/tool/ruby_vm/views/_insn_type_chars.erb b/tool/ruby_vm/views/_insn_type_chars.erb new file mode 100644 index 0000000000..27daec6c6d --- /dev/null +++ b/tool/ruby_vm/views/_insn_type_chars.erb @@ -0,0 +1,32 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +enum ruby_insn_type_chars { +% RubyVM::Typemap.each_value do |(c, t)| + <%= t %> = '<%= c %>', +% end +}; + +static inline union iseq_inline_storage_entry * +ISEQ_IS_ENTRY_START(const struct rb_iseq_constant_body *body, char op_type) +{ + unsigned int relative_ic_offset = 0; + switch (op_type) { + case TS_IC: + relative_ic_offset += body->ise_size; + case TS_ISE: + relative_ic_offset += body->icvarc_size; + case TS_ICVARC: + relative_ic_offset += body->ivc_size; + case TS_IVC: + break; + default: + rb_bug("Wrong op type"); + } + return &body->is_entries[relative_ic_offset]; +} diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb new file mode 100644 index 0000000000..2756fa2dec --- /dev/null +++ b/tool/ruby_vm/views/_leaf_helpers.erb @@ -0,0 +1,50 @@ +%# -*- C -*- +%# Copyright (c) 2018 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +#line <%= __LINE__ + 1 %> <%=cstr __FILE__ %> + +#include "iseq.h" + +static bool +leafness_of_defined(rb_num_t op_type) +{ + /* see also: vm_insnhelper.c:vm_defined() */ + switch (op_type) { + case DEFINED_IVAR: + case DEFINED_GVAR: + case DEFINED_CVAR: + case DEFINED_YIELD: + case DEFINED_REF: + case DEFINED_ZSUPER: + return true; + case DEFINED_CONST: + case DEFINED_CONST_FROM: + /* has rb_autoload_load(); */ + return false; + case DEFINED_FUNC: + case DEFINED_METHOD: + /* calls #respond_to_missing? */ + return false; + default: + rb_bug("unknown operand %ld: blame @shyouhei.", op_type); + } +} + +static bool +leafness_of_checkmatch(rb_num_t flag) +{ + /* see also: vm_insnhelper.c:check_match() */ + if (flag == VM_CHECKMATCH_TYPE_WHEN) { + return true; + } + else { + /* has rb_funcallv() */ + return false; + } +} +#pragma RubyVM reset source diff --git a/tool/ruby_vm/views/_notice.erb b/tool/ruby_vm/views/_notice.erb new file mode 100644 index 0000000000..d17e019727 --- /dev/null +++ b/tool/ruby_vm/views/_notice.erb @@ -0,0 +1,22 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%; +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +%; +/*******************************************************************/ +/*******************************************************************/ +/*******************************************************************/ +/** + This file <%= this_file %>. + + ---- + This file is auto generated by insns2vm.rb + DO NOT TOUCH! + + If you want to fix something, you must edit <%= cstr edit %> + or tool/insns2vm.rb + */ diff --git a/tool/ruby_vm/views/_sp_inc_helpers.erb b/tool/ruby_vm/views/_sp_inc_helpers.erb new file mode 100644 index 0000000000..740fe10142 --- /dev/null +++ b/tool/ruby_vm/views/_sp_inc_helpers.erb @@ -0,0 +1,37 @@ +%# -*- C -*- +%# Copyright (c) 2018 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; +#line <%= __LINE__ + 1 %> <%=cstr __FILE__ %> + +static rb_snum_t +sp_inc_of_sendish(const struct rb_callinfo *ci) +{ + /* Send-ish instructions will: + * + * 1. Pop block argument, if any. + * 2. Pop ordinal arguments. + * 3. Pop receiver. + * 4. Push return value. + */ + const int argb = (vm_ci_flag(ci) & (VM_CALL_ARGS_BLOCKARG | VM_CALL_FORWARDING)) ? 1 : 0; + const int argc = vm_ci_argc(ci); + const int recv = 1; + const int retn = 1; + + /* 1. 2. 3. 4. */ + return 0 - argb - argc - recv + retn; +} + +static rb_snum_t +sp_inc_of_invokeblock(const struct rb_callinfo *ci) +{ + /* sp_inc of invokeblock is almost identical to that of sendish + * instructions, except that it does not pop receiver. */ + return sp_inc_of_sendish(ci) + 1; +} +#pragma RubyVM reset source diff --git a/tool/ruby_vm/views/_trace_instruction.erb b/tool/ruby_vm/views/_trace_instruction.erb new file mode 100644 index 0000000000..3588207d39 --- /dev/null +++ b/tool/ruby_vm/views/_trace_instruction.erb @@ -0,0 +1,21 @@ +%# -*- C -*- +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +%; + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + vm_trace(ec, GET_CFP()); +% if insn.name =~ +% /\Atrace_opt_(plus|minus|mult|div|mod|eq|neq|lt|le|gt|ge|ltlt|and|or|aref|aset|length|size|empty_p|nil_p|succ|not|regexpmatch2)\z/ +% jump_dest = "opt_send_without_block" +% end + <%= 'ADD_PC(1);' if insn.name == 'trace_opt_neq' %> + DISPATCH_ORIGINAL_INSN(<%= jump_dest || insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} diff --git a/tool/ruby_vm/views/_zjit_helpers.erb b/tool/ruby_vm/views/_zjit_helpers.erb new file mode 100644 index 0000000000..1185dbd9d8 --- /dev/null +++ b/tool/ruby_vm/views/_zjit_helpers.erb @@ -0,0 +1,31 @@ +#if USE_ZJIT + +MAYBE_UNUSED(static int vm_bare_insn_to_zjit_insn(int insn)); +static int +vm_bare_insn_to_zjit_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case BIN(<%= insn.jump_destination %>): + return <%= insn.bin %>; +% end + default: + return insn; + } +} + +MAYBE_UNUSED(static int vm_zjit_insn_to_bare_insn(int insn)); +static int +vm_zjit_insn_to_bare_insn(int insn) +{ + switch (insn) { +% RubyVM::ZJITInstruction.all.each do |insn| + case <%= insn.bin %>: + return BIN(<%= insn.jump_destination %>); +% end + default: + return insn; + } +} + +#endif diff --git a/tool/ruby_vm/views/_zjit_instruction.erb b/tool/ruby_vm/views/_zjit_instruction.erb new file mode 100644 index 0000000000..7fd657697c --- /dev/null +++ b/tool/ruby_vm/views/_zjit_instruction.erb @@ -0,0 +1,12 @@ +#if USE_ZJIT + +/* insn <%= insn.pretty_name %> */ +INSN_ENTRY(<%= insn.name %>) +{ + START_OF_ORIGINAL_INSN(<%= insn.name %>); + rb_zjit_profile_insn(BIN(<%= insn.jump_destination %>), ec); + DISPATCH_ORIGINAL_INSN(<%= insn.jump_destination %>); + END_INSN(<%= insn.name %>); +} + +#endif diff --git a/tool/ruby_vm/views/insns.inc.erb b/tool/ruby_vm/views/insns.inc.erb new file mode 100644 index 0000000000..6521a89b8a --- /dev/null +++ b/tool/ruby_vm/views/insns.inc.erb @@ -0,0 +1,41 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'contains YARV instruction list', + edit: __FILE__, +} -%> + +#ifndef INSNS_INC +#define INSNS_INC 1 + +/* BIN : Basic Instruction Name */ +#define BIN(n) YARVINSN_##n + +enum ruby_vminsn_type { +% insns.each do |i| + <%= i.bin %>, +% end +#if USE_ZJIT +% zjit_insns.each do |i| + <%= i.bin %>, +% end +#endif + VM_INSTRUCTION_SIZE +}; + +#define VM_BARE_INSTRUCTION_SIZE <%= RubyVM::Instructions.count { |i| i.name !~ /\A(trace|zjit)_/ } %> + +#define ASSERT_VM_INSTRUCTION_SIZE(array) \ + STATIC_ASSERT(numberof_##array, numberof(array) == VM_INSTRUCTION_SIZE) + +#endif diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb new file mode 100644 index 0000000000..48dd0e8832 --- /dev/null +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -0,0 +1,26 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'contains instruction information for yarv instruction sequence.', + edit: __FILE__, +} %> +#ifndef INSNS_INFO_INC +#define INSNS_INFO_INC 1 +<%= render 'insn_type_chars' %> +<%= render 'insn_name_info' %> +<%= render 'insn_len_info' %> +<%= render 'insn_operand_info' %> +<%= render 'leaf_helpers' %> +<%= render 'sp_inc_helpers' %> +<%= render 'zjit_helpers' %> +<%= render 'attributes' %> +<%= render 'insn_leaf_info' %> +<%= render 'comptime_insn_stack_increase' %> +#endif diff --git a/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb new file mode 100644 index 0000000000..793528af5d --- /dev/null +++ b/tool/ruby_vm/views/lib/ruby_vm/rjit/instruction.rb.erb @@ -0,0 +1,14 @@ +module RubyVM::RJIT # :nodoc: all + Instruction = Data.define(:name, :bin, :len, :operands) + + INSNS = { +% RubyVM::Instructions.each_with_index do |insn, i| + <%= i %> => Instruction.new( + name: :<%= insn.name %>, + bin: <%= i %>, # BIN(<%= insn.name %>) + len: <%= insn.width %>, # insn_len + operands: <%= (insn.operands unless insn.name.start_with?(/trace_|zjit_/)).inspect %>, + ), +% end + } +end diff --git a/tool/ruby_vm/views/optinsn.inc.erb b/tool/ruby_vm/views/optinsn.inc.erb new file mode 100644 index 0000000000..9d9cf0a43a --- /dev/null +++ b/tool/ruby_vm/views/optinsn.inc.erb @@ -0,0 +1,71 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +<%= render 'copyright' -%> +<%= render 'notice', locals: { + this_file: 'is for threaded code', + edit: __FILE__, +} -%> + +static INSN * +insn_operands_unification(INSN *iobj) +{ +#ifdef OPT_OPERANDS_UNIFICATION + VALUE *op = iobj->operands; + + switch (iobj->insn_id) { + default: + /* do nothing */; + break; + +% RubyVM::OperandsUnification.each_group do |orig, unifs| + case <%= orig.bin %>: +% unifs.each do |insn| + + /* <%= insn.pretty_name %> */ + if ( <%= insn.condition('op') %> ) { +% insn.operands.each_with_index do |o, x| +% n = insn.operand_shift_of(o) +% if n != 0 then + op[<%= x %>] = op[<%= x + n %>]; +% end +% end + iobj->insn_id = <%= insn.bin %>; + iobj->operand_size = <%= insn.operands.size %>; + break; + } +% end + + break; +% end + } +#endif + return iobj; +} + +int +rb_insn_unified_local_var_level(VALUE insn) +{ +#ifdef OPT_OPERANDS_UNIFICATION + /* optimize rule */ + switch (insn) { + default: + return -1; /* do nothing */; +% RubyVM::OperandsUnification.each_group do |orig, unifs| +% unifs.each do|insn| + case <%= insn.bin %>: +% insn.spec.map{|(var,val)|val}.reject{|i| i == '*' }.each do |val| + return <%= val %>; +% break +% end +% end +% end + } +#endif + return -1; +} diff --git a/tool/ruby_vm/views/optunifs.inc.erb b/tool/ruby_vm/views/optunifs.inc.erb new file mode 100644 index 0000000000..c096712936 --- /dev/null +++ b/tool/ruby_vm/views/optunifs.inc.erb @@ -0,0 +1,18 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% raise ':FIXME:TBW' if RubyVM::VmOptsH['INSTRUCTIONS_UNIFICATION'] +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'is for threaded code', + edit: __FILE__, +} -%> + +/* Let .bss section automatically initialize this variable */ +/* cf. Section 6.7.8 of ISO/IEC 9899:1999 */ +static const int *const *const unified_insns_data[VM_INSTRUCTION_SIZE]; diff --git a/tool/ruby_vm/views/vm.inc.erb b/tool/ruby_vm/views/vm.inc.erb new file mode 100644 index 0000000000..38bf5f05ae --- /dev/null +++ b/tool/ruby_vm/views/vm.inc.erb @@ -0,0 +1,34 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +<%= render 'copyright' %> +<%= render 'notice', locals: { + this_file: 'is VM main loop', + edit: __FILE__, +} -%> + +#include "vm_insnhelper.h" +% RubyVM::BareInstruction.all.each do |insn| +<%= render 'insn_entry', locals: { insn: insn } -%> +% end +% +% RubyVM::OperandsUnification.all.each do |insn| +<%= render 'insn_entry', locals: { insn: insn } -%> +% end +% +% RubyVM::InstructionsUnification.all.each do |insn| +<%= render 'insn_entry', locals: { insn: insn } -%> +% end +% +% RubyVM::ZJITInstruction.all.each do |insn| +<%= render 'zjit_instruction', locals: { insn: insn } -%> +% end +% +% RubyVM::TraceInstruction.all.each do |insn| +<%= render 'trace_instruction', locals: { insn: insn } -%> +% end diff --git a/tool/ruby_vm/views/vmtc.inc.erb b/tool/ruby_vm/views/vmtc.inc.erb new file mode 100644 index 0000000000..39dc8bfa6b --- /dev/null +++ b/tool/ruby_vm/views/vmtc.inc.erb @@ -0,0 +1,29 @@ +/* -*- C -*- */ + +%# Copyright (c) 2017 Urabe, Shyouhei. All rights reserved. +%# +%# This file is a part of the programming language Ruby. Permission is hereby +%# granted, to either redistribute and/or modify this file, provided that the +%# conditions mentioned in the file COPYING are met. Consult the file for +%# details. +% +% zjit_insns, insns = RubyVM::Instructions.partition { |i| i.name.start_with?('zjit_') } +% +<%= render 'copyright' -%> +<%= render 'notice', locals: { + this_file: 'is for threaded code', + edit: __FILE__, +} -%> + +static const void *const insns_address_table[] = { +% insns.each do |i| + LABEL_PTR(<%= i.name %>), +% end +#if USE_ZJIT +% zjit_insns.each do |i| + LABEL_PTR(<%= i.name %>), +% end +#endif +}; + +ASSERT_VM_INSTRUCTION_SIZE(insns_address_table); diff --git a/tool/rubytest.rb b/tool/rubytest.rb deleted file mode 100755 index a06c400744..0000000000 --- a/tool/rubytest.rb +++ /dev/null @@ -1,30 +0,0 @@ -#! ./miniruby - -exit if defined?(CROSS_COMPILING) and CROSS_COMPILING -ruby = ENV["RUBY"] -unless ruby - load './rbconfig.rb' - ruby = "./#{RbConfig::CONFIG['ruby_install_name']}#{RbConfig::CONFIG['EXEEXT']}" -end -unless File.exist? ruby - print "#{ruby} is not found.\n" - print "Try `make' first, then `make test', please.\n" - exit false -end -ARGV[0] and opt = ARGV[0][/\A--run-opt=(.*)/, 1] and ARGV.shift - -$stderr.reopen($stdout) -error = '' - -srcdir = File.expand_path('..', File.dirname(__FILE__)) -`#{ruby} #{opt} #{srcdir}/sample/test.rb #{ARGV.join(' ')}`.each_line do |line| - if line =~ /^end of test/ - print "\ntest succeeded\n" - exit true - end - error << line if %r:^(sample/test.rb|not): =~ line -end -puts -print error -print "test failed\n" -exit false diff --git a/tool/run-gcov.rb b/tool/run-gcov.rb new file mode 100644 index 0000000000..46626e4703 --- /dev/null +++ b/tool/run-gcov.rb @@ -0,0 +1,55 @@ +#!ruby +require "pathname" +require "open3" + +Pathname.glob("**/*.gcda").sort.each do |gcda| + if gcda.fnmatch("ext/*") + cwd, gcda = gcda.split.map {|s| s.to_s } + objdir = "." + elsif gcda.fnmatch("rubyspec_temp/*") + next + else + cwd, objdir, gcda = ".", gcda.dirname.to_s, gcda.to_s + end + puts "$ gcov -lpbc -o #{ objdir } #{ gcda }" + out, err, _status = Open3.capture3("gcov", "-lpbc", "-o", objdir, gcda, chdir: cwd) + puts out + puts err + + # a black list of source files that contains wrong #line directives + if err !~ %r( + \A( + Cannot\ open\ source\ file\ ( + defs/keywords + |zonetab\.list + |enc/jis/props\.kwd + |parser\.c + |parser\.rl + )\n + )*\z + )x + raise "Unexpected gcov output" + end + + if out !~ %r( + \A( + File\ .*\nLines\ executed:.*\n + ( + Branches\ executed:.*\n + Taken\ at\ least\ once:.*\n + | + No\ branches\n + )? + ( + Calls\ executed:.*\n + | + No\ calls\n + )? + Creating\ .*\n + \n + )+ + (Lines\ executed:.*\n)?\z + )x + raise "Unexpected gcov output" + end +end diff --git a/tool/run-lcov.rb b/tool/run-lcov.rb new file mode 100644 index 0000000000..bdccc29a11 --- /dev/null +++ b/tool/run-lcov.rb @@ -0,0 +1,172 @@ +#!ruby +require "pathname" +require "open3" +require "tmpdir" + +def backup_gcda_files(gcda_files) + gcda_files = gcda_files.map do |gcda| + [gcda, gcda.sub_ext(".bak")] + end + begin + gcda_files.each do |before, after| + before.rename(after) + end + yield + ensure + gcda_files.each do |before, after| + after.rename(before) + end + end +end + +def run_lcov(*args) + system("lcov", "--rc", "geninfo_unexecuted_blocks=1", "--rc", "lcov_branch_coverage=1", *args, exception: true) +end + +$info_files = [] +def run_lcov_capture(dir, info) + $info_files << info + run_lcov("--capture", "-d", dir, "-o", info) +end + +def run_lcov_merge(files, info) + run_lcov(*files.flat_map {|f| ["--add-tracefile", f] }, "-o", info) +end + +def run_lcov_remove(info_src, info_out) + dirs = %w(/usr/*) + dirs << File.join(Dir.tmpdir, "*") + %w( + test/* + ext/-test-/* + ext/nkf/nkf-utf8/nkf.c + ).each {|f| dirs << File.join(File.dirname(__dir__), f) } + run_lcov("--ignore-errors", "unused", "--remove", info_src, *dirs, "-o", info_out) +end + +def run_genhtml(info, out) + base_dir = File.dirname(File.dirname(__dir__)) + ignore_errors = %w(source unmapped category).reject do |a| + Open3.capture3("genhtml", "--ignore-errors", a)[1].include?("unknown argument for --ignore-errors") + end + system("genhtml", + "--branch-coverage", + "--prefix", base_dir, + *ignore_errors.flat_map {|a| ["--ignore-errors", a] }, + info, "-o", out, exception: true) +end + +def gen_rb_lcov(file) + res = Marshal.load(File.binread(file)) + + open("lcov-rb-all.info", "w") do |f| + f.puts "TN:" # no test name + base_dir = File.dirname(__dir__) + res.each do |path, cov| + next unless path.start_with?(base_dir) + next if path.start_with?(File.join(base_dir, "test")) + f.puts "SF:#{ path }" + + total = covered = 0 + cov.each_with_index do |count, lineno| + next unless count + f.puts "DA:#{ lineno + 1 },#{ count }" + total += 1 + covered += 1 if count > 0 + end + f.puts "LF:#{ total }" + f.puts "LH:#{ covered }" + + f.puts "end_of_record" + end + end +end + +def gen_rb_lcov(file) + res = Marshal.load(File.binread(file)) + + open("lcov-rb-all.info", "w") do |f| + f.puts "TN:" # no test name + base_dir = File.dirname(File.dirname(__dir__)) + res.each do |path, cov| + next unless path.start_with?(base_dir) + next if path.start_with?(File.join(base_dir, "test")) + f.puts "SF:#{ path }" + + # function coverage + total = covered = 0 + cov[:methods].each do |(klass, name, lineno), count| + f.puts "FN:#{ lineno },#{ klass }##{ name }" + total += 1 + covered += 1 if count > 0 + end + f.puts "FNF:#{ total }" + f.puts "FNF:#{ covered }" + cov[:methods].each do |(klass, name, _), count| + f.puts "FNDA:#{ count },#{ klass }##{ name }" + end + + # line coverage + total = covered = 0 + cov[:lines].each_with_index do |count, lineno| + next unless count + f.puts "DA:#{ lineno + 1 },#{ count }" + total += 1 + covered += 1 if count > 0 + end + f.puts "LF:#{ total }" + f.puts "LH:#{ covered }" + + # branch coverage + total = covered = 0 + id = 0 + cov[:branches].each do |(_base_type, _, base_lineno), targets| + i = 0 + targets.each do |(_target_type, _target_lineno), count| + f.puts "BRDA:#{ base_lineno },#{ id },#{ i },#{ count }" + total += 1 + covered += 1 if count > 0 + i += 1 + end + id += 1 + end + f.puts "BRF:#{ total }" + f.puts "BRH:#{ covered }" + f.puts "end_of_record" + end + end +end + +gcda_files = Pathname.glob("**/*.gcda") +ext_gcda_files = gcda_files.select {|f| f.fnmatch("ext/*") } +rubyspec_temp_gcda_files = gcda_files.select {|f| f.fnmatch("rubyspec_temp/*") } + +backup_gcda_files(rubyspec_temp_gcda_files) do + if ext_gcda_files != [] + backup_gcda_files(ext_gcda_files) do + info = "lcov-root.info" + run_lcov_capture(".", info) + end + end + ext_gcda_files.group_by {|f| f.descend.to_a[1] }.each do |key, files| + info = "lcov-#{ key.to_s.gsub(File::Separator, "-") }.info" + run_lcov_capture(key.to_s, info) + end +end +if $info_files != [] + run_lcov_merge($info_files, "lcov-c-all.info") + run_lcov_remove("lcov-c-all.info", "lcov-c-all-filtered.info") + run_genhtml("lcov-c-all-filtered.info", "lcov-c-out") +end + +if File.readable?("test-coverage.dat") + gen_rb_lcov("test-coverage.dat") + run_lcov_remove("lcov-rb-all.info", "lcov-rb-all-filtered.info") + run_genhtml("lcov-rb-all-filtered.info", "lcov-rb-out") +end + +if File.readable?("lcov-c-all.info") && File.readable?("lcov-rb-all.info") + run_lcov_merge(%w(lcov-c-all.info lcov-rb-all.info), "lcov-all.info") + run_lcov_remove("lcov-all.info", "lcov-all-filtered.info") + run_genhtml("lcov-all-filtered.info", "lcov-out") +end diff --git a/tool/runruby.rb b/tool/runruby.rb index c316ff2d47..ec63d1008a 100755 --- a/tool/runruby.rb +++ b/tool/runruby.rb @@ -1,9 +1,29 @@ #!./miniruby +# Used by "make runruby", configure, and by hand to run a locally-built Ruby +# with correct environment variables and arguments. + show = false precommand = [] +srcdir = File.realpath('..', File.dirname(__FILE__)) +case +when ENV['RUNRUBY_USE_GDB'] == 'true' + debugger = :gdb +when ENV['RUNRUBY_USE_LLDB'] == 'true' + debugger = :lldb +when ENV['RUNRUBY_USE_RR'] == 'true' + debugger = :rr +when ENV['RUNRUBY_YJIT_STATS'] + use_yjit_stat = true +end while arg = ARGV[0] break ARGV.shift if arg == '--' + case arg + when '-C', /\A-C(.+)/m + ARGV.shift + Dir.chdir($1 || ARGV.shift) + next + end /\A--([-\w]+)(?:=(.*))?\z/ =~ arg or break arg, value = $1, $2 re = Regexp.new('\A'+arg.gsub(/\w+\b/, '\&\\w*')+'\z', "i") @@ -20,12 +40,22 @@ while arg = ARGV[0] # obsolete switch do nothing when re =~ "debugger" require 'shellwords' - precommand.concat(value ? (Shellwords.shellwords(value) unless value == "no") : %w"gdb --args") + case value + when nil + debugger = :gdb + when "lldb" + debugger = :lldb + when "no" + else + debugger = Shellwords.shellwords(value) + end and precommand |= [:debugger] when re =~ "precommand" require 'shellwords' precommand.concat(Shellwords.shellwords(value)) when re =~ "show" show = true + when re =~ "chdir" + Dir.chdir(value) else break end @@ -34,23 +64,33 @@ end unless defined?(File.realpath) def File.realpath(*args) - Dir.chdir(expand_path(*args)) do - Dir.pwd + path = expand_path(*args) + if File.stat(path).directory? + Dir.chdir(path) {Dir.pwd} + else + dir, base = File.split(path) + File.join(Dir.chdir(dir) {Dir.pwd}, base) end end end -srcdir ||= File.realpath('..', File.dirname(__FILE__)) -archdir ||= '.' +begin + conffile = File.realpath('rbconfig.rb', archdir) +rescue Errno::ENOENT => e + # retry if !archdir and ARGV[0] and File.directory?(archdir = ARGV.shift) + abort "#$0: rbconfig.rb not found, use --archdir option" +end -abs_archdir = File.expand_path(archdir) +abs_archdir = File.dirname(conffile) +archdir ||= abs_archdir $:.unshift(abs_archdir) -config = File.read(conffile = File.join(abs_archdir, 'rbconfig.rb')) +config = File.read(conffile) config.sub!(/^(\s*)RUBY_VERSION\b.*(\sor\s*)\n.*\n/, '') config = Module.new {module_eval(config, conffile)}::RbConfig::CONFIG -ruby = File.join(archdir, config["RUBY_INSTALL_NAME"]+config['EXEEXT']) +install_name = config["RUBY_INSTALL_NAME"]+config['EXEEXT'] +ruby = File.join(archdir, install_name) unless File.exist?(ruby) abort "#{ruby} is not found.\nTry `make' first, then `make test', please.\n" end @@ -64,38 +104,77 @@ end libs << File.expand_path("lib", srcdir) config["bindir"] = abs_archdir -env = {} +env = { + # Test with the smallest possible machine stack sizes. + # These values are clamped to machine-dependent minimum values in vm_core.h + 'RUBY_THREAD_MACHINE_STACK_SIZE' => '1', + 'RUBY_FIBER_MACHINE_STACK_SIZE' => '1', +} -runner = File.join(abs_archdir, "ruby-runner#{config['EXEEXT']}") -runner = File.expand_path(ruby) unless File.exist?(runner) -env["RUBY"] = runner -env["PATH"] = [abs_archdir, ENV["PATH"]].compact.join(File::PATH_SEPARATOR) +runner = File.join(abs_archdir, "exe/#{install_name}") +runner = nil unless File.exist?(runner) +abs_ruby = runner || File.expand_path(ruby) +env["RUBY"] = abs_ruby +env["GEM_PATH"] = env["GEM_HOME"] = File.expand_path(".bundle", srcdir) +env["GEM_COMMAND"] = "#{abs_ruby} -rrubygems #{srcdir}/bin/gem --backtrace" +env["PATH"] = [File.dirname(abs_ruby), abs_archdir, ENV["PATH"]].compact.join(File::PATH_SEPARATOR) if e = ENV["RUBYLIB"] libs |= e.split(File::PATH_SEPARATOR) end env["RUBYLIB"] = $:.replace(libs).join(File::PATH_SEPARATOR) +gem_path = [abs_archdir, srcdir].map {|d| File.realdirpath(".bundle", d)} +if e = ENV["GEM_PATH"] + gem_path |= e.split(File::PATH_SEPARATOR) +end +env["GEM_PATH"] = gem_path.join(File::PATH_SEPARATOR) + libruby_so = File.join(abs_archdir, config['LIBRUBY_SO']) if File.file?(libruby_so) if e = config['LIBPATHENV'] and !e.empty? env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) end - if e = config['PRELOADENV'] - e = nil if e.empty? - e ||= "LD_PRELOAD" if /linux/ =~ RUBY_PLATFORM - end - if e - env[e] = [libruby_so, ENV[e]].compact.join(File::PATH_SEPARATOR) - end end +# Work around a bug in FreeBSD 13.2 which can cause fork(2) to hang +# See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271490 +env['LD_BIND_NOW'] = 'yes' if /freebsd/ =~ RUBY_PLATFORM ENV.update env -cmd = [ruby] +if debugger + case debugger + when :gdb + debugger = %W'gdb -x #{srcdir}/.gdbinit' + if File.exist?(gdb = 'run.gdb') or + File.exist?(gdb = File.join(abs_archdir, 'run.gdb')) + debugger.push('-x', gdb) + end + debugger << '--args' + when :lldb + debugger = ['lldb', '-O', "command script import #{srcdir}/misc/lldb_cruby.py"] + if File.exist?(lldb = 'run.lldb') or + File.exist?(lldb = File.join(abs_archdir, 'run.lldb')) + debugger.push('-s', lldb) + end + debugger << '--' + when :rr + debugger = ['rr', 'record'] + end + + if idx = precommand.index(:debugger) + precommand[idx, 1] = debugger + else + precommand.concat(debugger) + end +end + +cmd = [runner || ruby] +if use_yjit_stat + cmd << '--yjit-stats' +end cmd.concat(ARGV) cmd.unshift(*precommand) unless precommand.empty? -cmd.push(:close_others => false) if show require 'shellwords' @@ -103,4 +182,4 @@ if show puts Shellwords.join(cmd) end -exec(*cmd) +exec(*cmd, close_others: false) diff --git a/tool/search-cgvars.rb b/tool/search-cgvars.rb new file mode 100644 index 0000000000..c62641a3ff --- /dev/null +++ b/tool/search-cgvars.rb @@ -0,0 +1,55 @@ +# +# Listing C's global variables in .so or .o, or .bundle on Mac OS using "objdump -t" (elf64-x86-64) +# to check ractor-safety. +# +# Usage: ruby search-cgvars.rb foo.so bar.o .ext/x86_64-darwin18/psych.bundle +# +MAC_OS = RbConfig::CONFIG['host_os'].match? /darwin|mac os/ + +def gvars file + # '0000000000031ac8 g O .bss 0000000000000008 rb_cSockIfaddr' + # On mac, with .bundle files: + # '0000000000004258 l O __DATA,__bss _passwd_blocking' + + strs = `objdump -t #{file}` + found = {} + strs.each_line{|line| + if /[\da-f]{16} / =~ line + addr = line[0...16] + flags = line[17...24].tr(' ', '').split(//).sort.uniq + rest = line[25..] + if MAC_OS + seg, name = rest.split(/\s+/) + else + seg, size, name = rest.split(/\s+/) + end + if flags.include?('O') + # p [addr, flags, seg, size, name] + found[name] = [flags, seg, *size] + end + end + } + puts "## #{file}:" + found.sort_by{|name, (flags, *)| + [flags, name] + }.each{|name, rest| + flags, seg, size = *rest + next if (size.to_i == 0 && !MAC_OS) && seg != '*UND*' + case seg + when ".rodata", ".data.rel.ro", ".got.plt", ".eh_frame", ".fini_array" + next + when /,__const$/ # Mac OS + next + end + case name + when /^id_/, /^rbimpl_id/, /^sym_/, /^rb_[cme]/, /\Acompleted\.\d+\z/ + next + when /^_id_/, /\.rbimpl_id(\.\d+)?$/ # Mac OS + next + end + puts " %40s %s" % [name, rest.inspect] + } +end +ARGV.each{|file| + gvars file +} diff --git a/tool/strip-rdoc.rb b/tool/strip-rdoc.rb index 1902cb538c..d8e311cdbf 100755 --- a/tool/strip-rdoc.rb +++ b/tool/strip-rdoc.rb @@ -1,23 +1,14 @@ #!ruby +# frozen_string_literal: true -ARGF.binmode -source = ARGF.read -source = source.gsub(%r{/\*([!*])((?!\*/).+?)\*/}m) do |comment| - marker, comment = $1, $2 - next "/**#{comment}*/" unless /^\s*\*\s?\-\-\s*$/ =~ comment - doxybody = nil - comment.each_line do |line| - if doxybody - if /^\s*\*\s?\+\+\s*$/ =~ line - break - end - doxybody << line - else - if /^\s*\*\s?--\s*$/ =~ line - doxybody = "\n" - end - end - end - "/*#{marker}#{doxybody}*/" -end -print source +# Filter for preventing Doxygen from processing RDoc comments. +# Used by the Doxygen template. + +print ARGF.binmode.read.tap {|src| + src.gsub!(%r|(/\*[!*])(?:(?!\*/).)+?^\s*\*\s?\-\-\s*$(.+?\*/)|m) { + marker = $1 + comment = $2 + comment.sub!(%r|^\s*\*\s?\+\+\s*$.+?(\s*\*/)\z|m, '\\1') + marker + comment + } +} diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb new file mode 100755 index 0000000000..14d7a3893d --- /dev/null +++ b/tool/sync_default_gems.rb @@ -0,0 +1,943 @@ +#!/usr/bin/env ruby +# Sync upstream github repositories to ruby repository. +# See `tool/sync_default_gems.rb --help` for how to use this. + +require 'fileutils' +require "rbconfig" +require "find" +require "tempfile" + +module SyncDefaultGems + include FileUtils + extend FileUtils + + module_function + + # upstream: "owner/repo" + # branch: "branch_name" + # mappings: [ ["path_in_upstream", "path_in_ruby"], ... ] + # NOTE: path_in_ruby is assumed to be "owned" by this gem, and the contents + # will be removed before sync + # exclude: [ "fnmatch_pattern_after_mapping", ... ] + Repository = Data.define(:upstream, :branch, :mappings, :exclude) do + def excluded?(newpath) + p = newpath + until p == "." + return true if exclude.any? {|pat| File.fnmatch?(pat, p, File::FNM_PATHNAME|File::FNM_EXTGLOB)} + p = File.dirname(p) + end + false + end + + def rewrite_for_ruby(path) + newpath = mappings.find do |src, dst| + if path == src || path.start_with?(src + "/") + break path.sub(src, dst) + end + end + return nil unless newpath + return nil if excluded?(newpath) + newpath + end + end + + CLASSICAL_DEFAULT_BRANCH = "master" + + def repo((upstream, branch), mappings, exclude: []) + branch ||= CLASSICAL_DEFAULT_BRANCH + exclude += ["ext/**/depend"] + Repository.new(upstream:, branch:, mappings:, exclude:) + end + + def lib((upstream, branch), gemspec_in_subdir: false) + _org, name = upstream.split("/") + gemspec_dst = gemspec_in_subdir ? "lib/#{name}/#{name}.gemspec" : "lib/#{name}.gemspec" + repo([upstream, branch], [ + ["lib/#{name}.rb", "lib/#{name}.rb"], + ["lib/#{name}", "lib/#{name}"], + ["test/test_#{name}.rb", "test/test_#{name}.rb"], + ["test/#{name}", "test/#{name}"], + ["#{name}.gemspec", gemspec_dst], + ]) + end + + # Note: tool/auto_review_pr.rb also depends on these constants. + NO_UPSTREAM = [ + "lib/unicode_normalize", # not to match with "lib/un" + ] + REPOSITORIES = { + Onigmo: repo("k-takata/Onigmo", [ + ["regcomp.c", "regcomp.c"], + ["regenc.c", "regenc.c"], + ["regenc.h", "regenc.h"], + ["regerror.c", "regerror.c"], + ["regexec.c", "regexec.c"], + ["regint.h", "regint.h"], + ["regparse.c", "regparse.c"], + ["regparse.h", "regparse.h"], + ["regsyntax.c", "regsyntax.c"], + ["onigmo.h", "include/ruby/onigmo.h"], + ["enc", "enc"], + ]), + "io-console": repo("ruby/io-console", [ + ["ext/io/console", "ext/io/console"], + ["test/io/console", "test/io/console"], + ["lib/io/console", "ext/io/console/lib/console"], + ["io-console.gemspec", "ext/io/console/io-console.gemspec"], + ]), + "io-nonblock": repo("ruby/io-nonblock", [ + ["ext/io/nonblock", "ext/io/nonblock"], + ["test/io/nonblock", "test/io/nonblock"], + ["io-nonblock.gemspec", "ext/io/nonblock/io-nonblock.gemspec"], + ]), + "io-wait": repo("ruby/io-wait", [ + ["ext/io/wait", "ext/io/wait"], + ["test/io/wait", "test/io/wait"], + ["io-wait.gemspec", "ext/io/wait/io-wait.gemspec"], + ]), + "net-http": repo("ruby/net-http", [ + ["lib/net/http.rb", "lib/net/http.rb"], + ["lib/net/http", "lib/net/http"], + ["test/net/http", "test/net/http"], + ["net-http.gemspec", "lib/net/http/net-http.gemspec"], + ]), + "net-protocol": repo("ruby/net-protocol", [ + ["lib/net/protocol.rb", "lib/net/protocol.rb"], + ["test/net/protocol", "test/net/protocol"], + ["net-protocol.gemspec", "lib/net/net-protocol.gemspec"], + ]), + "open-uri": lib("ruby/open-uri"), + "win32-registry": repo("ruby/win32-registry", [ + ["lib/win32/registry.rb", "ext/win32/lib/win32/registry.rb"], + ["test/win32/test_registry.rb", "test/win32/test_registry.rb"], + ["win32-registry.gemspec", "ext/win32/win32-registry.gemspec"], + ]), + English: lib("ruby/English"), + cgi: repo("ruby/cgi", [ + ["ext/cgi", "ext/cgi"], + ["lib/cgi/escape.rb", "lib/cgi/escape.rb"], + ["test/cgi/test_cgi_escape.rb", "test/cgi/test_cgi_escape.rb"], + ["test/cgi/update_env.rb", "test/cgi/update_env.rb"], + ]), + date: repo("ruby/date", [ + ["doc/date", "doc/date"], + ["ext/date", "ext/date"], + ["lib", "ext/date/lib"], + ["test/date", "test/date"], + ["date.gemspec", "ext/date/date.gemspec"], + ], exclude: [ + "ext/date/lib/date_core.bundle", + ]), + delegate: lib("ruby/delegate"), + did_you_mean: repo("ruby/did_you_mean", [ + ["lib/did_you_mean.rb", "lib/did_you_mean.rb"], + ["lib/did_you_mean", "lib/did_you_mean"], + ["test", "test/did_you_mean"], + ["did_you_mean.gemspec", "lib/did_you_mean/did_you_mean.gemspec"], + ], exclude: [ + "test/did_you_mean/lib", + "test/did_you_mean/tree_spell/test_explore.rb", + ]), + digest: repo("ruby/digest", [ + ["ext/digest/lib/digest/sha2", "ext/digest/sha2/lib/sha2"], + ["ext/digest", "ext/digest"], + ["lib/digest.rb", "ext/digest/lib/digest.rb"], + ["lib/digest/version.rb", "ext/digest/lib/digest/version.rb"], + ["lib/digest/sha2.rb", "ext/digest/sha2/lib/sha2.rb"], + ["test/digest", "test/digest"], + ["digest.gemspec", "ext/digest/digest.gemspec"], + ]), + erb: repo("ruby/erb", [ + ["ext/erb", "ext/erb"], + ["lib/erb", "lib/erb"], + ["lib/erb.rb", "lib/erb.rb"], + ["test/erb", "test/erb"], + ["erb.gemspec", "lib/erb/erb.gemspec"], + ["libexec/erb", "libexec/erb"], + ]), + error_highlight: repo("ruby/error_highlight", [ + ["lib/error_highlight.rb", "lib/error_highlight.rb"], + ["lib/error_highlight", "lib/error_highlight"], + ["test", "test/error_highlight"], + ["error_highlight.gemspec", "lib/error_highlight/error_highlight.gemspec"], + ]), + etc: repo("ruby/etc", [ + ["ext/etc", "ext/etc"], + ["test/etc", "test/etc"], + ["etc.gemspec", "ext/etc/etc.gemspec"], + ]), + fcntl: repo("ruby/fcntl", [ + ["ext/fcntl", "ext/fcntl"], + ["fcntl.gemspec", "ext/fcntl/fcntl.gemspec"], + ]), + fileutils: lib("ruby/fileutils"), + find: lib("ruby/find"), + forwardable: lib("ruby/forwardable", gemspec_in_subdir: true), + ipaddr: lib("ruby/ipaddr"), + json: repo("ruby/json", [ + ["ext/json/ext", "ext/json"], + ["test/json", "test/json"], + ["lib", "ext/json/lib"], + ["json.gemspec", "ext/json/json.gemspec"], + ], exclude: [ + "ext/json/lib/json/ext/.keep", + "ext/json/lib/json/pure.rb", + "ext/json/lib/json/pure", + "ext/json/lib/json/truffle_ruby", + "test/json/lib", + "ext/json/extconf.rb", + ]), + mmtk: repo(["ruby/mmtk", "main"], [ + ["gc/mmtk", "gc/mmtk"], + ]), + open3: lib("ruby/open3", gemspec_in_subdir: true).tap { + it.exclude << "lib/open3/jruby_windows.rb" + }, + openssl: repo("ruby/openssl", [ + ["ext/openssl", "ext/openssl"], + ["lib", "ext/openssl/lib"], + ["test/openssl", "test/openssl"], + ["sample", "sample/openssl"], + ["openssl.gemspec", "ext/openssl/openssl.gemspec"], + ["History.md", "ext/openssl/History.md"], + ], exclude: [ + "test/openssl/envutil.rb", + "ext/openssl/depend", + ]), + optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap { + it.mappings << ["doc/optparse", "doc/optparse"] + }, + pathname: repo("ruby/pathname", [ + ["ext/pathname/pathname.c", "pathname.c"], + ["lib/pathname_builtin.rb", "pathname_builtin.rb"], + ["lib/pathname.rb", "lib/pathname.rb"], + ["test/pathname", "test/pathname"], + ]), + pp: lib("ruby/pp"), + prettyprint: lib("ruby/prettyprint"), + prism: repo(["ruby/prism", "main"], [ + ["ext/prism", "prism"], + ["lib/prism.rb", "lib/prism.rb"], + ["lib/prism", "lib/prism"], + ["test/prism", "test/prism"], + ["src", "prism"], + ["prism.gemspec", "lib/prism/prism.gemspec"], + ["include/prism", "prism"], + ["include/prism.h", "prism/prism.h"], + ["config.yml", "prism/config.yml"], + ["templates", "prism/templates"], + ], exclude: [ + "prism/templates/{javascript,java,rbi,sig}", + "test/prism/snapshots_test.rb", + "test/prism/snapshots", + "prism/extconf.rb", + "prism/srcs.mk*", + ]), + psych: repo("ruby/psych", [ + ["ext/psych", "ext/psych"], + ["lib", "ext/psych/lib"], + ["test/psych", "test/psych"], + ["psych.gemspec", "ext/psych/psych.gemspec"], + ], exclude: [ + "ext/psych/lib/org", + "ext/psych/lib/psych.jar", + "ext/psych/lib/psych_jars.rb", + "ext/psych/lib/psych.{bundle,so}", + "ext/psych/lib/2.*", + "ext/psych/yaml/LICENSE", + "ext/psych/.gitignore", + ]), + resolv: repo("ruby/resolv", [ + ["lib/resolv.rb", "lib/resolv.rb"], + ["test/resolv", "test/resolv"], + ["resolv.gemspec", "lib/resolv.gemspec"], + ["ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32/resolv.rb"], + ["ext/win32/resolv", "ext/win32/resolv"], + ]), + rubygems: repo("ruby/rubygems", [ + ["lib/rubygems.rb", "lib/rubygems.rb"], + ["lib/rubygems", "lib/rubygems"], + ["test/rubygems", "test/rubygems"], + ["bundler/lib/bundler.rb", "lib/bundler.rb"], + ["bundler/lib/bundler", "lib/bundler"], + ["bundler/exe/bundle", "libexec/bundle"], + ["bundler/exe/bundler", "libexec/bundler"], + ["bundler/bundler.gemspec", "lib/bundler/bundler.gemspec"], + ["bundler/spec", "spec/bundler"], + *["bundle", "parallel_rspec", "rspec"].map {|binstub| + ["bundler/bin/#{binstub}", "spec/bin/#{binstub}"] + }, + *%w[dev_gems test_gems rubocop_gems standard_gems].flat_map {|gemfile| + ["rb.lock", "rb"].map do |ext| + ["tool/bundler/#{gemfile}.#{ext}", "tool/bundler/#{gemfile}.#{ext}"] + end + }, + ], exclude: [ + "spec/bundler/bin", + "spec/bundler/support/artifice/vcr_cassettes", + "spec/bundler/support/artifice/used_cassettes.txt", + "lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}", + ]), + securerandom: lib("ruby/securerandom"), + shellwords: lib("ruby/shellwords"), + singleton: lib("ruby/singleton"), + stringio: repo("ruby/stringio", [ + ["ext/stringio", "ext/stringio"], + ["test/stringio", "test/stringio"], + ["stringio.gemspec", "ext/stringio/stringio.gemspec"], + ["doc/stringio", "doc/stringio"], + ], exclude: [ + "ext/stringio/README.md", + ]), + strscan: repo("ruby/strscan", [ + ["ext/strscan", "ext/strscan"], + ["lib", "ext/strscan/lib"], + ["test/strscan", "test/strscan"], + ["strscan.gemspec", "ext/strscan/strscan.gemspec"], + ["doc/strscan", "doc/strscan"], + ], exclude: [ + "ext/strscan/regenc.h", + "ext/strscan/regint.h", + ]), + syntax_suggest: lib(["ruby/syntax_suggest", "main"], gemspec_in_subdir: true), + tempfile: lib("ruby/tempfile"), + time: lib("ruby/time"), + timeout: lib("ruby/timeout"), + tmpdir: lib("ruby/tmpdir"), + un: lib("ruby/un"), + uri: lib("ruby/uri", gemspec_in_subdir: true), + weakref: lib("ruby/weakref"), + yaml: lib("ruby/yaml", gemspec_in_subdir: true), + zlib: repo("ruby/zlib", [ + ["ext/zlib", "ext/zlib"], + ["test/zlib", "test/zlib"], + ["zlib.gemspec", "ext/zlib/zlib.gemspec"], + ]), + }.transform_keys(&:to_s) + + class << Repository + def find_upstream(file) + return if NO_UPSTREAM.any? {|dst| file.start_with?(dst) } + REPOSITORIES.find do |repo_name, repository| + if repository.mappings.any? {|_src, dst| file.start_with?(dst) } + break repo_name + end + end + end + + def group(files) + files.group_by {|file| find_upstream(file)} + end + end + + # Allow synchronizing commits up to this FETCH_DEPTH. We've historically merged PRs + # with about 250 commits to ruby/ruby, so we use this depth for ruby/ruby in general. + FETCH_DEPTH = 500 + + def pipe_readlines(args, rs: "\0", chomp: true) + IO.popen(args) do |f| + f.readlines(rs, chomp: chomp) + end + end + + def porcelain_status(*pattern) + pipe_readlines(%W"git status --porcelain --no-renames -z --" + pattern) + end + + def replace_rdoc_ref(file) + src = File.binread(file) + changed = false + changed |= src.gsub!(%r[\[\Khttps://docs\.ruby-lang\.org/en/master(?:/doc)?/(([A-Z]\w+(?:/[A-Z]\w+)*)|\w+_rdoc)\.html(\#\S+)?(?=\])]) do + name, mod, label = $1, $2, $3 + mod &&= mod.gsub('/', '::') + if label && (m = label.match(/\A\#(?:method-([ci])|(?:(?:class|module)-#{mod}-)?label)-([-+\w]+)\z/)) + scope, label = m[1], m[2] + scope = scope ? scope.tr('ci', '.#') : '@' + end + "rdoc-ref:#{mod || name.chomp("_rdoc") + ".rdoc"}#{scope}#{label}" + end + changed or return false + File.binwrite(file, src) + return true + end + + def replace_rdoc_ref_all + result = porcelain_status("*.c", "*.rb", "*.rdoc") + result.map! {|line| line[/\A.M (.*)/, 1]} + result.compact! + return if result.empty? + result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + result) + result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def replace_rdoc_ref_all_full + Dir.glob("**/*.{c,rb,rdoc}").inject(false) {|changed, file| changed | replace_rdoc_ref(file)} + end + + def rubygems_do_fixup + gemspec_content = File.readlines("lib/bundler/bundler.gemspec").map do |line| + next if line =~ /LICENSE\.md/ + + line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec") + end.compact.join + File.write("lib/bundler/bundler.gemspec", gemspec_content) + + ["bundle", "parallel_rspec", "rspec"].each do |binstub| + path = "spec/bin/#{binstub}" + next unless File.exist?(path) + content = File.read(path).gsub("../spec", "../bundler") + File.write(path, content) + chmod("+x", path) + end + end + + # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. + def sync_default_gems(gem) + config = REPOSITORIES[gem] + puts "Sync #{config.upstream}" + + upstream = File.join("..", "..", config.upstream) + + config.mappings.each do |src, dst| + rm_rf(dst) + end + + copied = Set.new + config.mappings.each do |src, dst| + prefix = File.join(upstream, src) + # Maybe mapping needs to be updated? + next unless File.exist?(prefix) + Find.find(prefix) do |path| + next if File.directory?(path) + if copied.add?(path) + newpath = config.rewrite_for_ruby(path.sub(%r{\A#{Regexp.escape(upstream)}/}, "")) + next unless newpath + mkdir_p(File.dirname(newpath)) + cp(path, newpath) + end + end + end + + porcelain_status().each do |line| + /\A(?:.)(?:.) (?<path>.*)\z/ =~ line or raise + if config.excluded?(path) + puts "Restoring excluded file: #{path}" + IO.popen(%W"git checkout --" + [path], "rb", &:read) + end + end + + # RubyGems/Bundler needs special care + if gem == "rubygems" + rubygems_do_fixup + end + + check_prerelease_version(gem) + + # Architecture-dependent files must not pollute libdir. + rm_rf(Dir["lib/**/*.#{RbConfig::CONFIG['DLEXT']}"]) + replace_rdoc_ref_all + end + + def check_prerelease_version(gem) + return if ["rubygems", "mmtk", "cgi", "pathname", "Onigmo"].include?(gem) + + require "net/https" + require "json" + require "uri" + + uri = URI("https://rubygems.org/api/v1/versions/#{gem.downcase}/latest.json") + response = Net::HTTP.get(uri) + latest_version = JSON.parse(response)["version"] + + gemspec = [ + "lib/#{gem}/#{gem}.gemspec", + "lib/#{gem}.gemspec", + "ext/#{gem}/#{gem}.gemspec", + "ext/#{gem.split("-").join("/")}/#{gem}.gemspec", + "lib/#{gem.split("-").first}/#{gem}.gemspec", + "ext/#{gem.split("-").first}/#{gem}.gemspec", + "lib/#{gem.split("-").join("/")}/#{gem}.gemspec", + ].find{|gemspec| File.exist?(gemspec)} + spec = Gem::Specification.load(gemspec) + puts "#{gem}-#{spec.version} is not latest version of rubygems.org" if spec.version.to_s != latest_version + end + + def message_filter(repo, sha, log, context: nil) + unless repo.count("/") == 1 and /\A\S+\z/ =~ repo + raise ArgumentError, "invalid repository: #{repo}" + end + unless /\A\h{10,40}\z/ =~ sha + raise ArgumentError, "invalid commit-hash: #{sha}" + end + repo_url = "https://github.com/#{repo}" + + # Log messages generated by GitHub web UI have inconsistent line endings + log = log.delete("\r") + log << "\n" if !log.end_with?("\n") + + # Split the subject from the log message according to git conventions. + # SPECIAL TREAT: when the first line ends with a dot `.` (which is not + # obeying the conventions too), takes only that line. + subject, log = log.split(/\A.+\.\K\n(?=\S)|\n(?:[ \t]*(?:\n|\z))/, 2) + conv = proc do |s| + mod = true if s.gsub!(/\b(?:(?i:fix(?:e[sd])?|close[sd]?|resolve[sd]?) +)\K#(?=\d+\b)|\bGH-#?(?=\d+\b)|\(\K#(?=\d+\))/) { + "#{repo_url}/pull/" + } + mod |= true if s.gsub!(%r{(?<![-\[\](){}\w@/])(?:(\w+(?:-\w+)*/\w+(?:-\w+)*)@)?(\h{10,40})\b}) {|c| + "https://github.com/#{$1 || repo}/commit/#{$2[0,12]}" + } + mod + end + subject = "[#{repo}] #{subject}" + subject.gsub!(/\s*\n\s*/, " ") + if conv[subject] + if subject.size > 68 + subject.gsub!(/\G.{,67}[^\s.,][.,]*\K\s+/, "\n") + end + end + commit_url = "#{repo_url}/commit/#{sha[0,10]}\n" + sync_note = context ? "#{commit_url}\n#{context}" : commit_url + if log and !log.empty? + log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last + conv[log] + log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) { + ($~.begin(1) ? "" : "\n\n") + sync_note + ($~.begin(2) ? "\n" : "") + } + else + log = sync_note + end + "#{subject}\n\n#{log}" + end + + def log_format(format, args, &block) + IO.popen(%W[git -c core.autocrlf=false -c core.eol=lf + log --no-show-signature --format=#{format}] + args, "rb", &block) + end + + def commits_in_range(upto, exclude, toplevel:) + args = [upto, *exclude.map {|s|"^#{s}"}] + log_format('%H,%P,%s', %W"--first-parent" + args) do |f| + f.read.split("\n").reverse.flat_map {|commit| + hash, parents, subject = commit.split(',', 3) + parents = parents.split + + # Non-merge commit + if parents.size <= 1 + puts "#{hash} #{subject}" + next [[hash, subject]] + end + + # Clean 2-parent merge commit: follow the other parent as long as it + # contains no potentially-non-clean merges + if parents.size == 2 && + IO.popen(%W"git diff-tree --remerge-diff #{hash}", "rb", &:read).empty? + puts "\e[2mChecking the other parent of #{hash} #{subject}\e[0m" + ret = catch(:quit) { + commits_in_range(parents[1], exclude + [parents[0]], toplevel: false) + } + next ret if ret + end + + unless toplevel + puts "\e[1mMerge commit with possible conflict resolution #{hash} #{subject}\e[0m" + throw :quit + end + + puts "#{hash} #{subject} " \ + "\e[1m[merge commit with possible conflicts, will do a squash merge]\e[0m" + [[hash, subject]] + } + end + end + + # Returns commit list as array of [commit_hash, subject, sync_note]. + def commits_in_ranges(ranges) + ranges.flat_map do |range| + exclude, upto = range.include?("..") ? range.split("..", 2) : ["#{range}~1", range] + puts "Looking for commits in range #{exclude}..#{upto}" + commits_in_range(upto, exclude.empty? ? [] : [exclude], toplevel: true) + end.uniq + end + + #-- + # Following methods used by sync_default_gems_with_commits return + # true: success + # false: skipped + # nil: failed + #++ + + def resolve_conflicts(gem, sha, edit) + # Discover unmerged files: any unstaged changes + changes = porcelain_status() + conflict = changes.grep(/\A(?:.[^ ?]) /) {$'} + # If -e option is given, open each conflicted file with an editor + unless conflict.empty? + if edit + case + when (editor = ENV["GIT_EDITOR"] and !editor.empty?) + when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) + end + if editor + system([editor, conflict].join(' ')) + conflict.delete_if {|f| !File.exist?(f)} + return true if conflict.empty? + return system(*%w"git add --", *conflict) + end + end + return false + end + + return true + end + + def collect_cacheinfo(tree) + pipe_readlines(%W"git ls-tree -r -t -z #{tree}").filter_map do |line| + fields, path = line.split("\t", 2) + mode, type, object = fields.split(" ", 3) + next unless type == "blob" + [mode, type, object, path] + end + end + + def rewrite_cacheinfo(gem, blobs) + config = REPOSITORIES[gem] + rewritten = [] + ignored = blobs.dup + ignored.delete_if do |mode, type, object, path| + newpath = config.rewrite_for_ruby(path) + next unless newpath + rewritten << [mode, type, object, newpath] + end + [rewritten, ignored] + end + + def make_commit_info(gem, sha) + config = REPOSITORIES[gem] + headers, orig = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + /^author (?<author_name>.+?) <(?<author_email>.*?)> (?<author_date>.+?)$/ =~ headers or + raise "unable to parse author info for commit #{sha}" + author = { + "GIT_AUTHOR_NAME" => author_name, + "GIT_AUTHOR_EMAIL" => author_email, + "GIT_AUTHOR_DATE" => author_date, + } + context = nil + if /^parent (?<first_parent>.{40})\nparent .{40}$/ =~ headers + # Squashing a merge commit: keep authorship information + context = IO.popen(%W"git shortlog #{first_parent}..#{sha} --", "rb", &:read) + end + message = message_filter(config.upstream, sha, orig, context: context) + [author, message] + end + + def fixup_commit(gem, commit) + wt = File.join("tmp", "sync_default_gems-fixup-worktree") + if File.directory?(wt) + IO.popen(%W"git -C #{wt} clean -xdf", "rb", &:read) + IO.popen(%W"git -C #{wt} reset --hard #{commit}", "rb", &:read) + else + IO.popen(%W"git worktree remove --force #{wt}", "rb", err: File::NULL, &:read) + IO.popen(%W"git worktree add --detach #{wt} #{commit}", "rb", &:read) + end + raise "git worktree prepare failed for commit #{commit}" unless $?.success? + + Dir.chdir(wt) do + if gem == "rubygems" + rubygems_do_fixup + end + replace_rdoc_ref_all_full + end + + IO.popen(%W"git -C #{wt} add -u", "rb", &:read) + IO.popen(%W"git -C #{wt} commit --amend --no-edit", "rb", &:read) + IO.popen(%W"git -C #{wt} rev-parse HEAD", "rb", &:read).chomp + end + + def make_and_fixup_commit(gem, original_commit, cacheinfo, parent: nil, message: nil, author: nil) + tree = Tempfile.create("sync_default_gems-#{gem}-index") do |f| + File.unlink(f.path) + IO.popen({"GIT_INDEX_FILE" => f.path}, + %W"git update-index --index-info", "wb", out: IO::NULL) do |io| + cacheinfo.each do |mode, type, object, path| + io.puts("#{mode} #{type} #{object}\t#{path}") + end + end + raise "git update-index failed" unless $?.success? + + IO.popen({"GIT_INDEX_FILE" => f.path}, %W"git write-tree --missing-ok", "rb", &:read).chomp + end + + args = ["-m", message || "Rewriten commit for #{original_commit}"] + args += ["-p", parent] if parent + commit = IO.popen({**author}, %W"git commit-tree #{tree}" + args, "rb", &:read).chomp + + # Apply changes that require a working tree + commit = fixup_commit(gem, commit) + + commit + end + + def rewrite_commit(gem, sha) + author, message = make_commit_info(gem, sha) + new_blobs = collect_cacheinfo("#{sha}") + new_rewritten, new_ignored = rewrite_cacheinfo(gem, new_blobs) + + headers, _ = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2) + first_parent = headers[/^parent (.{40})$/, 1] + unless first_parent + # Root commit, first time to sync this repo + return make_and_fixup_commit(gem, sha, new_rewritten, message: message, author: author) + end + + old_blobs = collect_cacheinfo(first_parent) + old_rewritten, old_ignored = rewrite_cacheinfo(gem, old_blobs) + if old_ignored != new_ignored + paths = (old_ignored + new_ignored - (old_ignored & new_ignored)) + .map {|*_, path| path}.uniq + puts "\e\[1mIgnoring file changes not in mappings: #{paths.join(" ")}\e\[0m" + end + changed_paths = (old_rewritten + new_rewritten - (old_rewritten & new_rewritten)) + .map {|*_, path| path}.uniq + if changed_paths.empty? + puts "Skip commit only for tools or toplevel" + return false + end + + # Build commit objects from "cacheinfo" + new_parent = make_and_fixup_commit(gem, first_parent, old_rewritten) + new_commit = make_and_fixup_commit(gem, sha, new_rewritten, parent: new_parent, message: message, author: author) + puts "Created a temporary commit for cherry-pick: #{new_commit}" + new_commit + end + + def pickup_commit(gem, sha, edit) + rewritten = rewrite_commit(gem, sha) + + # No changes remaining after rewriting + return false unless rewritten + + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{rewritten}", "rb", err: [:child, :out], &:read) + unless $?.success? + if result =~ /The previous cherry-pick is now empty/ + system(*%w"git cherry-pick --skip") + puts "Skip empty commit #{sha}" + return false + end + + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless resolve_conflicts(gem, sha, edit) + system(*%w"git --no-pager diff") if !edit # If failed, show `git diff` unless editing + `git reset` && `git checkout .` && `git clean -fd` # Clean up un-committed diffs + return nil # Fail unless cherry-picked + end + + # Commit cherry-picked commit + if porcelain_status().empty? + system(*%w"git cherry-pick --skip") + return false + else + system(*%w"git cherry-pick --continue --no-edit") + return nil unless $?.success? + end + end + + new_head = IO.popen(%W"git rev-parse HEAD", "rb", &:read).chomp + puts "Committed cherry-pick as #{new_head}" + return true + end + + # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. + # @param ranges [Array<String>, true] "commit", "before..after", or true. Note that it will NOT sync "before" (but commits after that). + # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. + def sync_default_gems_with_commits(gem, ranges, edit: nil) + config = REPOSITORIES[gem] + repo, default_branch = config.upstream, config.branch + puts "Sync #{repo} with commit history." + + # Fetch the repository to be synchronized + IO.popen(%W"git remote") do |f| + unless f.read.split.include?(gem) + `git remote add #{gem} https://github.com/#{repo}.git` + end + end + system(*%W"git fetch --no-tags --depth=#{FETCH_DEPTH} #{gem} #{default_branch}") + + # If -a is given, discover all commits since the last picked commit + if ranges == true + pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" + log = log_format('%B', %W"-E --grep=#{pattern} -n1 --", &:read) + ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] + end + commits = commits_in_ranges(ranges) + if commits.empty? + puts "No commits to pick" + return true + end + + failed_commits = [] + commits.each do |sha, subject| + puts "----" + puts "Pick #{sha} #{subject}" + case pickup_commit(gem, sha, edit) + when false + # skipped + when nil + failed_commits << [sha, subject] + end + end + + unless failed_commits.empty? + puts "---- failed commits ----" + failed_commits.each do |sha, subject| + puts "#{sha} #{subject}" + end + return false + end + return true + end + + def sync_lib(repo, upstream = nil) + unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}") + abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.] + end + rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"]) + cp_r(Dir.glob("#{upstream}/lib/*"), "lib") + tests = if File.directory?("test/#{repo}") + "test/#{repo}" + else + "test/test_#{repo}.rb" + end + cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}") + gemspec = if File.directory?("lib/#{repo}") + "lib/#{repo}/#{repo}.gemspec" + else + "lib/#{repo}.gemspec" + end + cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}") + end + + def update_default_gems(gem, release: false) + config = REPOSITORIES[gem] + author, repository = config.upstream.split('/') + default_branch = config.branch + + puts "Update #{author}/#{repository}" + + unless File.exist?("../../#{author}/#{repository}") + mkdir_p("../../#{author}") + `git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}` + end + + Dir.chdir("../../#{author}/#{repository}") do + unless `git remote`.match(/ruby\-core/) + `git remote add ruby-core git@github.com:ruby/ruby.git` + end + `git fetch ruby-core master --no-tags` + unless `git branch`.match(/ruby\-core/) + `git checkout ruby-core/master` + `git branch ruby-core` + end + `git checkout ruby-core` + `git rebase ruby-core/master` + `git fetch origin --tags` + + if release + last_release = `git tag | sort -V`.chomp.split.delete_if{|v| v =~ /pre|beta/ }.last + `git checkout #{last_release}` + else + `git checkout #{default_branch}` + `git rebase origin/#{default_branch}` + end + end + end + + case ARGV[0] + when "up" + if ARGV[1] + update_default_gems(ARGV[1]) + else + REPOSITORIES.each_key {|gem| update_default_gems(gem)} + end + when "all" + REPOSITORIES.each_key do |gem| + next if ["Onigmo"].include?(gem) + update_default_gems(gem, release: true) if ARGV[1] == "release" + sync_default_gems(gem) + end + when "list" + ARGV.shift + pattern = Regexp.new(ARGV.join('|')) + REPOSITORIES.each do |gem, config| + next unless pattern =~ gem or pattern =~ config.upstream + printf "%-15s https://github.com/%s\n", gem, config.upstream + end + when "rdoc-ref" + ARGV.shift + pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV + result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + pattern) + result.inject(false) do |changed, file| + if replace_rdoc_ref(file) + puts "replaced rdoc-ref in #{file}" + changed = true + end + changed + end + when nil, "-h", "--help" + puts <<-HELP +\e[1mSync with upstream code of default libraries\e[0m + +\e[1mImport all default gems through `git clone` and `cp -rf` (git commits are lost)\e[0m + ruby #$0 all + +\e[1mImport all released version of default gems\e[0m + ruby #$0 all release + +\e[1mImport a default gem with specific gem same as all command\e[0m + ruby #$0 rubygems + +\e[1mPick a single commit from the upstream repository\e[0m + ruby #$0 rubygems 97e9768612 + +\e[1mPick a commit range from the upstream repository\e[0m + ruby #$0 rubygems 97e9768612..9e53702832 + +\e[1mPick all commits since the last picked commit\e[0m + ruby #$0 -a rubygems + +\e[1mUpdate repositories of default gems\e[0m + ruby #$0 up + +\e[1mList known libraries\e[0m + ruby #$0 list + +\e[1mList known libraries matching with patterns\e[0m + ruby #$0 list read + HELP + + exit + else + while /\A-/ =~ ARGV[0] + case ARGV[0] + when "-e" + edit = true + ARGV.shift + when "-a" + auto = true + ARGV.shift + else + $stderr.puts "Unknown command line option: #{ARGV[0]}" + exit 1 + end + end + gem = ARGV.shift + if ARGV[0] + exit sync_default_gems_with_commits(gem, ARGV, edit: edit) + elsif auto + exit sync_default_gems_with_commits(gem, true, edit: edit) + else + sync_default_gems(gem) + end + end if $0 == __FILE__ +end diff --git a/tool/test-annocheck.sh b/tool/test-annocheck.sh new file mode 100755 index 0000000000..6896869e07 --- /dev/null +++ b/tool/test-annocheck.sh @@ -0,0 +1,40 @@ +#!/bin/sh -eu +# Run the `tool/test-annocheck.sh [binary files]` to check security issues +# by annocheck <https://sourceware.org/annobin/>. +# +# E.g. `tool/test-annocheck.sh ruby libruby.so.3.2.0`. +# +# Note that as the annocheck binary package is not available on Ubuntu, and it +# is working in progress in Debian, this script uses Fedora container for now. +# It requires docker or podman. +# https://www.debian.org/devel/wnpp/itp.en.html +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=926470 + +set -x + +DOCKER="$(command -v docker || command -v podman)" +TAG=ruby-fedora-annocheck +TOOL_DIR="$(dirname "${0}")" +TMP_DIR="tmp/annocheck" +DOCKER_RUN_VOLUME_OPTS= + +if [ -z "${CI-}" ]; then + # Use a volume option on local (non-CI). + DOCKER_RUN_VOLUME_OPTS="-v $(pwd):/work" + "${DOCKER}" build --rm -t "${TAG}" ${TOOL_DIR}/annocheck/ +else + # TODO: A temporary workaround on CI to build by copying binary files from + # host to container without volume option, as I couldn't find a way to use + # volume in container in container on GitHub Actions + # <.github/workflows/compilers.yml>. + TAG="${TAG}-copy" + rm -rf "${TMP_DIR}" + mkdir -p "${TMP_DIR}" + for file in "${@}"; do + cp -p "${file}" "${TMP_DIR}" + done + "${DOCKER}" build --rm -t "${TAG}" --build-arg=IN_DIR="${TMP_DIR}" -f ${TOOL_DIR}/annocheck/Dockerfile-copy . + rm -rf "${TMP_DIR}" +fi + +"${DOCKER}" run --rm -t ${DOCKER_RUN_VOLUME_OPTS} "${TAG}" annocheck --verbose ${TEST_ANNOCHECK_OPTS-} "${@}" diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb new file mode 100644 index 0000000000..778fe3311a --- /dev/null +++ b/tool/test-bundled-gems.rb @@ -0,0 +1,140 @@ +require 'rbconfig' +require 'timeout' +require 'fileutils' +require 'shellwords' +require_relative 'lib/colorize' +require_relative 'lib/gem_env' + +ENV.delete("GNUMAKEFLAGS") + +github_actions = ENV["GITHUB_ACTIONS"] == "true" + +DEFAULT_ALLOWED_FAILURES = RUBY_PLATFORM =~ /mswin|mingw/ ? [ + 'debug', + 'irb', + 'csv', +] : [] +allowed_failures = ENV['TEST_BUNDLED_GEMS_ALLOW_FAILURES'] || '' +allowed_failures = allowed_failures.split(',').concat(DEFAULT_ALLOWED_FAILURES).uniq.reject(&:empty?) + +# make test-bundled-gems BUNDLED_GEMS=gem1,gem2,gem3 +bundled_gems = nil if (bundled_gems = ARGV.first&.split(","))&.empty? + +colorize = Colorize.new +rake = File.realpath("../../.bundle/bin/rake", __FILE__) +gem_dir = File.realpath('../../gems', __FILE__) +rubylib = [gem_dir+'/lib', ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) +run_opts = ENV["RUN_OPTS"]&.shellsplit +exit_code = 0 +ruby = ENV['RUBY'] || RbConfig.ruby +failed = [] +File.foreach("#{gem_dir}/bundled_gems") do |line| + next unless gem = line[/^[^\s\#]+/] + next if bundled_gems&.none? {|pat| File.fnmatch?(pat, gem)} + next unless File.directory?("#{gem_dir}/src/#{gem}/test") + + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", rake, "test"] + first_timeout = 600 # 10min + + toplib = gem + unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + toplib = gem.tr("-", "/") + next unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") + end + + case gem + when "rbs" + # TODO: We should skip test file instead of test class/methods + skip_test_files = %w[ + ] + + skip_test_files.each do |file| + path = "#{gem_dir}/src/#{gem}/#{file}" + File.unlink(path) if File.exist?(path) + end + + rbs_skip_tests = [ + File.join(__dir__, "/rbs_skip_tests") + ] + + if /mswin|mingw/ =~ RUBY_PLATFORM + rbs_skip_tests << File.join(__dir__, "/rbs_skip_tests_windows") + end + + test_command.concat %W[stdlib_test validate RBS_SKIP_TESTS=#{rbs_skip_tests.join(File::PATH_SEPARATOR)} SKIP_RBS_VALIDATION=true] + first_timeout *= 3 + + when "debug" + # Since debug gem requires debug.so in child processes without + # activating the gem, we preset necessary paths in RUBYLIB + # environment variable. + load_path = true + + when "test-unit" + test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", "test/run.rb"] + + when "csv" + first_timeout = 30 + + when "win32ole" + next unless /mswin|mingw/ =~ RUBY_PLATFORM + + end + + if load_path + libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) + next unless $?.success? + ENV["RUBYLIB"] = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) + else + ENV["RUBYLIB"] = rubylib + end + + # 93(bright yellow) is copied from .github/workflows/mingw.yml + puts "#{github_actions ? "::group::\e\[93m" : "\n"}Testing the #{gem} gem#{github_actions ? "\e\[m" : ""}" + print "[command]" if github_actions + p test_command + timeouts = {nil => first_timeout, INT: 30, TERM: 10, KILL: nil} + if /mingw|mswin/ =~ RUBY_PLATFORM + timeouts.delete(:TERM) # Inner process signal on Windows + group = :new_pgroup + pg = "" + else + group = :pgroup + pg = "-" + end + pid = Process.spawn(*test_command, group => true) + timeouts.each do |sig, sec| + if sig + puts "Sending #{sig} signal" + Process.kill("#{pg}#{sig}", pid) + end + begin + break Timeout.timeout(sec) {Process.wait(pid)} + rescue Timeout::Error + end + rescue Interrupt + exit_code = Signal.list["INT"] + Process.kill("#{pg}KILL", pid) + Process.wait(pid) + break + end + + print "::endgroup::\n" if github_actions + unless $?.success? + + mesg = "Tests failed " + + ($?.signaled? ? "by SIG#{Signal.signame($?.termsig)}" : + "with exit code #{$?.exitstatus}") + puts colorize.decorate(mesg, "fail") + if allowed_failures.include?(gem) + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" + puts colorize.decorate(mesg, "skip") + else + failed << gem + exit_code = 1 + end + end +end + +puts "Failed gems: #{failed.join(', ')}" unless failed.empty? +exit exit_code diff --git a/tool/test-coverage.rb b/tool/test-coverage.rb new file mode 100644 index 0000000000..28ef0bf7f8 --- /dev/null +++ b/tool/test-coverage.rb @@ -0,0 +1,135 @@ +require "coverage" + +Coverage.start(lines: true, branches: true, methods: true) + +TEST_COVERAGE_DATA_FILE = "test-coverage.dat" + +FILTER_PATHS = %w[ + lib/bundler/vendor + lib/rubygems/resolver/molinillo + lib/rubygems/tsort + lib/rubygems/optparse + tool + test + spec +] + +def merge_coverage_data(res1, res2) + res1.each do |path, cov1| + cov2 = res2[path] + if cov2 + cov1[:lines].each_with_index do |count1, i| + next unless count1 + add_count(cov2[:lines], i, count1) + end + cov1[:branches].each do |base_key, targets1| + if cov2[:branches][base_key] + targets1.each do |target_key, count1| + add_count(cov2[:branches][base_key], target_key, count1) + end + else + cov2[:branches][base_key] = targets1 + end + end + cov1[:methods].each do |key, count1| + add_count(cov2[:methods], key, count1) + end + else + res2[path] = cov1 + end + end + res2 +end + +def add_count(h, key, count) + if h[key] + h[key] += count + else + h[key] = count + end +end + +def save_coverage_data(res1) + res1.each do |_path, cov| + if cov[:methods] + h = {} + cov[:methods].each do |(klass, *key), count| + h[[klass.name, *key]] = count + end + cov[:methods].replace h + end + end + File.open(TEST_COVERAGE_DATA_FILE, File::RDWR | File::CREAT | File::BINARY) do |f| + f.flock(File::LOCK_EX) + s = f.read + res2 = s.size > 0 ? Marshal.load(s) : {} + res1 = merge_coverage_data(res1, res2) + f.rewind + f << Marshal.dump(res2) + f.flush + f.truncate(f.pos) + end +end + +def invoke_simplecov_formatter + # XXX docile-x.y.z and simplecov-x.y.z, simplecov-html-x.y.z, simplecov_json_formatter-x.y.z + %w[simplecov simplecov-html simplecov_json_formatter docile].each do |f| + Dir.glob("#{__dir__}/../.bundle/gems/#{f}-*/lib").each do |d| + $LOAD_PATH.unshift d + end + end + + require "simplecov" + res = Marshal.load(File.binread(TEST_COVERAGE_DATA_FILE)) + simplecov_result = {} + base_dir = File.dirname(__dir__) + cur_dir = Dir.pwd + + res.each do |path, cov| + next unless path.start_with?(base_dir) || path.start_with?(cur_dir) + next if FILTER_PATHS.any? {|dir| path.start_with?(File.join(base_dir, dir))} + simplecov_result[path] = cov + end + + a, b = base_dir, cur_dir + until a == b + if a.size > b.size + a = File.dirname(a) + else + b = File.dirname(b) + end + end + root_dir = a + + SimpleCov.configure do + root(root_dir) + coverage_dir(File.join(cur_dir, "coverage")) + end + res = SimpleCov::Result.new(simplecov_result) + res.command_name = "Ruby's `make test-all`" + SimpleCov::Formatter::HTMLFormatter.new.format(res) +end + +pid = $$ +pwd = Dir.pwd + +at_exit do + # Some tests leave GC.stress enabled, causing slow coverage processing. + # Reset it here to avoid performance issues. + GC.stress = false + + exit_exc = $! + + Dir.chdir(pwd) do + save_coverage_data(Coverage.result) + if pid == $$ + begin + nil while Process.waitpid(-1) + rescue Errno::ECHILD + invoke_simplecov_formatter + end + end + end + + raise exit_exc if exit_exc +end diff --git a/tool/test/init.rb b/tool/test/init.rb new file mode 100644 index 0000000000..3fd1419a9c --- /dev/null +++ b/tool/test/init.rb @@ -0,0 +1,26 @@ +# This file includes the settings for "make test-all" and "make test-tool". +# Note that this file is loaded not only by test/runner.rb but also by tool/lib/test/unit/parallel.rb. + +# Prevent test-all from using bundled gems +["GEM_HOME", "GEM_PATH"].each do |gem_env| + # Preserve the gem environment prepared by tool/runruby.rb for test-tool, which uses bundled gems. + ENV["BUNDLED_#{gem_env}"] = ENV[gem_env] + + ENV[gem_env] = "".freeze +end +ENV["GEM_SKIP"] = "".freeze + +ENV.delete("RUBY_CODESIGN") + +Warning[:experimental] = false + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) + +require 'test/unit' + +require "profile_test_all" if ENV.key?('RUBY_TEST_ALL_PROFILE') +require "tracepointchecker" +require "zombie_hunter" +require "iseq_loader_checker" +require "gc_checker" +require_relative "../test-coverage.rb" if ENV.key?('COVERAGE') diff --git a/tool/test/runner.rb b/tool/test/runner.rb new file mode 100644 index 0000000000..9001fc2d06 --- /dev/null +++ b/tool/test/runner.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true +require 'rbconfig' + +require_relative "init" + +case $0 +when __FILE__ + dir = __dir__ +when "-e" + # No default directory +else + dir = File.realdirpath("..", $0) +end +exit Test::Unit::AutoRunner.run(true, dir) diff --git a/tool/test/test_commit_email.rb b/tool/test/test_commit_email.rb new file mode 100644 index 0000000000..db441584fd --- /dev/null +++ b/tool/test/test_commit_email.rb @@ -0,0 +1,102 @@ +require 'test/unit' +require 'shellwords' +require 'tmpdir' +require 'fileutils' +require 'open3' + +class TestCommitEmail < Test::Unit::TestCase + STDIN_DELIMITER = "---\n" + + def setup + omit 'git command is not available' unless system('git', '--version', out: File::NULL, err: File::NULL) + + @ruby = Dir.mktmpdir + Dir.chdir(@ruby) do + git('init', '--initial-branch=master') + git('config', 'user.name', 'Jóhän Grübél') + git('config', 'user.email', 'johan@example.com') + env = { + 'GIT_AUTHOR_DATE' => '2025-10-08T12:00:00Z', + 'GIT_CONFIG_GLOBAL' => @ruby + "/gitconfig", + 'TZ' => 'UTC', + } + git('commit', '--allow-empty', '-m', 'New repository initialized by cvs2svn.', env:) + git('commit', '--allow-empty', '-m', 'Initial revision', env:) + git('commit', '--allow-empty', '-m', 'version 1.0.0', env:) + end + + @sendmail = File.join(Dir.mktmpdir, 'sendmail') + File.write(@sendmail, <<~SENDMAIL, mode: "wx", perm: 0755) + #!/bin/sh + echo #{STDIN_DELIMITER.chomp.dump} + exec cat + SENDMAIL + + @commit_email = File.expand_path('../../tool/commit-email.rb', __dir__) + end + + def teardown + # Clean up temporary files if #setup was not omitted + if @sendmail + File.unlink(@sendmail) + Dir.rmdir(File.dirname(@sendmail)) + end + if @ruby + FileUtils.rm_rf(@ruby) + end + end + + def test_sendmail_encoding + omit 'the sendmail script does not work on windows' if windows? + + Dir.chdir(@ruby) do + before_rev = git('rev-parse', 'HEAD^').chomp + after_rev = git('rev-parse', 'HEAD').chomp + short_rev = after_rev[0...10] + + out, _, status = EnvUtil.invoke_ruby([ + { 'SENDMAIL' => @sendmail, 'TZ' => 'UTC' }.merge!(gem_env), + @commit_email, './', 'cvs-admin@ruby-lang.org', + before_rev, after_rev, 'refs/heads/master', + '--viewer-uri', 'https://github.com/ruby/ruby/commit/', + '--error-to', 'cvs-admin@ruby-lang.org', + ], '', true) + stdin = out.b.split(STDIN_DELIMITER.b, 2).last.force_encoding('UTF-8') + + assert_true(status.success?) + assert_equal(stdin, <<~EOS) + Mime-Version: 1.0 + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + From: =?UTF-8?B?SsOzaMOkbiBHcsO8YsOpbA==?= <noreply@ruby-lang.org> + To: cvs-admin@ruby-lang.org + Subject: #{short_rev} (master): =?UTF-8?B?dmVyc2lvbuOAgDEuMC4w?= + J=C3=B3h=C3=A4n Gr=C3=BCb=C3=A9l\t2025-10-08 12:00:00 +0000 (Wed, 08 Oct 2= + 025) + + New Revision: #{short_rev} + + https://github.com/ruby/ruby/commit/#{short_rev} + + Log: + version=E3=80=801.0.0= + EOS + end + end + + private + + # Resurrect the gem environment preserved by tool/test/init.rb. + # This should work as long as you have run `make up` or `make install`. + def gem_env + { 'GEM_PATH' => ENV['BUNDLED_GEM_PATH'], 'GEM_HOME' => ENV['BUNDLED_GEM_HOME'] } + end + + def git(*cmd, env: {}) + out, status = Open3.capture2(env, 'git', *cmd) + unless status.success? + raise "git #{cmd.shelljoin}\n#{out}" + end + out + end +end diff --git a/tool/test/test_jisx0208.rb b/tool/test/test_jisx0208.rb index d323c84745..98f216ff4f 100644 --- a/tool/test/test_jisx0208.rb +++ b/tool/test/test_jisx0208.rb @@ -1,6 +1,6 @@ require 'test/unit' -require '../tool/jisx0208' +require_relative '../lib/jisx0208' class Test_JISX0208_Char < Test::Unit::TestCase def test_create_with_row_cell diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb new file mode 100755 index 0000000000..252687f3f3 --- /dev/null +++ b/tool/test/test_sync_default_gems.rb @@ -0,0 +1,373 @@ +#!/usr/bin/ruby +require 'test/unit' +require 'stringio' +require 'tmpdir' +require 'rubygems/version' +require_relative '../sync_default_gems' + +module Test_SyncDefaultGems + class TestMessageFilter < Test::Unit::TestCase + def assert_message_filter(expected, trailers, input, repo = "ruby/test", sha = "0123456789") + subject, *expected = expected + expected = [ + "[#{repo}] #{subject}\n", + *expected.map {_1+"\n"}, + "\n", + "https://github.com/#{repo}/commit/#{sha[0, 10]}\n", + ] + if trailers + expected << "\n" + expected.concat(trailers.map {_1+"\n"}) + end + + out = SyncDefaultGems.message_filter(repo, sha, input) + assert_pattern_list(expected, out) + end + + def test_subject_only + expected = [ + "initial commit", + ] + assert_message_filter(expected, nil, "initial commit") + end + + def test_link_in_parenthesis + expected = [ + "fix (https://github.com/ruby/test/pull/1)", + ] + assert_message_filter(expected, nil, "fix (#1)") + end + + def test_co_authored_by + expected = [ + "commit something", + ] + trailers = [ + "Co-Authored-By: git <git@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers, ""].join("\n")) + end + + def test_multiple_co_authored_by + expected = [ + "many commits", + ] + trailers = [ + "Co-authored-by: git <git@ruby-lang.org>", + "Co-authored-by: svn <svn@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers, ""].join("\n")) + end + + def test_co_authored_by_no_newline + expected = [ + "commit something", + ] + trailers = [ + "Co-Authored-By: git <git@ruby-lang.org>", + ] + assert_message_filter(expected, trailers, [expected, "", trailers].join("\n")) + end + + def test_dot_ending_subject + expected = [ + "subject with a dot.", + "", + "- next body line", + ] + assert_message_filter(expected, nil, [expected[0], expected[2], ""].join("\n")) + end + end + + class TestSyncWithCommits < Test::Unit::TestCase + def setup + super + @target = nil + pend "No git" unless system("git --version", out: IO::NULL) + @testdir = Dir.mktmpdir("sync") + user, email = "Ruby", "test@ruby-lang.org" + @git_config = %W"HOME USER GIT_CONFIG_GLOBAL GNUPGHOME".each_with_object({}) {|k, c| c[k] = ENV[k]} + ENV["HOME"] = @testdir + ENV["USER"] = user + ENV["GNUPGHOME"] = @testdir + '/.gnupg' + expire = EnvUtil.apply_timeout_scale(30).to_i + # Generate a new unprotected key with default parameters that + # expires after 30 seconds. + if @gpgsign = system(*%w"gpg --quiet --batch --passphrase", "", + "--quick-generate-key", email, *%W"default default seconds=#{expire}", + err: IO::NULL) + # Fetch the generated public key. + signingkey = IO.popen(%W"gpg --quiet --list-public-key #{email}", &:read)[/^pub .*\n +\K\h+/] + end + ENV["GIT_CONFIG_GLOBAL"] = @testdir + "/gitconfig" + git(*%W"config --global user.email", email) + git(*%W"config --global user.name", user) + git(*%W"config --global init.defaultBranch default") + if signingkey + git(*%W"config --global user.signingkey", signingkey) + git(*%W"config --global commit.gpgsign true") + git(*%W"config --global gpg.program gpg") + git(*%W"config --global log.showSignature true") + end + @target = "sync-test" + SyncDefaultGems::REPOSITORIES[@target] = SyncDefaultGems.repo( + ["ruby/#{@target}", "default"], + [ + ["lib", "lib"], + ["test", "test"], + ], + exclude: [ + "test/fixtures/*", + ], + ) + @sha = {} + @origdir = Dir.pwd + Dir.chdir(@testdir) + ["src", @target].each do |dir| + git(*%W"init -q #{dir}") + File.write("#{dir}/.gitignore", "*~\n") + Dir.mkdir("#{dir}/lib") + File.write("#{dir}/lib/common.rb", ":ok\n") + Dir.mkdir("#{dir}/.github") + Dir.mkdir("#{dir}/.github/workflows") + File.write("#{dir}/.github/workflows/default.yml", "default:\n") + git(*%W"add .gitignore lib/common.rb .github", chdir: dir) + git(*%W"commit -q -m", "Initialize", chdir: dir) + if dir == "src" + File.write("#{dir}/lib/fine.rb", "return\n") + Dir.mkdir("#{dir}/test") + File.write("#{dir}/test/test_fine.rb", "return\n") + git(*%W"add lib/fine.rb test/test_fine.rb", chdir: dir) + git(*%W"commit -q -m", "Looks fine", chdir: dir) + end + Dir.mkdir("#{dir}/tool") + File.write("#{dir}/tool/ok", "#!/bin/sh\n""echo ok\n") + git(*%W"add tool/ok", chdir: dir) + git(*%W"commit -q -m", "Add tool #{dir}", chdir: dir) + @sha[dir] = top_commit(dir) + end + git(*%W"remote add #{@target} ../#{@target}", chdir: "src") + end + + def teardown + if @target + if @gpgsign + system(*%W"gpgconf --kill all") + end + Dir.chdir(@origdir) + SyncDefaultGems::REPOSITORIES.delete(@target) + ENV.update(@git_config) + FileUtils.rm_rf(@testdir) + end + super + end + + def capture_process_output_to(outputs) + return yield unless outputs&.empty? == false + IO.pipe do |r, w| + orig = outputs.map {|out| out.dup} + outputs.each {|out| out.reopen(w)} + w.close + reader = Thread.start {r.read} + yield + ensure + outputs.each {|out| o = orig.shift; out.reopen(o); o.close} + return reader.value + end + end + + def capture_process_outputs + out = err = nil + synchronize do + out = capture_process_output_to(STDOUT) do + err = capture_process_output_to(STDERR) do + yield + end + end + end + return out, err + end + + def git(*commands, **opts) + system("git", *commands, exception: true, **opts) + end + + def top_commit(dir, format: "%H") + IO.popen(%W[git log --no-show-signature --format=#{format} -1], chdir: dir, &:read)&.chomp + end + + def assert_sync(commits = true, success: true, editor: nil) + result = nil + out = capture_process_output_to([STDOUT, STDERR]) do + Dir.chdir("src") do + orig_editor = ENV["GIT_EDITOR"] + ENV["GIT_EDITOR"] = editor || 'false' + edit = true if editor + + result = SyncDefaultGems.sync_default_gems_with_commits(@target, commits, edit: edit) + ensure + ENV["GIT_EDITOR"] = orig_editor + end + end + assert_equal(success, result, out) + out + end + + def test_sync + File.write("#@target/lib/common.rb", "# OK!\n") + git(*%W"commit -q -m", "OK", "lib/common.rb", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# OK!\n", File.read("src/lib/common.rb")) + log = top_commit("src", format: "%B").lines + assert_equal("[ruby/#@target] OK\n", log.first, out) + assert_match(%r[/ruby/#{@target}/commit/\h+$], log.last, out) + assert_operator(top_commit(@target), :start_with?, log.last[/\h+$/], out) + end + + def test_skip_tool + git(*%W"rm -q tool/ok", chdir: @target) + git(*%W"commit -q -m", "Remove tool", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_test_fixtures + Dir.mkdir("#@target/test") + Dir.mkdir("#@target/test/fixtures") + File.write("#@target/test/fixtures/fixme.rb", "") + git(*%W"add test/fixtures/fixme.rb", chdir: @target) + git(*%W"commit -q -m", "Add fixtures", chdir: @target) + out = assert_sync(["#{@sha[@target]}..#{@target}/default"]) + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_skip_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= NEWS!!!\n") + git(*%W"add --", "docs/NEWS.md", chdir: @target) + File.write("#@target/docs/hello.md", "Hello\n") + git(*%W"add --", "docs/hello.md", chdir: @target) + git(*%W"commit -q -m", "It's a news", chdir: @target) + out = assert_sync() + assert_equal(@sha["src"], top_commit("src"), out) + end + + def test_adding_toplevel + Dir.mkdir("#@target/docs") + File.write("#@target/docs/NEWS.md", "= New library\n") + File.write("#@target/lib/news.rb", "return\n") + git(*%W"add --", "docs/NEWS.md", "lib/news.rb", chdir: @target) + git(*%W"commit -q -m", "New lib", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal "return\n", File.read("src/lib/news.rb") + assert_include top_commit("src", format: "oneline"), "[ruby/#{@target}] New lib" + assert_not_operator File, :exist?, "src/docs" + end + + def test_gitignore + File.write("#@target/.gitignore", "*.bak\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add .github", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", + *%W".gitignore lib/common.rb .github", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("*~\n", File.read("src/.gitignore"), out) + assert_equal("#!/bin/sh\n""echo ok\n", File.read("src/tool/ok"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_gitignore_after_conflict + File.write("src/Gemfile", "# main\n") + git(*%W"add Gemfile", chdir: "src") + git(*%W"commit -q -m", "Add Gemfile", chdir: "src") + File.write("#@target/Gemfile", "# conflict\n", mode: "a") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + File.write("#@target/.github/workflows/main.yml", "# Should not merge\n", mode: "a") + git(*%W"add Gemfile .github lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Should be common.rb only", chdir: @target) + out = assert_sync() + assert_not_equal(@sha["src"], top_commit("src"), out) + assert_equal("# main\n", File.read("src/Gemfile"), out) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/.github/workflows/main.yml", out) + end + + def test_delete_after_conflict + File.write("#@target/lib/bad.rb", "raise\n") + git(*%W"add lib/bad.rb", chdir: @target) + git(*%W"commit -q -m", "Add bad.rb", chdir: @target) + out = assert_sync + assert_equal("raise\n", File.read("src/lib/bad.rb")) + + git(*%W"rm lib/bad.rb", chdir: "src", out: IO::NULL) + git(*%W"commit -q -m", "Remove bad.rb", chdir: "src") + + File.write("#@target/lib/bad.rb", "raise 'bar'\n") + File.write("#@target/lib/common.rb", "Should.be_merged\n", mode: "a") + git(*%W"add lib/bad.rb lib/common.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict", chdir: @target) + + head = top_commit("src") + out = assert_sync(editor: "git rm -f lib/bad.rb") + assert_not_equal(head, top_commit("src")) + assert_equal(":ok\n""Should.be_merged\n", File.read("src/lib/common.rb"), out) + assert_not_operator(File, :exist?, "src/lib/bad.rb", out) + end + + def test_squash_merge + # This test is known to fail with git 2.43.0, which is used by Ubuntu 24.04. + # We don't know which exact version fixed it, but we know git 2.52.0 works. + stdout, status = Open3.capture2('git', '--version', err: File::NULL) + omit 'git version check failed' unless status.success? + git_version = stdout[/\Agit version \K\S+/] + omit "git #{git_version} is too old" if Gem::Version.new(git_version) < Gem::Version.new('2.44.0') + + # 2---. <- branch + # / \ + # 1---3---3'<- merge commit with conflict resolution + File.write("#@target/lib/conflict.rb", "# 1\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Add conflict.rb", chdir: @target) + + git(*%W"checkout -q -b branch", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 2\n") + File.write("#@target/lib/new.rb", "# new\n") + git(*%W"add lib/conflict.rb lib/new.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in branch", chdir: @target) + + git(*%W"checkout -q default", chdir: @target) + File.write("#@target/lib/conflict.rb", "# 3\n") + git(*%W"add lib/conflict.rb", chdir: @target) + git(*%W"commit -q -m", "Commit in default", chdir: @target) + + # How can I suppress "Auto-merging ..." message from git merge? + git(*%W"merge -X ours -m", "Merge commit", "branch", chdir: @target, out: IO::NULL) + + out = assert_sync() + assert_equal("# 3\n", File.read("src/lib/conflict.rb"), out) + subject, body = top_commit("src", format: "%B").split("\n\n", 2) + assert_equal("[ruby/#@target] Merge commit", subject, out) + assert_includes(body, "Commit in branch", out) + end + + def test_no_upstream_file + group = SyncDefaultGems::Repository.group(%w[ + lib/un.rb + lib/unicode_normalize/normalize.rb + lib/unicode_normalize/tables.rb + lib/net/https.rb + ]) + expected = { + "un" => %w[lib/un.rb], + "net-http" => %w[lib/net/https.rb], + nil => %w[lib/unicode_normalize/normalize.rb lib/unicode_normalize/tables.rb], + } + assert_equal(expected, group) + end + end if /darwin|linux/ =~ RUBY_PLATFORM +end diff --git a/tool/test/testunit/metametameta.rb b/tool/test/testunit/metametameta.rb new file mode 100644 index 0000000000..e494038939 --- /dev/null +++ b/tool/test/testunit/metametameta.rb @@ -0,0 +1,70 @@ +# encoding: utf-8 +# frozen_string_literal: false + +require 'tempfile' +require 'stringio' + +class Test::Unit::TestCase + def clean s + s.gsub(/^ {6}/, '') + end +end + +class MetaMetaMetaTestCase < Test::Unit::TestCase + def assert_report expected, flags = %w[--seed 42] + header = clean <<-EOM + Run options: #{flags.map { |s| s =~ /\|/ ? s.inspect : s }.join " "} + + # Running tests: + + EOM + + with_output do + @tu.run flags + end + + output = @output.string.dup + output.sub!(/Finished tests in .*/, "Finished tests in 0.00") + output.sub!(/Loaded suite .*/, 'Loaded suite blah') + + output.gsub!(/ = \d+.\d\d s = /, ' = 0.00 s = ') + output.gsub!(/0x[A-Fa-f0-9]+/, '0xXXX') + + if windows? then + output.gsub!(/\[(?:[A-Za-z]:)?[^\]:]+:\d+\]/, '[FILE:LINE]') + output.gsub!(/^(\s+)(?:[A-Za-z]:)?[^:]+:\d+:in/, '\1FILE:LINE:in') + else + output.gsub!(/\[[^\]:]+:\d+\]/, '[FILE:LINE]') + output.gsub!(/^(\s+)[^:]+:\d+:in/, '\1FILE:LINE:in') + end + + assert_equal header + expected, output + end + + def setup + super + srand 42 + Test::Unit::TestCase.reset + @tu = Test::Unit::Runner.new + + Test::Unit::Runner.runner = nil # protect the outer runner from the inner tests + end + + def teardown + super + end + + def with_output + synchronize do + begin + save = Test::Unit::Runner.output + @output = StringIO.new("") + Test::Unit::Runner.output = @output + + yield + ensure + Test::Unit::Runner.output = save + end + end + end +end diff --git a/tool/test/testunit/test4test_hideskip.rb b/tool/test/testunit/test4test_hideskip.rb new file mode 100644 index 0000000000..14f79a5743 --- /dev/null +++ b/tool/test/testunit/test4test_hideskip.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: false +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' + +class TestForTestHideSkip < Test::Unit::TestCase + def test_omit + omit "do nothing" + end + + def test_pend + pend "do nothing" + end +end diff --git a/tool/test/testunit/test4test_load_failure.rb b/tool/test/testunit/test4test_load_failure.rb new file mode 100644 index 0000000000..e1570c2542 --- /dev/null +++ b/tool/test/testunit/test4test_load_failure.rb @@ -0,0 +1 @@ +raise LoadError, "no-such-library" diff --git a/tool/test/testunit/test4test_redefinition.rb b/tool/test/testunit/test4test_redefinition.rb new file mode 100644 index 0000000000..ad3c5e7113 --- /dev/null +++ b/tool/test/testunit/test4test_redefinition.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: false +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' + +class TestForTestRedefinition < Test::Unit::TestCase + def test_redefinition + skip "do nothing (1)" + end + + def test_redefinition + skip "do nothing (2)" + end +end diff --git a/tool/test/testunit/test4test_sorting.rb b/tool/test/testunit/test4test_sorting.rb new file mode 100644 index 0000000000..f5a6866425 --- /dev/null +++ b/tool/test/testunit/test4test_sorting.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: false +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' + +class TestForTestHideSkip < Test::Unit::TestCase + def test_c + omit "do nothing" + end + + def test_b + assert_equal true, false + end + + def test_a + raise + end +end diff --git a/tool/test/testunit/test4test_timeout.rb b/tool/test/testunit/test4test_timeout.rb new file mode 100644 index 0000000000..3225f66398 --- /dev/null +++ b/tool/test/testunit/test4test_timeout.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../lib" + +require 'test/unit' +require 'timeout' + +class TestForTestTimeout < Test::Unit::TestCase + 10.times do |i| + define_method("test_timeout_#{i}") do + Timeout.timeout(0.001) do + sleep + end + end + end +end diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb new file mode 100644 index 0000000000..d9bdc8f3c5 --- /dev/null +++ b/tool/test/testunit/test_assertion.rb @@ -0,0 +1,228 @@ +# frozen_string_literal: false +require 'test/unit' +class TestAssertion < Test::Unit::TestCase + def test_wrong_assertion + error, line = assert_raise(ArgumentError) {assert(true, true)}, __LINE__ + assert_match(/assertion message must be String or Proc, but TrueClass was given/, error.message) + assert_match(/\A#{Regexp.quote(__FILE__)}:#{line}:/, error.backtrace[0]) + end + + def test_timeout_separately + pend "hang-up" if /mswin|mingw/ =~ RUBY_PLATFORM + + assert_raise(Timeout::Error) do + assert_separately([], <<~"end;", timeout: 0.1) + sleep + end; + end + end + + def test_assertion_count_separately + beginning = self._assertions + + assert_separately([], "") + assertions_at_nothing = self._assertions - beginning + + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "assert true") + assert_equal(1, self._assertions - prev_assertions) + + omit unless Process.respond_to?(:fork) + prev_assertions = self._assertions + assertions_at_nothing + assert_separately([], "Process.fork {assert true}; assert true") + assert_equal(2, self._assertions - prev_assertions) + + prev_assertions = self._assertions + assertions_at_nothing + # TODO: assertions before `fork` are counted twice; it is possible + # to reset `_assertions` at `Process._fork`, but the hook can + # interfere in other tests. + assert_separately([], "assert true; Process.fork {assert true}") + assert_equal(3, self._assertions - prev_assertions) + end + + def return_in_assert_raise + assert_raise(RuntimeError) do + return + end + end + + def test_assert_raise + assert_raise(Test::Unit::AssertionFailedError) do + return_in_assert_raise + end + end + + def test_assert_raise_with_message + my_error = Class.new(StandardError) + + assert_raise_with_message(my_error, "with message") do + raise my_error, "with message" + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(RuntimeError, "with message") do + raise my_error, "with message" + end + end + + assert_raise(Test::Unit::AssertionFailedError) do + assert_raise_with_message(my_error, "without message") do + raise my_error, "with message" + end + end + end + + def test_assert_raise_kind_of + my_error = Class.new(StandardError) + + assert_raise_kind_of(my_error) do + raise my_error + end + + assert_raise_kind_of(StandardError) do + raise my_error + end + end + + def test_assert_pattern_list + assert_pattern_list([/foo?/], "foo") + assert_not_pattern_list([/foo?/], "afoo") + assert_not_pattern_list([/foo?/], "foo?") + assert_pattern_list([:*, /foo?/, :*], "foo") + assert_pattern_list([:*, /foo?/], "afoo") + assert_not_pattern_list([:*, /foo?/], "afoo?") + assert_pattern_list([/foo?/, :*], "foo?") + + assert_not_pattern_list(["foo?"], "foo") + assert_not_pattern_list(["foo?"], "afoo") + assert_pattern_list(["foo?"], "foo?") + assert_not_pattern_list([:*, "foo?", :*], "foo") + assert_not_pattern_list([:*, "foo?"], "afoo") + assert_pattern_list([:*, "foo?"], "afoo?") + assert_pattern_list(["foo?", :*], "foo?") + end + + def assert_not_pattern_list(pattern_list, actual, message=nil) + assert_raise(Test::Unit::AssertionFailedError) do + assert_pattern_list(pattern_list, actual, message) + end + end + + def test_caller_bactrace_location + begin + line = __LINE__; assert_fail_for_backtrace_location + rescue Test::Unit::AssertionFailedError => e + end + location = Test::Unit::Runner.new.location(e) + assert_equal "#{__FILE__}:#{line}", location + end + + def assert_fail_for_backtrace_location + assert false + end + + VersionClass = Struct.new(:version) do + def version?(*ver) + Test::Unit::CoreAssertions.version_match?(ver, self.version) + end + end + + V14_6_0 = VersionClass.new([14, 6, 0]) + V15_0_0 = VersionClass.new([15, 0, 0]) + + def test_version_match_integer + assert_not_operator(V14_6_0, :version?, 13) + assert_operator(V14_6_0, :version?, 14) + assert_not_operator(V14_6_0, :version?, 15) + assert_not_operator(V15_0_0, :version?, 14) + assert_operator(V15_0_0, :version?, 15) + end + + def test_version_match_integer_range + assert_operator(V14_6_0, :version?, 13..14) + assert_not_operator(V15_0_0, :version?, 13..14) + assert_not_operator(V14_6_0, :version?, 13...14) + assert_not_operator(V15_0_0, :version?, 13...14) + end + + def test_version_match_array_range + assert_operator(V14_6_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]..[14, 6]) + assert_not_operator(V14_6_0, :version?, [14, 0]...[14, 6]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[14, 6]) + assert_operator(V14_6_0, :version?, [14, 0]..[15]) + assert_operator(V15_0_0, :version?, [14, 0]..[15]) + assert_operator(V14_6_0, :version?, [14, 0]...[15]) + assert_not_operator(V15_0_0, :version?, [14, 0]...[15]) + end + + def test_version_match_integer_endless_range + assert_operator(V14_6_0, :version?, 14..) + assert_operator(V15_0_0, :version?, 14..) + assert_not_operator(V14_6_0, :version?, 15..) + assert_operator(V15_0_0, :version?, 15..) + end + + def test_version_match_integer_endless_range_exclusive + assert_operator(V14_6_0, :version?, 14...) + assert_operator(V15_0_0, :version?, 14...) + assert_not_operator(V14_6_0, :version?, 15...) + assert_operator(V15_0_0, :version?, 15...) + end + + def test_version_match_array_endless_range + assert_operator(V14_6_0, :version?, [14, 5]..) + assert_operator(V15_0_0, :version?, [14, 5]..) + assert_not_operator(V14_6_0, :version?, [14, 7]..) + assert_operator(V15_0_0, :version?, [14, 7]..) + assert_not_operator(V14_6_0, :version?, [15]..) + assert_operator(V15_0_0, :version?, [15]..) + assert_not_operator(V14_6_0, :version?, [15, 0]..) + assert_operator(V15_0_0, :version?, [15, 0]..) + end + + def test_version_match_array_endless_range_exclude_end + assert_operator(V14_6_0, :version?, [14, 5]...) + assert_operator(V15_0_0, :version?, [14, 5]...) + assert_not_operator(V14_6_0, :version?, [14, 7]...) + assert_operator(V15_0_0, :version?, [14, 7]...) + assert_not_operator(V14_6_0, :version?, [15]...) + assert_operator(V15_0_0, :version?, [15]...) + assert_not_operator(V14_6_0, :version?, [15, 0]...) + assert_operator(V15_0_0, :version?, [15, 0]...) + end + + def test_version_match_integer_beginless_range + assert_operator(V14_6_0, :version?, ..14) + assert_not_operator(V15_0_0, :version?, ..14) + assert_operator(V14_6_0, :version?, ..15) + assert_operator(V15_0_0, :version?, ..15) + + assert_not_operator(V14_6_0, :version?, ...14) + assert_not_operator(V15_0_0, :version?, ...14) + assert_operator(V14_6_0, :version?, ...15) + assert_not_operator(V15_0_0, :version?, ...15) + end + + def test_version_match_array_beginless_range + assert_not_operator(V14_6_0, :version?, ..[14, 5]) + assert_not_operator(V15_0_0, :version?, ..[14, 5]) + assert_operator(V14_6_0, :version?, ..[14, 6]) + assert_not_operator(V15_0_0, :version?, ..[14, 6]) + assert_operator(V14_6_0, :version?, ..[15]) + assert_operator(V15_0_0, :version?, ..[15]) + assert_operator(V14_6_0, :version?, ..[15, 0]) + assert_operator(V15_0_0, :version?, ..[15, 0]) + end + + def test_version_match_array_beginless_range_exclude_end + assert_not_operator(V14_6_0, :version?, ...[14, 5]) + assert_not_operator(V15_0_0, :version?, ...[14, 5]) + assert_not_operator(V14_6_0, :version?, ...[14, 6]) + assert_not_operator(V15_0_0, :version?, ...[14, 6]) + assert_operator(V14_6_0, :version?, ...[15]) + assert_not_operator(V15_0_0, :version?, ...[15]) + assert_operator(V14_6_0, :version?, ...[15, 0]) + assert_not_operator(V15_0_0, :version?, ...[15, 0]) + end +end diff --git a/tool/test/testunit/test_hideskip.rb b/tool/test/testunit/test_hideskip.rb new file mode 100644 index 0000000000..a470368bca --- /dev/null +++ b/tool/test/testunit/test_hideskip.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestHideSkip < Test::Unit::TestCase + def test_hideskip + assert_not_match(/^ *1\) Skipped/, hideskip) + assert_match(/^ *1\) Skipped.*^ *2\) Skipped/m, hideskip("--show-skip")) + output = hideskip("--hide-skip") + assert_match(/assertions\/s.\n+2 tests, 0 assertions, 0 failures, 0 errors, 2 skips/, output) + end + + private + + def hideskip(*args) + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_hideskip.rb", + "--verbose", *args], err: [:child, :out]) {|f| + f.read + } + end +end diff --git a/tool/test/testunit/test_launchable.rb b/tool/test/testunit/test_launchable.rb new file mode 100644 index 0000000000..76be876456 --- /dev/null +++ b/tool/test/testunit/test_launchable.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: false +require 'test/unit' +require 'tempfile' +require 'json' +require_relative '../../lib/launchable' + +class TestLaunchable < Test::Unit::TestCase + def test_json_stream_writer + Tempfile.create(['launchable-test-', '.json']) do |f| + json_stream_writer = Launchable::JsonStreamWriter.new(f.path) + json_stream_writer.write_array('testCases') + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 42, + status: "TEST_FAILED", + stdout: nil, + stderr: nil, + createdAt: "2021-10-05T12:34:00", + data: { + lineNumber: 1 + } + } + ) + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 45, + status: "TEST_PASSED", + stdout: "This is stdout", + stderr: "This is stderr", + createdAt: "2021-10-05T12:36:00", + data: { + lineNumber: 10 + } + } + ) + json_stream_writer.close() + expected = <<JSON +{ + "testCases": [ + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 42, + "status": "TEST_FAILED", + "stdout": null, + "stderr": null, + "createdAt": "2021-10-05T12:34:00", + "data": { + "lineNumber": 1 + } + }, + { + "testPath": "file=test/test_a.rb#class=class1#testcase=testcase899", + "duration": 45, + "status": "TEST_PASSED", + "stdout": "This is stdout", + "stderr": "This is stderr", + "createdAt": "2021-10-05T12:36:00", + "data": { + "lineNumber": 10 + } + } + ] +} +JSON + assert_equal(expected, f.read) + end + end +end diff --git a/tool/test/testunit/test_load_failure.rb b/tool/test/testunit/test_load_failure.rb new file mode 100644 index 0000000000..8defa9e39a --- /dev/null +++ b/tool/test/testunit/test_load_failure.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +require 'test/unit' + +class TestLoadFailure < Test::Unit::TestCase + def test_load_failure + assert_not_predicate(load_failure, :success?) + end + + def test_load_failure_parallel + assert_not_predicate(load_failure("-j2"), :success?) + end + + private + + def load_failure(*args) + IO.popen([*@__runner_options__[:ruby], "#{__dir__}/../runner.rb", + "#{__dir__}/test4test_load_failure.rb", + "--verbose", *args], err: [:child, :out]) {|f| + assert_include(f.read, "test4test_load_failure.rb") + } + $? + end +end diff --git a/tool/test/testunit/test_minitest_unit.rb b/tool/test/testunit/test_minitest_unit.rb new file mode 100644 index 0000000000..84b6cf688c --- /dev/null +++ b/tool/test/testunit/test_minitest_unit.rb @@ -0,0 +1,1488 @@ +# encoding: utf-8 +# frozen_string_literal: false + +require 'pathname' +require_relative 'metametameta' + +module MyModule; end +class AnError < StandardError; include MyModule; end +class ImmutableString < String; def inspect; super.freeze; end; end + +class TestMiniTestUnit < MetaMetaMetaTestCase + pwd = Pathname.new File.expand_path Dir.pwd + basedir = Pathname.new(File.expand_path "lib/test") + basedir = basedir.relative_path_from(pwd).to_s + MINITEST_BASE_DIR = basedir[/\A\./] ? basedir : "./#{basedir}" + BT_MIDDLE = ["#{MINITEST_BASE_DIR}/test.rb:161:in `each'", + "#{MINITEST_BASE_DIR}/test.rb:158:in `each'", + "#{MINITEST_BASE_DIR}/test.rb:139:in `run'", + "#{MINITEST_BASE_DIR}/test.rb:106:in `run'"] + + def test_class_puke_with_assertion_failed + exception = Test::Unit::AssertionFailedError.new "Oh no!" + exception.set_backtrace ["unhappy"] + assert_equal 'F', @tu.puke('SomeClass', 'method_name', exception) + assert_equal 1, @tu.failures + assert_match(/^Failure.*Oh no!/m, @tu.report.first) + assert_match("SomeClass#method_name [unhappy]", @tu.report.first) + end + + def test_class_puke_with_assertion_failed_and_long_backtrace + bt = (["test/test_some_class.rb:615:in `method_name'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", + "test/test_some_class.rb:615:in `each'", + "test/test_some_class.rb:614:in `test_method_name'", + "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29"]) + bt = util_expand_bt bt + + ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first + + exception = Test::Unit::AssertionFailedError.new "Oh no!" + exception.set_backtrace bt + assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) + assert_equal 1, @tu.failures + assert_match(/^Failure.*Oh no!/m, @tu.report.first) + assert_match("TestSomeClass#test_method_name [#{ex_location}]", @tu.report.first) + end + + def test_class_puke_with_assertion_failed_and_user_defined_assertions + bt = (["lib/test/my/util.rb:16:in `another_method_name'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", + "lib/test/my/util.rb:15:in `block in assert_something'", + "lib/test/my/util.rb:14:in `each'", + "lib/test/my/util.rb:14:in `assert_something'", + "test/test_some_class.rb:615:in `each'", + "test/test_some_class.rb:614:in `test_method_name'", + "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29"]) + bt = util_expand_bt bt + + ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first + + exception = Test::Unit::AssertionFailedError.new "Oh no!" + exception.set_backtrace bt + assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) + assert_equal 1, @tu.failures + assert_match(/^Failure.*Oh no!/m, @tu.report.first) + assert_match("TestSomeClass#test_method_name [#{ex_location}]", @tu.report.first) + end + + def test_class_puke_with_failure_and_flunk_in_backtrace + exception = begin + Test::Unit::TestCase.new('fake tc').flunk + rescue Test::Unit::AssertionFailedError => failure + failure + end + assert_equal 'F', @tu.puke('SomeClass', 'method_name', exception) + refute @tu.report.any?{|line| line =~ /in .flunk/} + end + + def test_class_puke_with_flunk_and_user_defined_assertions + bt = (["lib/test/my/util.rb:16:in `flunk'", + "#{MINITEST_BASE_DIR}/unit.rb:140:in `assert_raise'", + "lib/test/my/util.rb:15:in `block in assert_something'", + "lib/test/my/util.rb:14:in `each'", + "lib/test/my/util.rb:14:in `assert_something'", + "test/test_some_class.rb:615:in `each'", + "test/test_some_class.rb:614:in `test_method_name'", + "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29"]) + bt = util_expand_bt bt + + ex_location = util_expand_bt(["test/test_some_class.rb:615"]).first + + exception = Test::Unit::AssertionFailedError.new "Oh no!" + exception.set_backtrace bt + assert_equal 'F', @tu.puke('TestSomeClass', 'test_method_name', exception) + assert_equal 1, @tu.failures + assert_match(/^Failure.*Oh no!/m, @tu.report.first) + assert_match("TestSomeClass#test_method_name [#{ex_location}]", @tu.report.first) + end + + def test_class_puke_with_non_failure_exception + exception = Exception.new("Oh no again!") + assert_equal 'E', @tu.puke('SomeClass', 'method_name', exception) + assert_equal 1, @tu.errors + assert_match(/^Exception.*Oh no again!/m, @tu.report.first) + end + + def test_filter_backtrace + # this is a semi-lame mix of relative paths. + # I cheated by making the autotest parts not have ./ + bt = (["lib/autotest.rb:571:in `add_exception'", + "test/test_autotest.rb:62:in `test_add_exception'", + "#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29", + "test/test_autotest.rb:422"]) + bt = util_expand_bt bt + + ex = ["lib/autotest.rb:571:in `add_exception'", + "test/test_autotest.rb:62:in `test_add_exception'"] + ex = util_expand_bt ex + + fu = Test::filter_backtrace(bt) + + assert_equal ex, fu + end + + def test_filter_backtrace_all_unit + bt = (["#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/test.rb:29"]) + ex = bt.clone + fu = Test::filter_backtrace(bt) + assert_equal ex, fu + end + + def test_filter_backtrace_unit_starts + bt = (["#{MINITEST_BASE_DIR}/test.rb:165:in `__send__'"] + + BT_MIDDLE + + ["#{MINITEST_BASE_DIR}/mini/test.rb:29", + "-e:1"]) + + bt = util_expand_bt bt + + ex = ["-e:1"] + fu = Test::filter_backtrace bt + assert_equal ex, fu + end + + def test_default_runner_is_minitest_unit + assert_instance_of Test::Unit::Runner, Test::Unit::Runner.runner + end + + + def test_passed_eh_teardown_good + test_class = Class.new Test::Unit::TestCase do + def teardown; assert true; end + def test_omg; assert true; end + end + + test = test_class.new :test_omg + test.run @tu + assert test.passed? + end + + def test_passed_eh_teardown_skipped + test_class = Class.new Test::Unit::TestCase do + def teardown; assert true; end + def test_omg; omit "bork"; end + end + + test = test_class.new :test_omg + test.run @tu + assert test.passed? + end + + def test_passed_eh_teardown_flunked + test_class = Class.new Test::Unit::TestCase do + def teardown; flunk; end + def test_omg; assert true; end + end + + test = test_class.new :test_omg + test.run @tu + refute test.passed? + end + + def util_expand_bt bt + bt.map { |f| (f =~ /^\./) ? File.expand_path(f) : f } + end +end + +class TestMiniTestUnitInherited < MetaMetaMetaTestCase + def with_overridden_include + Class.class_eval do + def inherited_with_hacks klass + throw :inherited_hook + end + + alias inherited_without_hacks inherited + alias inherited inherited_with_hacks + alias IGNORE_ME! inherited # 1.8 bug. god I love venture bros + end + + yield + ensure + Class.class_eval do + alias inherited inherited_without_hacks + + undef_method :inherited_with_hacks + undef_method :inherited_without_hacks + end + + refute_respond_to Class, :inherited_with_hacks + refute_respond_to Class, :inherited_without_hacks + end + + def test_inherited_hook_plays_nice_with_others + with_overridden_include do + assert_throws :inherited_hook do + Class.new Test::Unit::TestCase + end + end + end +end + +class TestMiniTestRunner < MetaMetaMetaTestCase + # do not parallelize this suite... it just can't handle it. + + def test_class_test_suites + @assertion_count = 0 + + tc = Class.new(Test::Unit::TestCase) + + assert_equal 2, Test::Unit::TestCase.test_suites.size + assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites.sort_by {|ts| ts.name.to_s} + end + + def assert_filtering name, expected, a = false + args = %W[--name #{name} --seed 42] + + alpha = Class.new Test::Unit::TestCase do + define_method :test_something do + assert a + end + end + Object.const_set(:Alpha, alpha) + + beta = Class.new Test::Unit::TestCase do + define_method :test_something do + assert true + end + end + Object.const_set(:Beta, beta) + + assert_report expected, args + ensure + Object.send :remove_const, :Alpha + Object.send :remove_const, :Beta + end + + def test_run_with_other_runner + pend "We don't imagine to replace the default runner with ruby/ruby test suite." + Test::Unit::Runner.runner = Class.new Test::Unit::Runner do + def _run_suite suite, type + suite.before_suite # Run once before each suite + super suite, type + end + end.new + + Class.new Test::Unit::TestCase do + def self.name; "wacky!" end + + def self.before_suite + Test::Unit::Runner.output.puts "Running #{self.name} tests" + @@foo = 1 + end + + def test_something + assert_equal 1, @@foo + end + + def test_something_else + assert_equal 1, @@foo + end + end + + expected = clean <<-EOM + Running wacky! tests + .. + + Finished tests in 0.00 + + 2 tests, 2 assertions, 0 failures, 0 errors, 0 skips + EOM + + assert_report expected + end + + require 'monitor' + + class Latch + def initialize count = 1 + @count = count + @lock = Monitor.new + @cv = @lock.new_cond + end + + def release + @lock.synchronize do + @count -= 1 if @count > 0 + @cv.broadcast if @count == 0 + end + end + + def await + @lock.synchronize { @cv.wait_while { @count > 0 } } + end + end +end + +class TestMiniTestUnitOrder < MetaMetaMetaTestCase + # do not parallelize this suite... it just can't handle it. + + def test_before_setup + pend "Surpressing the raise message when running with tests" + + call_order = [] + Class.new Test::Unit::TestCase do + define_method :setup do + super() + call_order << :setup + end + + define_method :before_setup do + call_order << :before_setup + end + + def test_omg; assert true; end + end + + with_output do + @tu.run %w[--seed 42] + end + + expected = [:before_setup, :setup] + assert_equal expected, call_order + end + + def test_after_teardown + pend "Surpressing the result message of this tests" + + call_order = [] + Class.new Test::Unit::TestCase do + define_method :teardown do + super() + call_order << :teardown + end + + define_method :after_teardown do + call_order << :after_teardown + end + + def test_omg; assert true; end + end + + with_output do + @tu.run %w[--seed 42] + end + + expected = [:teardown, :after_teardown] + assert_equal expected, call_order + end + + def test_all_teardowns_are_guaranteed_to_run + pend "Surpressing the raise message when running with tests" + + call_order = [] + Class.new Test::Unit::TestCase do + define_method :after_teardown do + super() + call_order << :after_teardown + raise + end + + define_method :teardown do + super() + call_order << :teardown + raise + end + + define_method :before_teardown do + super() + call_order << :before_teardown + raise + end + + def test_omg; assert true; end + end + + with_output do + @tu.run %w[--seed 42] + end + + expected = [:before_teardown, :teardown, :after_teardown] + assert_equal expected, call_order + end + + def test_setup_and_teardown_survive_inheritance + pend "Surpressing the result message of this tests" + + call_order = [] + + parent = Class.new Test::Unit::TestCase do + define_method :setup do + call_order << :setup_method + end + + define_method :teardown do + call_order << :teardown_method + end + + define_method :test_something do + call_order << :test + end + end + + _ = Class.new parent + + with_output do + @tu.run %w[--seed 42] + end + + # Once for the parent class, once for the child + expected = [:setup_method, :test, :teardown_method] * 2 + + assert_equal expected, call_order + end +end + +class TestMiniTestUnitTestCase < Test::Unit::TestCase + # do not call parallelize_me! - teardown accesses @tc._assertions + # which is not threadsafe. Nearly every method in here is an + # assertion test so it isn't worth splitting it out further. + + def setup + super + + Test::Unit::TestCase.reset + + @tc = Test::Unit::TestCase.new 'fake tc' + @zomg = "zomg ponies!" + @assertion_count = 1 + end + + def teardown + assert_equal(@assertion_count, @tc._assertions, + "expected #{@assertion_count} assertions to be fired during the test, not #{@tc._assertions}") if @tc.passed? + end + + def non_verbose + orig_verbose = $VERBOSE + $VERBOSE = false + + yield + ensure + $VERBOSE = orig_verbose + end + + def test_assert + @assertion_count = 2 + + @tc.assert_equal true, @tc.assert(true), "returns true on success" + end + + def test_assert__triggered + util_assert_triggered "Failed assertion, no message given." do + @tc.assert false + end + end + + def test_assert__triggered_message + util_assert_triggered @zomg do + @tc.assert false, @zomg + end + end + + def test_assert_empty + @assertion_count = 2 + + @tc.assert_empty [] + end + + def test_assert_empty_triggered + @assertion_count = 2 + + util_assert_triggered "Expected [1] to be empty." do + @tc.assert_empty [1] + end + end + + def test_assert_equal + @tc.assert_equal 1, 1 + end + + def test_assert_equal_different_collection_array_hex_invisible + object1 = Object.new + object2 = Object.new + msg = "<[#{object1.inspect}]> expected but was + <[#{object2.inspect}]>.".gsub(/^ +/, "") + util_assert_triggered msg do + @tc.assert_equal [object1], [object2] + end + end + + def test_assert_equal_different_collection_hash_hex_invisible + h1, h2 = {}, {} + h1[1] = Object.new + h2[1] = Object.new + msg = "<#{h1.inspect}> expected but was + <#{h2.inspect}>.".gsub(/^ +/, "") + + util_assert_triggered msg do + @tc.assert_equal h1, h2 + end + end + + def test_assert_equal_different_diff_deactivated + without_diff do + util_assert_triggered util_msg("haha" * 10, "blah" * 10) do + o1 = "haha" * 10 + o2 = "blah" * 10 + + @tc.assert_equal o1, o2 + end + end + end + + def test_assert_equal_different_hex + c = Class.new do + def initialize s; @name = s; end + end + + o1 = c.new "a" + o2 = c.new "b" + msg = "<#{o1.inspect}> expected but was + <#{o2.inspect}>.".gsub(/^ +/, "") + + util_assert_triggered msg do + @tc.assert_equal o1, o2 + end + end + + def test_assert_equal_different_hex_invisible + o1 = Object.new + o2 = Object.new + + msg = "<#{o1.inspect}> expected but was + <#{o2.inspect}>.".gsub(/^ +/, "") + + util_assert_triggered msg do + @tc.assert_equal o1, o2 + end + end + + def test_assert_equal_different_long + msg = "<\"hahahahahahahahahahahahahahahahahahahaha\"> expected but was + <\"blahblahblahblahblahblahblahblahblahblah\">.".gsub(/^ +/, "") + + util_assert_triggered msg do + o1 = "haha" * 10 + o2 = "blah" * 10 + + @tc.assert_equal o1, o2 + end + end + + def test_assert_equal_different_long_invisible + msg = "<\"blahblahblahblahblahblahblahblahblahblah\"> (UTF-8) expected but was + <\"blahblahblahblahblahblahblahblahblahblah\"> (UTF-8).".gsub(/^ +/, "") + + util_assert_triggered msg do + o1 = "blah" * 10 + o2 = "blah" * 10 + def o1.== o + false + end + @tc.assert_equal o1, o2 + end + end + + def test_assert_equal_different_long_msg + msg = "message. + <\"hahahahahahahahahahahahahahahahahahahaha\"> expected but was + <\"blahblahblahblahblahblahblahblahblahblah\">.".gsub(/^ +/, "") + + util_assert_triggered msg do + o1 = "haha" * 10 + o2 = "blah" * 10 + @tc.assert_equal o1, o2, "message" + end + end + + def test_assert_equal_different_short + util_assert_triggered util_msg(1, 2) do + @tc.assert_equal 1, 2 + end + end + + def test_assert_equal_different_short_msg + util_assert_triggered util_msg(1, 2, "message") do + @tc.assert_equal 1, 2, "message" + end + end + + def test_assert_equal_different_short_multiline + msg = "<\"a\\n\" + \"b\"> expected but was\n<\"a\\n\" + \"c\">." + util_assert_triggered msg do + @tc.assert_equal "a\nb", "a\nc" + end + end + + def test_assert_equal_different_escaped_newline + msg = "<\"xxx\\n\" + \"a\\\\nb\"> expected but was\n<\"xxx\\n\" + \"a\\\\nc\">." + util_assert_triggered msg do + @tc.assert_equal "xxx\na\\nb", "xxx\na\\nc" + end + end + + def test_assert_in_delta + @tc.assert_in_delta 0.0, 1.0 / 1000, 0.1 + end + + def test_delta_consistency + @tc.assert_in_delta 0, 1, 1 + + util_assert_triggered "Expected |0 - 1| (1) to not be <= 1." do + @tc.refute_in_delta 0, 1, 1 + end + end + + def test_assert_in_delta_triggered + x = "1.0e-06" + util_assert_triggered "Expected |0.0 - 0.001| (0.001) to be <= #{x}." do + @tc.assert_in_delta 0.0, 1.0 / 1000, 0.000001 + end + end + + def test_assert_in_epsilon + @assertion_count = 10 + + @tc.assert_in_epsilon 10000, 9991 + @tc.assert_in_epsilon 9991, 10000 + @tc.assert_in_epsilon 1.0, 1.001 + @tc.assert_in_epsilon 1.001, 1.0 + + @tc.assert_in_epsilon 10000, 9999.1, 0.0001 + @tc.assert_in_epsilon 9999.1, 10000, 0.0001 + @tc.assert_in_epsilon 1.0, 1.0001, 0.0001 + @tc.assert_in_epsilon 1.0001, 1.0, 0.0001 + + @tc.assert_in_epsilon(-1, -1) + @tc.assert_in_epsilon(-10000, -9991) + end + + def test_epsilon_consistency + @tc.assert_in_epsilon 1.0, 1.001 + + msg = "Expected |1.0 - 1.001| (0.000999xxx) to not be <= 0.001." + util_assert_triggered msg do + @tc.refute_in_epsilon 1.0, 1.001 + end + end + + def test_assert_in_epsilon_triggered + util_assert_triggered 'Expected |10000 - 9990| (10) to be <= 9.99.' do + @tc.assert_in_epsilon 10000, 9990 + end + end + + def test_assert_in_epsilon_triggered_negative_case + x = "0.100000xxx" + y = "0.1" + util_assert_triggered "Expected |-1.1 - -1| (#{x}) to be <= #{y}." do + @tc.assert_in_epsilon(-1.1, -1, 0.1) + end + end + + def test_assert_includes + @assertion_count = 2 + + @tc.assert_includes [true], true + end + + def test_assert_includes_triggered + @assertion_count = 3 + + e = @tc.assert_raise Test::Unit::AssertionFailedError do + @tc.assert_includes [true], false + end + + expected = "Expected [true] to include false." + assert_equal expected, e.message + end + + def test_assert_instance_of + @tc.assert_instance_of String, "blah" + end + + def test_assert_instance_of_triggered + util_assert_triggered 'Expected "blah" to be an instance of Array, not String.' do + @tc.assert_instance_of Array, "blah" + end + end + + def test_assert_kind_of + @tc.assert_kind_of String, "blah" + end + + def test_assert_kind_of_triggered + util_assert_triggered 'Expected "blah" to be a kind of Array, not String.' do + @tc.assert_kind_of Array, "blah" + end + end + + def test_assert_match + @assertion_count = 2 + @tc.assert_match(/\w+/, "blah blah blah") + end + + def test_assert_match_matcher_object + @assertion_count = 2 + + pattern = Object.new + def pattern.=~(other) true end + + @tc.assert_match pattern, 5 + end + + def test_assert_match_matchee_to_str + @assertion_count = 2 + + obj = Object.new + def obj.to_str; "blah" end + + @tc.assert_match "blah", obj + end + + def test_assert_match_object_triggered + @assertion_count = 2 + + pattern = Object.new + def pattern.=~(other) false end + def pattern.inspect; "[Object]" end + + util_assert_triggered 'Expected [Object] to match 5.' do + @tc.assert_match pattern, 5 + end + end + + def test_assert_match_triggered + @assertion_count = 2 + util_assert_triggered 'Expected /\d+/ to match "blah blah blah".' do + @tc.assert_match(/\d+/, "blah blah blah") + end + end + + def test_assert_nil + @tc.assert_nil nil + end + + def test_assert_nil_triggered + util_assert_triggered 'Expected 42 to be nil.' do + @tc.assert_nil 42 + end + end + + def test_assert_operator + @tc.assert_operator 2, :>, 1 + end + + def test_assert_operator_bad_object + bad = Object.new + def bad.==(other) true end + + @tc.assert_operator bad, :equal?, bad + end + + def test_assert_operator_triggered + util_assert_triggered "Expected 2 to be < 1." do + @tc.assert_operator 2, :<, 1 + end + end + + def test_assert_output_both + @assertion_count = 2 + + @tc.assert_output "yay", "blah" do + print "yay" + $stderr.print "blah" + end + end + + def test_assert_output_both_regexps + @assertion_count = 4 + + @tc.assert_output(/y.y/, /bl.h/) do + print "yay" + $stderr.print "blah" + end + end + + def test_assert_output_err + @tc.assert_output nil, "blah" do + $stderr.print "blah" + end + end + + def test_assert_output_neither + @assertion_count = 0 + + @tc.assert_output do + # do nothing + end + end + + def test_assert_output_out + @tc.assert_output "blah" do + print "blah" + end + end + + def test_assert_output_triggered_both + util_assert_triggered util_msg("blah", "blah blah", "In stderr") do + @tc.assert_output "yay", "blah" do + print "boo" + $stderr.print "blah blah" + end + end + end + + def test_assert_output_triggered_err + util_assert_triggered util_msg("blah", "blah blah", "In stderr") do + @tc.assert_output nil, "blah" do + $stderr.print "blah blah" + end + end + end + + def test_assert_output_triggered_out + util_assert_triggered util_msg("blah", "blah blah", "In stdout") do + @tc.assert_output "blah" do + print "blah blah" + end + end + end + + def test_assert_predicate + @tc.assert_predicate "", :empty? + end + + def test_assert_predicate_triggered + util_assert_triggered 'Expected "blah" to be empty?.' do + @tc.assert_predicate "blah", :empty? + end + end + + def test_assert_raise + @tc.assert_raise RuntimeError do + raise "blah" + end + end + + def test_assert_raise_module + @tc.assert_raise MyModule do + raise AnError + end + end + + ## + # *sigh* This is quite an odd scenario, but it is from real (albeit + # ugly) test code in ruby-core: + # + # https://github.com/ruby/ruby/commit/6bab4ea9917dc05cd2c94aead2e96eb7df7d4be1 + + def test_assert_raise_skip + @assertion_count = 0 + + util_assert_triggered "skipped", Test::Unit::PendedError do + @tc.assert_raise ArgumentError do + begin + raise "blah" + rescue + omit "skipped" + end + end + end + end + + def test_assert_raise_triggered_different + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise RuntimeError do + raise SyntaxError, "icky" + end + end + + expected = clean <<-EOM.chomp + [RuntimeError] exception expected, not #<SyntaxError: icky>. + EOM + + actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') + actual.gsub!(/block \(\d+ levels\) in /, '') if RUBY_VERSION >= '1.9.0' + + assert_equal expected, actual + end + + def test_assert_raise_triggered_different_msg + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise RuntimeError, "XXX" do + raise SyntaxError, "icky" + end + end + + expected = clean <<-EOM + XXX. + [RuntimeError] exception expected, not #<SyntaxError: icky>. + EOM + + actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') + actual.gsub!(/block \(\d+ levels\) in /, '') if RUBY_VERSION >= '1.9.0' + + assert_equal expected.chomp, actual + end + + def test_assert_raise_triggered_none + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise Test::Unit::AssertionFailedError do + # do nothing + end + end + + expected = "Test::Unit::AssertionFailedError expected but nothing was raised." + + assert_equal expected, e.message + end + + def test_assert_raise_triggered_none_msg + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise Test::Unit::AssertionFailedError, "XXX" do + # do nothing + end + end + + expected = "XXX.\nTest::Unit::AssertionFailedError expected but nothing was raised." + + assert_equal expected, e.message + end + + def test_assert_raise_triggered_subclass + e = assert_raise Test::Unit::AssertionFailedError do + @tc.assert_raise StandardError do + raise AnError + end + end + + expected = clean <<-EOM.chomp + [StandardError] exception expected, not #<AnError: AnError>. + EOM + + actual = e.message.gsub(/^.+:\d+/, 'FILE:LINE') + actual.gsub!(/block \(\d+ levels\) in /, '') if RUBY_VERSION >= '1.9.0' + + assert_equal expected, actual + end + + def test_assert_respond_to + @tc.assert_respond_to "blah", :empty? + end + + def test_assert_respond_to_triggered + util_assert_triggered 'Expected "blah" (String) to respond to #rawr!.' do + @tc.assert_respond_to "blah", :rawr! + end + end + + def test_assert_same + @assertion_count = 3 + + o = "blah" + @tc.assert_same 1, 1 + @tc.assert_same :blah, :blah + @tc.assert_same o, o + end + + def test_assert_same_triggered + @assertion_count = 2 + + util_assert_triggered 'Expected 2 (oid=N) to be the same as 1 (oid=N).' do + @tc.assert_same 1, 2 + end + + s1 = "blah" + s2 = "blah" + + util_assert_triggered 'Expected "blah" (oid=N) to be the same as "blah" (oid=N).' do + @tc.assert_same s1, s2 + end + end + + def test_assert_send + @tc.assert_send [1, :<, 2] + end + + def test_assert_send_bad + util_assert_triggered "Expected 1.>(2) to return true." do + @tc.assert_send [1, :>, 2] + end + end + + def test_assert_silent + @assertion_count = 2 + + @tc.assert_silent do + # do nothing + end + end + + def test_assert_silent_triggered_err + util_assert_triggered util_msg("", "blah blah", "In stderr") do + @tc.assert_silent do + $stderr.print "blah blah" + end + end + end + + def test_assert_silent_triggered_out + @assertion_count = 2 + + util_assert_triggered util_msg("", "blah blah", "In stdout") do + @tc.assert_silent do + print "blah blah" + end + end + end + + def test_assert_throws + @tc.assert_throws :blah do + throw :blah + end + end + + def test_assert_throws_different + util_assert_triggered 'Expected :blah to have been thrown, not :not_blah.' do + @tc.assert_throws :blah do + throw :not_blah + end + end + end + + def test_assert_throws_unthrown + util_assert_triggered 'Expected :blah to have been thrown.' do + @tc.assert_throws :blah do + # do nothing + end + end + end + + def test_capture_output + @assertion_count = 0 + + non_verbose do + out, err = capture_output do + puts 'hi' + $stderr.puts 'bye!' + end + + assert_equal "hi\n", out + assert_equal "bye!\n", err + end + end + + def test_flunk + util_assert_triggered 'Epic Fail!' do + @tc.flunk + end + end + + def test_flunk_message + util_assert_triggered @zomg do + @tc.flunk @zomg + end + end + + def test_message + @assertion_count = 0 + + assert_equal "blah2.", @tc.message { "blah2" }.call + assert_equal "blah2.", @tc.message("") { "blah2" }.call + assert_equal "blah1.\nblah2.", @tc.message(:blah1) { "blah2" }.call + assert_equal "blah1.\nblah2.", @tc.message("blah1") { "blah2" }.call + + message = proc { "blah1" } + assert_equal "blah1.\nblah2.", @tc.message(message) { "blah2" }.call + + message = @tc.message { "blah1" } + assert_equal "blah1.\nblah2.", @tc.message(message) { "blah2" }.call + end + + def test_message_message + util_assert_triggered "whoops.\n<1> expected but was\n<2>." do + @tc.assert_equal 1, 2, message { "whoops" } + end + end + + def test_message_lambda + util_assert_triggered "whoops.\n<1> expected but was\n<2>." do + @tc.assert_equal 1, 2, lambda { "whoops" } + end + end + + def test_message_deferred + @assertion_count, var = 0, nil + + msg = message { var = "blah" } + + assert_nil var + + msg.call + + assert_equal "blah", var + end + + def test_pass + @tc.pass + end + + def test_prints + printer = Class.new { extend Test::Unit::CoreAssertions } + @tc.assert_equal '"test"', printer.mu_pp(ImmutableString.new 'test') + end + + def test_refute + @assertion_count = 2 + + @tc.assert_equal false, @tc.refute(false), "returns false on success" + end + + def test_refute_empty + @assertion_count = 2 + + @tc.refute_empty [1] + end + + def test_refute_empty_triggered + @assertion_count = 2 + + util_assert_triggered "Expected [] to not be empty." do + @tc.refute_empty [] + end + end + + def test_refute_equal + @tc.refute_equal "blah", "yay" + end + + def test_refute_equal_triggered + util_assert_triggered 'Expected "blah" to not be equal to "blah".' do + @tc.refute_equal "blah", "blah" + end + end + + def test_refute_in_delta + @tc.refute_in_delta 0.0, 1.0 / 1000, 0.000001 + end + + def test_refute_in_delta_triggered + x = "0.1" + util_assert_triggered "Expected |0.0 - 0.001| (0.001) to not be <= #{x}." do + @tc.refute_in_delta 0.0, 1.0 / 1000, 0.1 + end + end + + def test_refute_in_epsilon + @tc.refute_in_epsilon 10000, 9990-1 + end + + def test_refute_in_epsilon_triggered + util_assert_triggered 'Expected |10000 - 9990| (10) to not be <= 10.0.' do + @tc.refute_in_epsilon 10000, 9990 + fail + end + end + + def test_refute_includes + @assertion_count = 2 + + @tc.refute_includes [true], false + end + + def test_refute_includes_triggered + @assertion_count = 3 + + e = @tc.assert_raise Test::Unit::AssertionFailedError do + @tc.refute_includes [true], true + end + + expected = "Expected [true] to not include true." + assert_equal expected, e.message + end + + def test_refute_instance_of + @tc.refute_instance_of Array, "blah" + end + + def test_refute_instance_of_triggered + util_assert_triggered 'Expected "blah" to not be an instance of String.' do + @tc.refute_instance_of String, "blah" + end + end + + def test_refute_kind_of + @tc.refute_kind_of Array, "blah" + end + + def test_refute_kind_of_triggered + util_assert_triggered 'Expected "blah" to not be a kind of String.' do + @tc.refute_kind_of String, "blah" + end + end + + def test_refute_match + @assertion_count = 2 + @tc.refute_match(/\d+/, "blah blah blah") + end + + def test_refute_match_matcher_object + @assertion_count = 2 + non_verbose do + obj = Object.new + def obj.=~(other); false; end + @tc.refute_match obj, 5 + end + end + + def test_refute_match_object_triggered + @assertion_count = 2 + + pattern = Object.new + def pattern.=~(other) true end + def pattern.inspect; "[Object]" end + + util_assert_triggered 'Expected [Object] to not match 5.' do + @tc.refute_match pattern, 5 + end + end + + def test_refute_match_triggered + @assertion_count = 2 + util_assert_triggered 'Expected /\w+/ to not match "blah blah blah".' do + @tc.refute_match(/\w+/, "blah blah blah") + end + end + + def test_refute_nil + @tc.refute_nil 42 + end + + def test_refute_nil_triggered + util_assert_triggered 'Expected nil to not be nil.' do + @tc.refute_nil nil + end + end + + def test_refute_predicate + @tc.refute_predicate "42", :empty? + end + + def test_refute_predicate_triggered + util_assert_triggered 'Expected "" to not be empty?.' do + @tc.refute_predicate "", :empty? + end + end + + def test_refute_operator + @tc.refute_operator 2, :<, 1 + end + + def test_refute_operator_bad_object + bad = Object.new + def bad.==(other) true end + + @tc.refute_operator true, :equal?, bad + end + + def test_refute_operator_triggered + util_assert_triggered "Expected 2 to not be > 1." do + @tc.refute_operator 2, :>, 1 + end + end + + def test_refute_respond_to + @tc.refute_respond_to "blah", :rawr! + end + + def test_refute_respond_to_triggered + util_assert_triggered 'Expected "blah" to not respond to empty?.' do + @tc.refute_respond_to "blah", :empty? + end + end + + def test_refute_same + @tc.refute_same 1, 2 + end + + def test_refute_same_triggered + util_assert_triggered 'Expected 1 (oid=N) to not be the same as 1 (oid=N).' do + @tc.refute_same 1, 1 + end + end + + def test_omit + @assertion_count = 0 + + util_assert_triggered "haha!", Test::Unit::PendedError do + @tc.omit "haha!" + end + end + + def test_pend + @assertion_count = 0 + + util_assert_triggered "haha!", Test::Unit::PendedError do + @tc.pend "haha!" + end + end + + def test_test_methods + @assertion_count = 0 + + sample_test_case = Class.new Test::Unit::TestCase do + def test_test1; assert "does not matter" end + def test_test2; assert "does not matter" end + def test_test3; assert "does not matter" end + end + + expected = %i(test_test1 test_test2 test_test3) + assert_equal expected, sample_test_case.test_methods.sort + end + + def assert_triggered expected, klass = Test::Unit::AssertionFailedError + e = assert_raise klass do + yield + end + + msg = e.message.sub(/(---Backtrace---).*/m, '\1') + msg.gsub!(/\(oid=[-0-9]+\)/, '(oid=N)') + msg.gsub!(/(\d\.\d{6})\d+/, '\1xxx') # normalize: ruby version, impl, platform + + assert_equal expected, msg + end + alias util_assert_triggered assert_triggered + + def util_msg exp, act, msg = nil + s = "<#{exp.inspect}> expected but was\n<#{act.inspect}>." + s = "#{msg}.\n#{s}" if msg + s + end + + def without_diff + old_diff = Test::Unit::Assertions.diff + Test::Unit::Assertions.diff = nil + + yield + ensure + Test::Unit::Assertions.diff = old_diff + end +end + +class TestMiniTestGuard < Test::Unit::TestCase + def test_mri_eh + assert self.class.mri? "ruby blah" + assert self.mri? "ruby blah" + end + + def test_jruby_eh + assert self.class.jruby? "java" + assert self.jruby? "java" + end + + def test_windows_eh + assert self.class.windows? "mswin" + assert self.windows? "mswin" + end +end + +class TestMiniTestUnitRecording < MetaMetaMetaTestCase + # do not parallelize this suite... it just can't handle it. + + def assert_run_record(*expected, &block) + pend "Test::Unit::Runner#run was changed about recoding feature. We should fix it." + def @tu.record suite, method, assertions, time, error + recording[method] << error + end + + def @tu.recording + @recording ||= Hash.new { |h,k| h[k] = [] } + end + + Test::Unit::Runner.runner = @tu + + Class.new Test::Unit::TestCase, &block + + with_output do + @tu.run + end + + recorded = @tu.recording.fetch("test_method").map(&:class) + + assert_equal expected, recorded + end + + def test_record_passing + assert_run_record NilClass do + def test_method + assert true + end + end + end + + def test_record_failing + assert_run_record Test::Unit::AssertionFailedError do + def test_method + assert false + end + end + end + + def test_record_error + assert_run_record RuntimeError do + def test_method + raise "unhandled exception" + end + end + end + + def test_record_error_teardown + assert_run_record NilClass, RuntimeError do + def test_method + assert true + end + + def teardown + raise "unhandled exception" + end + end + end + + def test_record_error_in_test_and_teardown + assert_run_record AnError, RuntimeError do + def test_method + raise AnError + end + + def teardown + raise "unhandled exception" + end + end + end + + def test_record_skip + assert_run_record Test::Unit::PendedError do + def test_method + omit "not yet" + end + end + + assert_run_record Test::Unit::PendedError do + def test_method + pend "not yet" + end + end + end +end diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb new file mode 100644 index 0000000000..adf7d62ecd --- /dev/null +++ b/tool/test/testunit/test_parallel.rb @@ -0,0 +1,223 @@ +# frozen_string_literal: false +require 'test/unit' +require 'timeout' + +module TestParallel + PARALLEL_RB = "#{__dir__}/../../lib/test/unit/parallel.rb" + TESTS = "#{__dir__}/tests_for_parallel" + # use large timeout for --jit-wait + TIMEOUT = EnvUtil.apply_timeout_scale(100) + + def self.timeout(n, &blk) + start_time = Time.now + Timeout.timeout(n, &blk) + rescue Timeout::Error + end_time = Time.now + raise Timeout::Error, "execution expired (start: #{ start_time }, end: #{ end_time })" + end + + class TestParallelWorker < Test::Unit::TestCase + def setup + i, @worker_in = IO.pipe + @worker_out, o = IO.pipe + @worker_pid = spawn(*@__runner_options__[:ruby], PARALLEL_RB, + "--ruby", @__runner_options__[:ruby].join(" "), + "-j", "t1", "-v", out: o, in: i) + [i,o].each(&:close) + end + + def teardown + if @worker_pid && @worker_in + begin + begin + @worker_in.puts "quit normal" + rescue IOError, Errno::EPIPE + end + ::TestParallel.timeout(2) do + Process.waitpid(@worker_pid) + end + rescue Timeout::Error + begin + Process.kill(:KILL, @worker_pid) + rescue Errno::ESRCH + end + end + end + ensure + begin + @worker_in.close + @worker_out.close + rescue Errno::EPIPE + # may already broken and rescue'ed in above code + end + end + + def test_run + ::TestParallel.timeout(TIMEOUT) do + assert_match(/^ready/,@worker_out.gets) + @worker_in.puts "run #{TESTS}/ptest_first.rb test" + assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^ready/,@worker_out.gets) + end + end + + def test_run_multiple_testcase_in_one_file + ::TestParallel.timeout(TIMEOUT) do + assert_match(/^ready/,@worker_out.gets) + @worker_in.puts "run #{TESTS}/ptest_second.rb test" + assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^ready/,@worker_out.gets) + end + end + + def test_accept_run_command_multiple_times + ::TestParallel.timeout(TIMEOUT) do + assert_match(/^ready/,@worker_out.gets) + @worker_in.puts "run #{TESTS}/ptest_first.rb test" + assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^ready/,@worker_out.gets) + @worker_in.puts "run #{TESTS}/ptest_second.rb test" + assert_match(/^okay/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^start/,@worker_out.gets) + assert_match(/^record/,@worker_out.gets) + assert_match(/^p/,@worker_out.gets) + assert_match(/^done/,@worker_out.gets) + assert_match(/^ready/,@worker_out.gets) + end + end + + def test_p + ::TestParallel.timeout(TIMEOUT) do + @worker_in.puts "run #{TESTS}/ptest_first.rb test" + while buf = @worker_out.gets + break if /^p (.+?)$/ =~ buf + end + assert_not_nil($1, "'p' was not found") + assert_match(/TestA#test_nothing_test = \d+\.\d+ s = \.\n/, $1.chomp.unpack1("m")) + end + end + + def test_done + ::TestParallel.timeout(TIMEOUT) do + @worker_in.puts "run #{TESTS}/ptest_forth.rb test" + while buf = @worker_out.gets + break if /^done (.+?)$/ =~ buf + end + assert_not_nil($1, "'done' was not found") + + result = Marshal.load($1.chomp.unpack1("m")) + tests, asserts, reports, failures, loadpaths, suite = result + assert_equal(5, tests) + assert_equal(12, asserts) + assert_kind_of(Array, reports) + assert_kind_of(Array, failures) + assert_kind_of(Array, loadpaths) + reports.sort_by! {|_, t| t} + assert_kind_of(Array, reports[1]) + assert_kind_of(Test::Unit::AssertionFailedError, reports[0][2]) + assert_kind_of(Test::Unit::PendedError, reports[1][2]) + assert_kind_of(Test::Unit::PendedError, reports[2][2]) + assert_kind_of(Exception, reports[3][2]) + assert_equal("TestE", suite) + end + end + + def test_quit + ::TestParallel.timeout(TIMEOUT) do + @worker_in.puts "quit normal" + assert_match(/^bye$/m,@worker_out.read) + end + end + end + + class TestParallel < Test::Unit::TestCase + def spawn_runner(*opt_args, jobs: "t1", env: {}) + @test_out, o = IO.pipe + @test_pid = spawn(env, *@__runner_options__[:ruby], TESTS+"/runner.rb", + "--ruby", @__runner_options__[:ruby].join(" "), + "-j", jobs, *opt_args, out: o, err: o) + o.close + end + + def teardown + begin + if @test_pid + ::TestParallel.timeout(2) do + Process.waitpid(@test_pid) + end + end + rescue Timeout::Error + Process.kill(:KILL, @test_pid) if @test_pid + ensure + @test_out&.close + end + end + + def test_ignore_jzero + spawn_runner(jobs: "0") + ::TestParallel.timeout(TIMEOUT) { + assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read) + } + end + + def test_should_run_all_without_any_leaks + spawn_runner + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} + assert_match(/^9 tests/,buf) + end + + def test_should_retry_failed_on_workers + spawn_runner "--retry" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} + assert_match(/^Retrying\.+$/,buf) + end + + def test_no_retry_option + spawn_runner "--no-retry" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} + refute_match(/^Retrying\.+$/,buf) + assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf) + end + + def test_jobs_status + spawn_runner "--jobs-status" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} + assert_match(/\d+=ptest_(first|second|third|forth) */,buf) + end + + def test_separate + # this test depends to --jobs-status + spawn_runner "--jobs-status", "--separate" + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} + assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1, + message("retried tests should run in different processes") {buf}) + end + + def test_hungup + spawn_runner("--worker-timeout=1", "--retry", "test4test_hungup.rb", env: {"RUBY_CRASH_REPORT"=>nil}) + buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read} + assert_match(/^Retrying hung up testcases\.+$/, buf) + assert_match(/^2 tests,.* 0 failures,/, buf) + end + end +end diff --git a/tool/test/testunit/test_redefinition.rb b/tool/test/testunit/test_redefinition.rb new file mode 100644 index 0000000000..b4f5cabd4f --- /dev/null +++ b/tool/test/testunit/test_redefinition.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestRedefinition < Test::Unit::TestCase + def test_redefinition + message = %r[test/unit: method TestForTestRedefinition#test_redefinition is redefined$] + assert_raise_with_message(Test::Unit::AssertionFailedError, message) do + require_relative("test4test_redefinition.rb") + end + end +end diff --git a/tool/test/testunit/test_sorting.rb b/tool/test/testunit/test_sorting.rb new file mode 100644 index 0000000000..3e5d7bfdcc --- /dev/null +++ b/tool/test/testunit/test_sorting.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestTestUnitSorting < Test::Unit::TestCase + def test_sorting + result = sorting("--show-skip") + assert_match(/^ 1\) Skipped:/, result) + assert_match(/^ 2\) Failure:/, result) + assert_match(/^ 3\) Error:/, result) + end + + def sorting(*args) + IO.popen([*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_sorting.rb", + "--verbose", *args], err: [:child, :out]) {|f| + f.read + } + end + + Item = Struct.new(:name) + SEED = 0x50975eed + + def make_test_list + (1..16).map {"test_%.3x" % rand(0x1000)}.freeze + end + + def test_sort_alpha + sorter = Test::Unit::Order::Types[:alpha].new(SEED) + assert_kind_of(Test::Unit::Order::Types[:sorted], sorter) + + list = make_test_list + sorted = list.sort + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + sorted = list.sort_by(&:name) + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + end + + def test_sort_nosort + sorter = Test::Unit::Order::Types[:nosort].new(SEED) + + list = make_test_list + 16.times do + assert_equal(list, sorter.sort_by_string(list)) + end + + list = list.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(list, sorter.sort_by_name(list)) + end + end + + def test_sort_random + type = Test::Unit::Order::Types[:random] + sorter = type.new(SEED) + + list = make_test_list + sorted = type.new(SEED).sort_by_string(list).freeze + 16.times do + assert_equal(sorted, sorter.sort_by_string(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_string(list)) + + list = list.map {|s| Item.new(s)}.freeze + sorted = sorted.map {|s| Item.new(s)}.freeze + 16.times do + assert_equal(sorted, sorter.sort_by_name(list)) + end + assert_not_equal(sorted, type.new(SEED+1).sort_by_name(list)) + end +end diff --git a/tool/test/testunit/test_timeout.rb b/tool/test/testunit/test_timeout.rb new file mode 100644 index 0000000000..452f5e1a7e --- /dev/null +++ b/tool/test/testunit/test_timeout.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestTiemout < Test::Unit::TestCase + def test_timeout + cmd = [*@__runner_options__[:ruby], "#{File.dirname(__FILE__)}/test4test_timeout.rb"] + result = IO.popen(cmd, err: [:child, :out], &:read) + assert_not_match(/^T{10}$/, result) + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_first.rb b/tool/test/testunit/tests_for_parallel/ptest_first.rb new file mode 100644 index 0000000000..f8687335b5 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/ptest_first.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestA < Test::Unit::TestCase + def test_nothing_test + end +end + diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb new file mode 100644 index 0000000000..54474c828d --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestE < Test::Unit::TestCase + class UnknownError < RuntimeError; end + + def test_not_fail + assert_equal(1,1) + end + + def test_always_omit + omit "always" + end + + def test_always_fail + assert_equal(0,1) + end + + def test_pend_after_unknown_error + begin + raise UnknownError, "unknown error" + rescue + pend "after raise" + end + end + + def test_unknown_error + raise UnknownError, "unknown error" + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_second.rb b/tool/test/testunit/tests_for_parallel/ptest_second.rb new file mode 100644 index 0000000000..a793c17eb3 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/ptest_second.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestB < Test::Unit::TestCase + def test_nothing + end +end + +class TestC < Test::Unit::TestCase + def test_nothing + end +end diff --git a/tool/test/testunit/tests_for_parallel/ptest_third.rb b/tool/test/testunit/tests_for_parallel/ptest_third.rb new file mode 100644 index 0000000000..3f448ecfc1 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/ptest_third.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestD < Test::Unit::TestCase + def test_fail_at_worker + #if /test\/unit\/parallel\.rb/ =~ $0 + if on_parallel_worker? + assert_equal(0,1) + end + end +end diff --git a/tool/test/testunit/tests_for_parallel/runner.rb b/tool/test/testunit/tests_for_parallel/runner.rb new file mode 100644 index 0000000000..02699e271e --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/runner.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: false +require 'rbconfig' + +$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../../../lib" + +require 'test/unit' + +src_testdir = File.dirname(File.expand_path(__FILE__)) + +class Test::Unit::Runner + @@testfile_prefix = "ptest" +end + +exit Test::Unit::AutoRunner.run(true, src_testdir) diff --git a/tool/test/testunit/tests_for_parallel/slow_helper.rb b/tool/test/testunit/tests_for_parallel/slow_helper.rb new file mode 100644 index 0000000000..38067c1f47 --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/slow_helper.rb @@ -0,0 +1,8 @@ +require 'test/unit' + +module TestSlowTimeout + def test_slow + sleep_for = EnvUtil.apply_timeout_scale((ENV['sec'] || 3).to_i) + sleep sleep_for if on_parallel_worker? + end +end diff --git a/tool/test/testunit/tests_for_parallel/test4test_hungup.rb b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb new file mode 100644 index 0000000000..49f503ba9e --- /dev/null +++ b/tool/test/testunit/tests_for_parallel/test4test_hungup.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +require_relative '../../../lib/test/unit' + +class TestHung < Test::Unit::TestCase + def test_success_at_worker + assert true + end + + def test_hungup_at_worker + if on_parallel_worker? + sleep EnvUtil.apply_timeout_scale(10) + end + assert true + end +end diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb index 3ff60112d8..1257a92d38 100644 --- a/tool/transcode-tblgen.rb +++ b/tool/transcode-tblgen.rb @@ -1,5 +1,4 @@ -# -# -*- frozen_string_literal: true -*- +# frozen_string_literal: true require 'optparse' require 'erb' @@ -64,7 +63,7 @@ class ArrayCode end def insert_at_last(num, str) - newnum = self.length + num + # newnum = self.length + num @content << str @len += num end @@ -146,7 +145,7 @@ class ActionMap else b = $1.to_i(16) e = $2.to_i(16) - b.upto(e) {|c| set[c] = true } + b.upto(e) {|_| set[_] = true } end } i = nil @@ -236,7 +235,7 @@ class ActionMap all_rects = [] rects1.each {|rect| - min, max, action = rect + _, _, action = rect rect[2] = actions.length actions << action all_rects << rect @@ -245,7 +244,7 @@ class ActionMap boundary = actions.length rects2.each {|rect| - min, max, action = rect + _, _, action = rect rect[2] = actions.length actions << action all_rects << rect @@ -274,7 +273,7 @@ class ActionMap singleton_rects = [] region_rects = [] rects.each {|rect| - min, max, action = rect + min, max, = rect if min == max singleton_rects << rect else @@ -294,14 +293,14 @@ class ActionMap if region_rects.empty? ? s_rect[0].length == prefix.length : region_rects[0][0].empty? h = TMPHASH while (s_rect = @singleton_rects.last) && s_rect[0].start_with?(prefix) - min, max, action = @singleton_rects.pop + min, _, action = @singleton_rects.pop raise ArgumentError, "ambiguous pattern: #{prefix}" if min.length != prefix.length h[action] = true end - region_rects.each {|min, max, action| + for min, _, action in region_rects raise ArgumentError, "ambiguous pattern: #{prefix}" if !min.empty? h[action] = true - } + end tree = Action.new(block.call(prefix, h.keys)) h.clear else @@ -725,14 +724,14 @@ def citrus_decode_mapsrc(ces, csid, mapsrcs) path = File.join(*path) path << ".src" path[path.rindex('/')] = '%' - STDERR.puts 'load mapsrc %s' % path if VERBOSE_MODE - open(path) do |f| + STDOUT.puts 'load mapsrc %s' % path if VERBOSE_MODE > 1 + File.open(path, 'rb') do |f| f.each_line do |l| break if /^BEGIN_MAP/ =~ l end f.each_line do |l| next if /^\s*(?:#|$)/ =~ l - break if /^END_MAP/ =~ l + break if /^END_MAP/ =~ l case mode when :from_ucs case l @@ -741,14 +740,14 @@ def citrus_decode_mapsrc(ces, csid, mapsrcs) when /(0x\w+)\s*=\s*(0x\w+)/ table.push << [plane | $1.hex, citrus_cstomb(ces, csid, $2.hex)] else - raise "unknown notation '%s'"% l + raise "unknown notation '%s'"% l.chomp end when :to_ucs case l when /(0x\w+)\s*=\s*(0x\w+)/ table.push << [citrus_cstomb(ces, csid, $1.hex), plane | $2.hex] else - raise "unknown notation '%s'"% l + raise "unknown notation '%s'"% l.chomp end end end @@ -823,11 +822,11 @@ TRANSCODERS = [] TRANSCODE_GENERATED_TRANSCODER_CODE = ''.dup def transcode_tbl_only(from, to, map, valid_encoding=UnspecifiedValidEncoding) - if VERBOSE_MODE + if VERBOSE_MODE > 1 if from.empty? || to.empty? - STDERR.puts "converter for #{from.empty? ? to : from}" + STDOUT.puts "converter for #{from.empty? ? to : from}" else - STDERR.puts "converter from #{from} to #{to}" + STDOUT.puts "converter from #{from} to #{to}" end end id_from = from.tr('^0-9A-Za-z', '_') @@ -843,7 +842,45 @@ def transcode_tbl_only(from, to, map, valid_encoding=UnspecifiedValidEncoding) return map, tree_name, real_tree_name, max_input end -def transcode_tblgen(from, to, map, valid_encoding=UnspecifiedValidEncoding) +# +# call-seq: +# transcode_tblgen(from_name, to_name, map [, valid_encoding_check [, ascii_compatibility]]) -> '' +# +# Returns an empty string just in case the result is used somewhere. +# Stores the actual product for later output with transcode_generated_code and +# transcode_register_code. +# +# The first argument is a string that will be used for the source (from) encoding. +# The second argument is a string that will be used for the target (to) encoding. +# +# The third argument is the actual data, a map represented as an array of two-element +# arrays. Each element of the array stands for one character being converted. The +# first element of each subarray is the code of the character in the source encoding, +# the second element of each subarray is the code of the character in the target encoding. +# +# Each code (i.e. byte sequence) is represented as a string of hexadecimal characters +# of even length. Codes can also be represented as integers (usually in the form Ox...), +# in which case they are interpreted as Unicode codepoints encoded in UTF-8. So as +# an example, 0x677E is the same as "E69DBE" (but somewhat easier to produce and check). +# +# In addition, the following symbols can also be used instead of actual codes in the +# second element of a subarray: +# :nomap (no mapping, just copy input to output), :nomap0 (same as :nomap, but low priority), +# :undef (input code undefined in the destination encoding), +# :invalid (input code is an invalid byte sequence in the source encoding), +# :func_ii, :func_si, :func_io, :func_so (conversion by function with specific call +# convention). +# +# The forth argument specifies the overall structure of the encoding. For examples, +# see ValidEncoding below. This is used to cross-check the data in the third argument +# and to automatically add :undef and :invalid mappings where necessary. +# +# The fifth argument gives the ascii-compatibility of the transcoding. See +# rb_transcoder_asciicompat_type_t in transcode_data.h for details. In most +# cases, this argument can be left out. +# +def transcode_tblgen(from, to, map, valid_encoding=UnspecifiedValidEncoding, + ascii_compatibility='asciicompat_converter') map, tree_name, real_tree_name, max_input = transcode_tbl_only(from, to, map, valid_encoding) transcoder_name = "rb_#{tree_name}" TRANSCODERS << transcoder_name @@ -857,10 +894,10 @@ static const rb_transcoder #{input_unit_length}, /* input_unit_length */ #{max_input}, /* max_input */ #{max_output}, /* max_output */ - asciicompat_converter, /* asciicompat_type */ - 0, NULL, NULL, /* state_size, state_init, state_fini */ - NULL, NULL, NULL, NULL, - NULL, NULL, NULL + #{ascii_compatibility}, /* asciicompat_type */ + 0, 0, 0, /* state_size, state_init, state_fini */ + 0, 0, 0, 0, + 0, 0, 0 }; End TRANSCODE_GENERATED_TRANSCODER_CODE << transcoder_code @@ -868,8 +905,8 @@ End end def transcode_generate_node(am, name_hint=nil) - STDERR.puts "converter for #{name_hint}" if VERBOSE_MODE - name = am.gennode(TRANSCODE_GENERATED_BYTES_CODE, TRANSCODE_GENERATED_WORDS_CODE, name_hint) + STDOUT.puts "converter for #{name_hint}" if VERBOSE_MODE > 1 + am.gennode(TRANSCODE_GENERATED_BYTES_CODE, TRANSCODE_GENERATED_WORDS_CODE, name_hint) '' end @@ -981,12 +1018,12 @@ if __FILE__ == $0 start_time = Time.now output_filename = nil - verbose_mode = false + verbose_mode = 0 force_mode = false op = OptionParser.new op.def_option("--help", "show help message") { puts op; exit 0 } - op.def_option("--verbose", "verbose mode") { verbose_mode = true } + op.def_option("--verbose", "verbose mode, twice for more verbose") { verbose_mode += 1 } op.def_option("--force", "force table generation") { force_mode = true } op.def_option("--output=FILE", "specify output file") {|arg| output_filename = arg } op.parse! @@ -994,7 +1031,7 @@ if __FILE__ == $0 VERBOSE_MODE = verbose_mode OUTPUT_FILENAME = output_filename - OUTPUT_PREFIX = output_filename ? File.basename(output_filename)[/\A[A-Za-z0-9_]*/] : "" + OUTPUT_PREFIX = output_filename ? File.basename(output_filename)[/\A[A-Za-z0-9_]*/] : "".dup OUTPUT_PREFIX.sub!(/\A_+/, '') OUTPUT_PREFIX.sub!(/_*\z/, '_') @@ -1029,19 +1066,19 @@ if __FILE__ == $0 if old_signature == chk_signature now = Time.now File.utime(now, now, output_filename) - STDERR.puts "already up-to-date: #{output_filename}" if VERBOSE_MODE + STDOUT.puts "already up-to-date: #{output_filename}" if VERBOSE_MODE > 0 exit end end - if VERBOSE_MODE + if VERBOSE_MODE > 0 if output_filename - STDERR.puts "generating #{output_filename} ..." + STDOUT.puts "generating #{output_filename} ..." end end libs1 = $".dup - erb = ERB.new(src, nil, '%') + erb = ERB.new(src, trim_mode: '%') erb.filename = arg erb_result = erb.result(binding) libs2 = $".dup @@ -1070,7 +1107,7 @@ if __FILE__ == $0 File.rename(new_filename, output_filename) tms = Process.times elapsed = Time.now - start_time - STDERR.puts "done. (#{'%.2f' % tms.utime}user #{'%.2f' % tms.stime}system #{'%.2f' % elapsed}elapsed)" if VERBOSE_MODE + STDOUT.puts "done. (#{'%.2f' % tms.utime}user #{'%.2f' % tms.stime}system #{'%.2f' % elapsed}elapsed)" if VERBOSE_MODE > 1 else print result end diff --git a/tool/travis_retry.sh b/tool/travis_retry.sh new file mode 100755 index 0000000000..9b79c56550 --- /dev/null +++ b/tool/travis_retry.sh @@ -0,0 +1,13 @@ +#!/bin/sh -eu +# The modified version of `travis_retry` to support custom backoffs, which is used by .travis.yml. +# https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_retry.bash + +for sleep in 0 ${WAITS:- 1 25 100}; do + sleep "$sleep" + + echo "+ $@" + if "$@"; then + exit 0 + fi +done +exit 1 diff --git a/tool/travis_wait.sh b/tool/travis_wait.sh new file mode 100755 index 0000000000..471b765df8 --- /dev/null +++ b/tool/travis_wait.sh @@ -0,0 +1,18 @@ +#!/bin/bash -eu +# The modified version of `travis_wait` to output a log as the command goes. +# https://github.com/travis-ci/travis-ci/issues/4190#issuecomment-353342526 + +# Produce an output log every 9 minutes as the timeout without output is 10 +# minutes. A job finishes with a timeout if it takes longer than 50 minutes. +# https://docs.travis-ci.com/user/customizing-the-build#build-timeouts +while sleep 9m; do + # Print message with bash variable SECONDS. + echo "====[ $SECONDS seconds still running ]====" +done & + +echo "+ $@" +"$@" + +jobs +kill %1 +exit 0 diff --git a/tool/update-NEWS-gemlist.rb b/tool/update-NEWS-gemlist.rb new file mode 100755 index 0000000000..0b5503580d --- /dev/null +++ b/tool/update-NEWS-gemlist.rb @@ -0,0 +1,52 @@ +#!/usr/bin/env ruby +require 'json' +news = File.read("NEWS.md") +prev = news[/since the \*+(\d+\.\d+\.\d+)\*+/, 1] +prevs = [prev, prev.sub(/\.\d+\z/, '')] + +update = ->(list, type, desc = "updated") do + item = ->(mark = "* ") do + "### The following #{type} gem#{list.size == 1 ? ' is' : 's are'} #{desc}.\n\n" + + list.map {|g, v|"#{mark}#{g} #{v}\n"}.join("") + "\n" + end + news.sub!(/^(?:\*( +)|#+ *)?The following #{type} gems? (?:are|is) #{desc}\.\n+(?:(?(1) \1)\*( *).*\n)*\n*/) do + item["#{$1&.<< " "}*#{$2 || ' '}"] + end or news.sub!(/^## Stdlib updates(?:\n+The following.*(?:\n+( *\* *).*)*)*\n+\K/) do + item[$1 || "* "] + end +end + +load_gems_json = ->(type) do + JSON.parse(File.read("#{type}_gems.json"))['gems'].filter_map do |g| + v = g['versions'].values_at(*prevs).compact.first + g = g['gem'] + g = 'RubyGems' if g == 'rubygems' + [g, v] if v + end.to_h +end + +ARGV.each do |type| + last = load_gems_json[type] + changed = File.foreach("gems/#{type}_gems").filter_map do |l| + next if l.start_with?("#") + g, v = l.split(" ", 3) + next unless v + [g, v] unless last[g] == v + end + changed, added = changed.partition {|g, _| last[g]} + update[changed, type] or next + if added and !added.empty? + if type == 'bundled' + default_gems = load_gems_json['default'] + promoted = {} + added.delete_if do |k, v| + default_gems.key?(k) && promoted[k] = v + end + update[added, type, 'added'] + update[promoted, type, 'promoted from default gems'] or next + else + update[added, type, 'added'] or next + end + end + File.write("NEWS.md", news) +end diff --git a/tool/update-NEWS-refs.rb b/tool/update-NEWS-refs.rb new file mode 100644 index 0000000000..f48cac5ee1 --- /dev/null +++ b/tool/update-NEWS-refs.rb @@ -0,0 +1,38 @@ +# Usage: ruby tool/update-NEWS-refs.rb + +orig_src = File.read(File.join(__dir__, "../NEWS.md")) +lines = orig_src.lines(chomp: true) + +links = {} +while lines.last =~ %r{\A\[(.*?)\]:\s+(.*)\z} + links[$1] = $2 + lines.pop +end + +if links.empty? || lines.last != "" + raise "NEWS.md must end with a sequence of links" +end + +trackers = ["Feature", "Bug", "Misc"] +labels = links.keys.reject {|k| k.start_with?(*trackers)} +new_src = lines.join("\n").gsub(/\[?\[(#{Regexp.union(trackers)}\s+#(\d+))\]\]?/) do + links[$1] ||= "https://bugs.ruby-lang.org/issues/#$2" + "[[#$1]]" +end.gsub(/\[\[#{Regexp.union(labels)}\]\]?/) do + "[#$1]" +end.chomp + "\n\n" + +label_width = links.max_by {|k, _| k.size}.first.size + 4 +redmine_links, non_redmine_links = links.partition {|k,| k =~ /\A#{Regexp.union(trackers)}\s+#\d+\z/ } + +(redmine_links.sort_by {|k,| k[/\d+/].to_i } + non_redmine_links.reverse).each do |k, v| + new_src << "[#{k}]:".ljust(label_width) << v << "\n" +end + +if orig_src != new_src + print "Update NEWS.md? [y/N]" + $stdout.flush + if gets.chomp == "y" + File.write(File.join(__dir__, "../NEWS.md"), new_src) + end +end diff --git a/tool/update-bundled_gems.rb b/tool/update-bundled_gems.rb new file mode 100755 index 0000000000..dec6b49cee --- /dev/null +++ b/tool/update-bundled_gems.rb @@ -0,0 +1,41 @@ +#!ruby -alpF\s+|#.* +BEGIN { + require 'rubygems' + date = nil + # STDOUT is not usable in inplace edit mode + output = $-i ? STDOUT : STDERR +} +output = STDERR if ARGF.file == STDIN +END { + output.print date.strftime("latest_date=%F") if date +} +if gem = $F[0] + ver = Gem::Version.new($F[1]) + (gem, src), = Gem::SpecFetcher.fetcher.detect(:latest) {|s| + s.platform == "ruby" && s.name == gem + } + if gem.version > ver + gem = src.fetch_spec(gem) + if ENV["UPDATE_BUNDLED_GEMS_ALL"] + uri = gem.metadata["source_code_uri"] || gem.homepage + uri = uri.sub(%r[\Ahttps://github\.com/[^/]+/[^/]+\K/tree/.*], "").chomp(".git") + else + uri = $F[2] + end + if (!date or gem.date && gem.date > date) and gem.date.to_i != 315_619_200 + # DEFAULT_SOURCE_DATE_EPOCH is meaningless + date = gem.date + end + if $F[3] + if $F[3].include?($F[1]) + $F[3][$F[1]] = gem.version.to_s + elsif Gem::Version.new($F[1]) != gem.version and /\A\h+\z/ =~ $F[3] + $F[3..-1] = [] + end + end + f = [gem.name, gem.version.to_s, uri, *$F[3..-1]] + $_.gsub!(/\S+\s*(?=\s|$)/) {|s| (f.shift || "").ljust(s.size)} + $_ = [$_, *f].join(" ") unless f.empty? + $_.rstrip! + end +end diff --git a/tool/update-deps b/tool/update-deps index 4c20052263..2d4a5674be 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -13,14 +13,22 @@ # 1. Compile ruby with -save-temps=obj option. # Ex. ./configure debugflags='-save-temps=obj -g' && make all golf # 2. Run tool/update-deps to show dependency problems. -# Ex. ruby tool/update-deps +# Ex. ./ruby tool/update-deps # 3. Use --fix to fix makefiles. -# Ex. ruby tool/update-deps --fix +# Ex. ./ruby tool/update-deps --fix +# +# Usage to create a depend file initially: +# 1. Copy the dependency section from the Makefile generated by extconf.rb. +# Ex. ext/cgi/escape/Makefile +# 2. Add `# AUTOGENERATED DEPENDENCIES START` and `# AUTOGENERATED DEPENDENCIES END` +# sections to top and end of the depend file. +# 3. Run tool/update-deps --fix to fix the depend file. +# 4. Commit the depend file. # # Other usages: # * Fix makefiles using previously detected dependency problems # Ex. ruby tool/update-deps --actual-fix [file] -# "ruby tool/update-deps --fix" is same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix". +# "ruby tool/update-deps --fix" is the same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix". require 'optparse' require 'stringio' @@ -28,7 +36,7 @@ require 'pathname' require 'open3' require 'pp' -# When out-of-place bulid, files may be built in source directory or +# When out-of-place build, files may be built in source directory or # build directory. # Some files are always built in the source directory. # Some files are always built in the build directory. @@ -42,7 +50,7 @@ REV=48577 tar xf ruby-$VER-r$REV.tar.xz cp -a ruby-$VER-r$REV tarball_source_dir_original mv ruby-$VER-r$REV tarball_source_dir_after_build -svn co -q -r$REV http://svn.ruby-lang.org/repos/ruby/trunk ruby +svn co -q -r$REV https://svn.ruby-lang.org/repos/ruby/trunk ruby (cd ruby; autoconf) cp -a ruby repo_source_dir_original mv ruby repo_source_dir_after_build @@ -88,7 +96,15 @@ result.each {|k,v| # They can be referenced as $(top_srcdir)/filename. # % ruby -e 'def g(d) Dir.chdir(d) { Dir["**/*.{c,h,inc,dmyh}"] } end; puts((g("repo_source_dir_after_build") - g("repo_source_dir_original")).sort)' FILES_IN_SOURCE_DIRECTORY = %w[ - revision.h + prism/api_node.c + prism/ast.h + prism/diagnostic.c + prism/diagnostic.h + prism/node.c + prism/prettyprint.c + prism/serialize.c + prism/token_type.c + prism/version.h ] # Files built in the build directory (except extconf.h). @@ -112,6 +128,7 @@ FILES_NEED_VPATH = %w[ ext/ripper/eventids1.c ext/ripper/eventids2table.c ext/ripper/ripper.c + ext/ripper/ripper_init.c golf_prelude.c id.c id.h @@ -122,13 +139,12 @@ FILES_NEED_VPATH = %w[ miniprelude.c newline.c node_name.inc - opt_sc.inc optinsn.inc optunifs.inc parse.c parse.h - prelude.c probes.dmyh + revision.h vm.inc vmtc.inc @@ -156,26 +172,28 @@ FILES_NEED_VPATH = %w[ # It is not good idea to refer them using VPATH. # Files in FILES_SAME_NAME_INC is referenced using $(hdrdir). # Files in FILES_SAME_NAME_TOP is referenced using $(top_srcdir). -# include/ruby.h is referenced using $(top_srcdir) because mkmf.rb replaces -# $(hdrdir)/ruby.h to $(hdrdir)/ruby/ruby.h FILES_SAME_NAME_INC = %w[ + include/ruby.h include/ruby/ruby.h include/ruby/version.h ] FILES_SAME_NAME_TOP = %w[ - include/ruby.h version.h ] +# Files that may or may not exist on CI for some reason. +# Windows build generally seems to have missing dependencies. +UNSTABLE_FILES = %r{\Awin32/[^/]+\.o\z} + # Other source files exist in the source directory. def in_makefile(target, source) target = target.to_s source = source.to_s case target - when %r{\A[^/]*\z} + when %r{\A[^/]*\z}, %r{\Acoroutine/}, %r{\Aprism/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source when *FILES_IN_SOURCE_DIRECTORY then source2 = "$(top_srcdir)/#{source}" @@ -192,9 +210,10 @@ def in_makefile(target, source) when %r{\Amissing/} then source2 = "{$(VPATH)}#{$'}" when %r{\Accan/} then source2 = "$(CCAN_DIR)/#{$'}" when %r{\Adefs/} then source2 = "{$(VPATH)}#{source}" + when %r{\Acoroutine/} then source2 = "{$(VPATH)}$(COROUTINE_H)" else source2 = "$(top_srcdir)/#{source}" end - ["common.mk", target2, source2] + ["depend", target2, source2] when %r{\Aenc/} target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}" case source @@ -205,13 +224,15 @@ def in_makefile(target, source) when *FILES_SAME_NAME_TOP then source2 = "$(top_srcdir)/#{source}" when %r{\A\.ext/include/[^/]+/ruby/} then source2 = $' when %r{\Ainclude/ruby/} then source2 = $' + when %r{\Aenc/unicode/[\d.]+/} then source2 = '$(UNICODE_HDR_DIR)/' + $' when %r{\Aenc/} then source2 = source else source2 = "$(top_srcdir)/#{source}" end ["enc/depend", target2, source2] when %r{\Aext/} - unless File.exist?("#{File.dirname(target)}/extconf.rb") - warn "warning: not found: #{File.dirname(target)}/extconf.rb" + targetdir = File.dirname(target) + unless File.exist?("#{targetdir}/extconf.rb") + warn "warning: not found: #{targetdir}/extconf.rb" end target2 = File.basename(target) relpath = Pathname(source).relative_path_from(Pathname(target).dirname).to_s @@ -223,11 +244,16 @@ def in_makefile(target, source) when *FILES_SAME_NAME_TOP then source2 = "$(top_srcdir)/#{source}" when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "$(arch_hdrdir)/ruby/#{$'}" when %r{\Ainclude/} then source2 = "$(hdrdir)/#{$'}" - when %r{\A#{Regexp.escape File.dirname(target)}/extconf\.h\z} then source2 = "$(RUBY_EXTCONF_H)" - when %r{\A#{Regexp.escape File.dirname(target)}/} then source2 = $' + when %r{\A#{Regexp.escape targetdir}/extconf\.h\z} then source2 = "$(RUBY_EXTCONF_H)" + when %r{\A#{Regexp.escape targetdir}/} then source2 = $' + when %r{\A#{Regexp.escape File.dirname(targetdir)}/} then source2 = "$(srcdir)/../#{$'}" else source2 = "$(top_srcdir)/#{source}" end ["#{File.dirname(target)}/depend", target2, source2] + # Files that may or may not exist on CI for some reason. + # Windows build generally seems to have missing dependencies. + when UNSTABLE_FILES + warn "warning: ignoring: #{target}" else raise "unexpected target: #{target}" end @@ -238,6 +264,10 @@ DEPENDENCIES_SECTION_END_MARK = "\# AUTOGENERATED DEPENDENCIES END\n" def init_global ENV['LC_ALL'] = 'C' + if mkflag0 = ENV['GNUMAKEFLAGS'] and (mkflag = mkflag0.sub(/(\A|\s+)-j\d*(?=\s+|\z)/, '')) != mkflag0 + mkflag.strip! + ENV['GNUMAKEFLAGS'] = mkflag + end $opt_fix = false $opt_a = false @@ -256,7 +286,7 @@ end def read_make_deps(cwd) dependencies = {} - make_p, make_p_stderr, make_p_status = Open3.capture3("make -p all miniruby ruby golf") + make_p, make_p_stderr, make_p_status = Open3.capture3("make -p all miniruby exe/ruby golf") File.open('update-deps.make.out.log', 'w') {|f| f.print make_p } File.open('update-deps.make.err.log', 'w') {|f| f.print make_p_stderr } if !make_p_status.success? @@ -293,7 +323,12 @@ def read_make_deps(cwd) target = rule_target deps = rule_sources deps = deps.scan(%r{[/0-9a-zA-Z._-]+}) + deps.delete_if {|dep| /\.time\z/ =~ dep} # skip timestamp next if /\.o\z/ !~ target.to_s + next if /libyjit.o\z/ =~ target.to_s # skip YJIT Rust object (no corresponding C source) + next if /libzjit.o\z/ =~ target.to_s # skip ZJIT Rust object (no corresponding C source) + next if /target\/release\/libruby.o\z/ =~ target.to_s # skip YJIT+ZJIT Rust object (no corresponding C source) + next if /\.bundle\// =~ target.to_s next if /\A\./ =~ target.to_s # skip rules such as ".c.o" #p [curdir, target, deps] dir = curdir || dirstack.last @@ -325,23 +360,27 @@ end # raise ArgumentError, "can not find #{filename} (hint: #{hint0})" #end -def read_single_cc_deps(path_i, cwd) +def read_single_cc_deps(path_i, cwd, fn_o) files = {} - path_i.each_line.with_index {|line, lineindex| + compiler_wd = nil + path_i.each_line {|line| next if /\A\# \d+ "(.*)"/ !~ line - files[$1] = lineindex + dep = $1 + + next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc. + next if /\.e?rb\z/ =~ dep + # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. + if /\/\/\z/ =~ dep + compiler_wd = Pathname(dep.sub(%r{//\z}, '')) + next + end + + files[dep] = true } - # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line. - compiler_wd = files.keys.find {|f| %r{\A/.*//\z} =~ f } - if compiler_wd - files.delete compiler_wd - compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, '')) - else - raise "compiler working directory not found: #{path_i}" - end + compiler_wd ||= fn_o.to_s.start_with?("enc/") ? cwd : path_i.parent + deps = [] files.each_key {|dep| - next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc. dep = Pathname(dep) if dep.relative? dep = compiler_wd + dep @@ -353,6 +392,9 @@ def read_single_cc_deps(path_i, cwd) next if !dep.to_s.start_with?(cwd.to_s) # omit system headers. deps << dep } + if deps.include?(cwd + "probes.h") + deps << (cwd + "probes.dmyh") + end deps end @@ -361,13 +403,14 @@ def read_cc_deps(cwd) Pathname.glob('**/*.o').sort.each {|fn_o| fn_i = fn_o.sub_ext('.i') if !fn_i.exist? + next if fn_o.sub_ext('.S').exist? warn "warning: not found: #{fn_i}" $i_not_found = true next end path_o = cwd + fn_o path_i = cwd + fn_i - deps[path_o] = read_single_cc_deps(path_i, cwd) + deps[path_o] = read_single_cc_deps(path_i, cwd, fn_o) } deps end @@ -389,6 +432,8 @@ def concentrate(dependencies, cwd) if %r{\A\.\.(/|\z)} =~ s.to_s warn "warning: out of tree source: #{s}" true + elsif %r{/\.time\z} =~ s.to_s + true else false end @@ -453,7 +498,7 @@ def compare_deps(make_deps, cc_deps, out=$stdout) } } - makefiles.keys.sort.each {|makefile| + makefiles.keys.compact.sort.each {|makefile| cc_lines = cc_lines_hash[makefile] || Hash.new(false) make_lines = make_lines_hash[makefile] || Hash.new(false) content = begin @@ -506,7 +551,20 @@ def compare_deps(make_deps, cc_deps, out=$stdout) } end +def prepare_build + unless File.exist?("Makefile") + if File.exist?("autogen.sh") + system("./autogen.sh") + elsif !File.exist?("configure") + system("autoreconf", "-i", "-s") + end + system("./configure", "-q", "--enable-load-relative", "--prefix=/.", + "--disable-install-doc", "debugflags=-save-temps=obj -g") + end +end + def main_show(out=$stdout) + prepare_build make_deps, cc_deps = detect_dependencies(out) compare_deps(make_deps, cc_deps, out) end diff --git a/tool/vcs.rb b/tool/vcs.rb deleted file mode 100644 index e38d1cfc56..0000000000 --- a/tool/vcs.rb +++ /dev/null @@ -1,352 +0,0 @@ -# vcs -require 'fileutils' - -ENV.delete('PWD') - -unless File.respond_to? :realpath - require 'pathname' - def File.realpath(arg) - Pathname(arg).realpath.to_s - end -end - -def IO.pread(*args) - STDERR.puts(*args.inspect) if $DEBUG - popen(*args) {|f|f.read} -end - -if RUBY_VERSION < "1.9" - class IO - @orig_popen = method(:popen) - - if defined?(fork) - def self.popen(command, *rest, &block) - if Hash === (opts = rest[-1]) - dir = opts.delete(:chdir) - rest pop if opts.empty? - end - if block - @orig_popen.call("-", *rest) do |f| - if f - yield(f) - else - Dir.chdir(dir) if dir - exec(*command) - end - end - else - f = @orig_popen.call("-", *rest) - unless f - Dir.chdir(dir) if dir - exec(*command) - end - f - end - end - else - require 'shellwords' - def self.popen(command, *rest, &block) - if Hash === (opts = rest[-1]) - dir = opts.delete(:chdir) - rest pop if opts.empty? - end - command = command.shelljoin if Array === command - Dir.chdir(dir || ".") do - @orig_popen.call(command, *rest, &block) - end - end - end - end -end - -class VCS - class NotFoundError < RuntimeError; end - - @@dirs = [] - def self.register(dir) - @@dirs << [dir, self] - end - - def self.detect(path) - @@dirs.each do |dir, klass| - return klass.new(path) if File.directory?(File.join(path, dir)) - prev = path - loop { - curr = File.realpath(File.join(prev, '..')) - break if curr == prev # stop at the root directory - return klass.new(path) if File.directory?(File.join(curr, dir)) - prev = curr - } - end - raise VCS::NotFoundError, "does not seem to be under a vcs: #{path}" - end - - def self.local_path?(path) - String === path or path.respond_to?(:to_path) - end - - def initialize(path) - @srcdir = path - super() - end - - NullDevice = defined?(IO::NULL) ? IO::NULL : - %w[/dev/null NUL NIL: NL:].find {|dev| File.exist?(dev)} - - # return a pair of strings, the last revision and the last revision in which - # +path+ was modified. - def get_revisions(path) - if self.class.local_path?(path) - path = relative_to(path) - end - last, changed, modified, *rest = ( - begin - if NullDevice - save_stderr = STDERR.dup - STDERR.reopen NullDevice, 'w' - end - self.class.get_revisions(path, @srcdir) - ensure - if save_stderr - STDERR.reopen save_stderr - save_stderr.close - end - end - ) - last or raise VCS::NotFoundError, "last revision not found" - changed or raise VCS::NotFoundError, "changed revision not found" - if modified - /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ modified or - raise "unknown time format - #{modified}" - match = $~[1..6].map { |x| x.to_i } - off = $7 ? "#{$7}:#{$8}" : "+00:00" - match << off - begin - modified = Time.new(*match) - rescue ArgumentError - modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60 - end - end - return last, changed, modified, *rest - end - - def modified(path) - last, changed, modified, *rest = get_revisions(path) - modified - end - - def relative_to(path) - if path - srcdir = File.realpath(@srcdir) - path = File.realdirpath(path) - list1 = srcdir.split(%r{/}) - list2 = path.split(%r{/}) - while !list1.empty? && !list2.empty? && list1.first == list2.first - list1.shift - list2.shift - end - if list1.empty? && list2.empty? - "." - else - ([".."] * list1.length + list2).join("/") - end - else - '.' - end - end - - def after_export(dir) - end - - class SVN < self - register(".svn") - - def self.get_revisions(path, srcdir = nil) - if srcdir and local_path?(path) - path = File.join(srcdir, path) - end - if srcdir - info_xml = IO.pread(%W"svn info --xml #{srcdir}") - info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s - end - info_xml ||= IO.pread(%W"svn info --xml #{path}") - _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) - modified = info_xml[/<date>([^<>]*)/, 1] - branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] - [last, changed, modified, branch] - end - - def self.search_root(path) - return unless local_path?(path) - parent = File.realpath(path) - begin - parent = File.dirname(wkdir = parent) - return wkdir if File.directory?(wkdir + "/.svn") - end until parent == wkdir - end - - def get_info - @info ||= IO.pread(%W"svn info --xml #{@srcdir}") - end - - def url - unless @url - url = get_info[/<root>(.*)<\/root>/, 1] - @url = URI.parse(url+"/") if url - end - @url - end - - def wcroot - unless @wcroot - info = get_info - @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] - @wcroot ||= self.class.search_root(@srcdir) - end - @wcroot - end - - def branch(name) - url + "branches/#{name}" - end - - def tag(name) - url + "tags/#{name}" - end - - def trunk - url + "trunk" - end - - def branch_list(pat) - IO.popen(%W"svn ls #{branch('')}") do |f| - f.each do |line| - line.chomp! - line.chomp!('/') - yield(line) if File.fnmatch?(pat, line) - end - end - end - - def grep(pat, tag, *files, &block) - cmd = %W"svn cat" - files.map! {|n| File.join(tag, n)} if tag - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files]) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end - - def export(revision, url, dir, keep_temp = false) - if @srcdir and (rootdir = wcroot) - srcdir = File.realpath(@srcdir) - rootdir << "/" - if srcdir.start_with?(rootdir) - subdir = srcdir[rootdir.size..-1] - subdir = nil if subdir.empty? - FileUtils.mkdir_p(svndir = dir+"/.svn") - FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) - system("svn", "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false - FileUtils.rm_rf(svndir) unless keep_temp - if subdir - tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") - File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") - FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) - begin - Dir.rmdir("#{dir}/#{subdir}") - end until (subdir = File.dirname(subdir)) == '.' - FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) - Dir.rmdir(tmpdir) - end - return true - end - end - IO.popen(%W"svn export -r #{revision} #{url} #{dir}") do |pipe| - pipe.each {|line| /^A/ =~ line or yield line} - end - $?.success? - end - - def after_export(dir) - FileUtils.rm_rf(dir+"/.svn") - end - end - - class GIT < self - register(".git") - - def self.get_revisions(path, srcdir = nil) - gitcmd = %W[git] - logcmd = gitcmd + %W[log -n1 --date=iso] - logcmd << "--grep=^ *git-svn-id: .*@[0-9][0-9]*" - idpat = /git-svn-id: .*?@(\d+) \S+\Z/ - log = IO.pread(logcmd, :chdir => srcdir) - commit = log[/\Acommit (\w+)/, 1] - last = log[idpat, 1] - if path - cmd = logcmd - cmd += [path] unless path == '.' - log = IO.pread(cmd, :chdir => srcdir) - changed = log[idpat, 1] - else - changed = last - end - modified = log[/^Date:\s+(.*)/, 1] - branch = IO.pread(gitcmd + %W[symbolic-ref HEAD], :chdir => srcdir)[%r'\A(?:refs/heads/)?(.+)', 1] - title = IO.pread(gitcmd + %W[log --format=%s -n1 #{commit}..HEAD], :chdir => srcdir) - title = nil if title.empty? - [last, changed, modified, branch, title] - end - - Branch = Struct.new(:to_str) - - def branch(name) - Branch.new(name) - end - - alias tag branch - - def trunk - branch("trunk") - end - - def stable - cmd = %W"git for-each-ref --format=\%(refname:short) refs/heads/ruby_[0-9]*" - branch(IO.pread(cmd, :chdir => srcdir)[/.*^(ruby_\d+_\d+)$/m, 1]) - end - - def branch_list(pat) - cmd = %W"git for-each-ref --format=\%(refname:short) refs/heads/#{pat}" - IO.popen(cmd, :chdir => srcdir) {|f| - f.each {|line| - line.chomp! - yield line - } - } - end - - def grep(pat, tag, *files, &block) - cmd = %W[git grep -h --perl-regexp #{tag} --] - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files], :chdir => srcdir) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end - - def export(revision, url, dir, keep_temp = false) - ret = system("git", "clone", "-s", (@srcdir || '.'), "-b", url, dir) - FileUtils.rm_rf("#{dir}/.git") if ret and !keep_temp - ret - end - - def after_export(dir) - FileUtils.rm_rf("#{dir}/.git") - end - end -end diff --git a/tool/vtlh.rb b/tool/vtlh.rb index fcd3630821..2e1faf2ce8 100644 --- a/tool/vtlh.rb +++ b/tool/vtlh.rb @@ -1,3 +1,5 @@ +# Convert addresses to line numbers for MiniRuby. + # ARGF = open('ha') cd = `pwd`.chomp + '/' ARGF.each{|line| diff --git a/tool/wasm-clangw b/tool/wasm-clangw new file mode 100755 index 0000000000..9ebdfda75a --- /dev/null +++ b/tool/wasm-clangw @@ -0,0 +1,9 @@ +#!/bin/sh +# A Clang wrapper script to fake the clang linker driver. +# Clang linker automatically uses wasm-opt with -O if it found. +# However optimization before asyncify causes misoptimization, +# so wrap clang to insert our fake wasm-opt, which does nothing, in PATH. + +src_dir="$(cd "$(dirname "$0")/../wasm" && pwd)" +export PATH="$src_dir:$PATH" +exec "$@" diff --git a/tool/ytab.sed b/tool/ytab.sed deleted file mode 100755 index 46317db284..0000000000 --- a/tool/ytab.sed +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sed -f -/^int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^extern int yydebug;/{ -i\ -#ifndef yydebug -a\ -#endif -} -/^yydestruct.*yymsg/,/#endif/{ - /^yydestruct/{ - /parser/!{ - h - s/^/ruby_parser_&/ - s/)$/, parser)/ - /\*/s/parser)$/struct parser_params *&/ - } - } - /^#endif/{ - x - /^./{ - i\ - struct parser_params *parser; - a\ -#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, parser) - } - x - } -} -s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/ -s!^ *extern char \*getenv();!/* & */! -s/^\(#.*\)".*\.tab\.c"/\1"parse.c"/ -/^\(#.*\)".*\.y"/s:\\\\:/:g diff --git a/tool/zjit_bisect.rb b/tool/zjit_bisect.rb new file mode 100755 index 0000000000..997c572f51 --- /dev/null +++ b/tool/zjit_bisect.rb @@ -0,0 +1,158 @@ +#!/usr/bin/env ruby +require 'logger' +require 'optparse' +require 'shellwords' +require 'tempfile' +require 'timeout' + +ARGS = {timeout: 5} +OptionParser.new do |opts| + opts.banner += " <path_to_ruby> -- <options>" + opts.on("--timeout=TIMEOUT_SEC", "Seconds until child process is killed") do |timeout| + ARGS[:timeout] = Integer(timeout) + end + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end +end.parse! + +usage = "Usage: zjit_bisect.rb <path_to_ruby> -- <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +raise(usage) if OPTIONS.empty? +LOGGER = Logger.new($stdout) + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +# Attempt to reduce the `items` argument as much as possible, returning the +# shorter version. `fixed` will always be used as part of the items when +# running `command`. +# `command` should return True if the command succeeded (the failure did not +# reproduce) and False if the command failed (the failure reproduced). +def bisect_impl(command, fixed, items, indent="") + LOGGER.info("#{indent}step fixed[#{fixed.length}] and items[#{items.length}]") + while items.length > 1 + LOGGER.info("#{indent}#{fixed.length + items.length} candidates") + # Return two halves of the given list. For odd-length lists, the second + # half will be larger. + half = items.length / 2 + left = items[0...half] + right = items[half..] + if !command.call(fixed + left) + items = left + next + end + if !command.call(fixed + right) + items = right + next + end + # We need something from both halves to trigger the failure. Try + # holding each half fixed and bisecting the other half to reduce the + # candidates. + new_right = bisect_impl(command, fixed + left, right, indent + "< ") + new_left = bisect_impl(command, fixed + new_right, left, indent + "> ") + return new_left + new_right + end + items +end + +# From https://github.com/tekknolagi/omegastar +# MIT License +# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms +def run_bisect(command, items) + LOGGER.info("Verifying items") + if command.call(items) + raise StandardError.new("Command succeeded with full items") + end + if !command.call([]) + raise StandardError.new("Command failed with empty items") + end + bisect_impl(command, [], items) +end + +def add_zjit_options cmd + if RUBY == "make" + # Automatically detect that we're running a make command instead of a Ruby + # one. Pass the bisection options via RUN_OPTS/SPECOPTS instead. + zjit_opts = cmd.select { |arg| arg.start_with?("--zjit") } + run_opts_index = cmd.find_index { |arg| arg.start_with?("RUN_OPTS=") } + specopts_index = cmd.find_index { |arg| arg.start_with?("SPECOPTS=") } + if run_opts_index + run_opts = Shellwords.split(cmd[run_opts_index].delete_prefix("RUN_OPTS=")) + run_opts.concat(zjit_opts) + cmd[run_opts_index] = "RUN_OPTS=#{run_opts.shelljoin}" + elsif specopts_index + specopts = Shellwords.split(cmd[specopts_index].delete_prefix("SPECOPTS=")) + specopts.concat(zjit_opts) + cmd[specopts_index] = "SPECOPTS=#{specopts.shelljoin}" + else + raise "Expected RUN_OPTS or SPECOPTS to be present in make command" + end + cmd = cmd - zjit_opts + end + cmd +end + +def run_ruby *cmd + cmd = add_zjit_options(cmd) + pid = Process.spawn(*cmd, { + in: :close, + out: [File::NULL, File::RDWR], + err: [File::NULL, File::RDWR], + }) + begin + status = Timeout.timeout(ARGS[:timeout]) do + Process::Status.wait(pid) + end + rescue Timeout::Error + Process.kill("KILL", pid) + LOGGER.warn("Timed out after #{ARGS[:timeout]} seconds") + status = Process::Status.wait(pid) + end + + status +end + +def run_with_jit_list(ruby, options, jit_list) + # Make a new temporary file containing the JIT list + Tempfile.create("jit_list") do |temp_file| + temp_file.write(jit_list.join("\n")) + temp_file.flush + temp_file.close + # Run the JIT with the temporary file + run_ruby ruby, "--zjit-allowed-iseqs=#{temp_file.path}", *options + end +end + +# Try running with no JIT list to get a stable baseline +unless run_with_jit_list(RUBY, OPTIONS, []).success? + cmd = [RUBY, "--zjit-allowed-iseqs=/dev/null", *OPTIONS].shelljoin + raise "The command failed unexpectedly with an empty JIT list. To reproduce, try running the following: `#{cmd}`" +end +# Collect the JIT list from the failing Ruby process +jit_list = nil +Tempfile.create "jit_list" do |temp_file| + run_ruby RUBY, "--zjit-log-compiled-iseqs=#{temp_file.path}", *OPTIONS + jit_list = File.readlines(temp_file.path).map(&:strip).reject(&:empty?) +end +LOGGER.info("Starting with JIT list of #{jit_list.length} items.") +# Try running without the optimizer +status = run_with_jit_list(RUBY, ["--zjit-disable-hir-opt", *OPTIONS], jit_list) +if status.success? + LOGGER.warn "*** Command suceeded with HIR optimizer disabled. HIR optimizer is probably at fault. ***" +end +# Now narrow it down +command = lambda do |items| + run_with_jit_list(RUBY, OPTIONS, items).success? +end +result = run_bisect(command, jit_list) +File.open("jitlist.txt", "w") do |file| + file.puts(result) +end +puts "Run:" +jitlist_path = File.expand_path("jitlist.txt") +puts add_zjit_options([RUBY, "--zjit-allowed-iseqs=#{jitlist_path}", *OPTIONS]).shelljoin +puts "Reduced JIT list (available in jitlist.txt):" +puts result diff --git a/tool/zjit_iongraph.html b/tool/zjit_iongraph.html new file mode 100644 index 0000000000..993cce9045 --- /dev/null +++ b/tool/zjit_iongraph.html @@ -0,0 +1,551 @@ +<!-- Copyright Mozilla and licensed under Mozilla Public License Version 2.0. + Source can be found at https://github.com/mozilla-spidermonkey/iongraph --> +<!-- Generated by `npm run build-www` on + 39b04fa18f23cbf3fd2ca7339a45341ff3351ba1 in tekknolagi/iongraph --> +<!DOCTYPE html> + +<head> + <title>iongraph</title> + <style>/* iongraph-specific styles inspired by Tachyons */ + +:root { + --ig-size-1: 1rem; + --ig-size-2: 2rem; + --ig-size-3: 4rem; + --ig-size-4: 8rem; + --ig-size-5: 16rem; + + --ig-spacing-1: .25rem; + --ig-spacing-2: .5rem; + --ig-spacing-3: 1rem; + --ig-spacing-4: 2rem; + --ig-spacing-5: 4rem; + --ig-spacing-6: 8rem; + --ig-spacing-7: 16rem; + + --ig-text-color: black; + --ig-text-color-dim: #777; + --ig-background-primary: #ffb54e; + --ig-background-light: white; + --ig-border-color: #0c0c0d; + + --ig-block-header-color: #0c0c0d; + --ig-loop-header-color: #1fa411; + --ig-movable-color: #1048af; + --ig-rob-color: #444; + --ig-in-worklist-color: red; + + --ig-block-selected: #ffc863; + --ig-block-last-selected: #ffb54e; + + --ig-highlight-0: #ffb54e; + --ig-highlight-1: #ffb5c5; + --ig-highlight-2: #a4cbff; + --ig-highlight-3: #8be182; + --ig-highlight-4: #d9a4fd; + + --ig-flash-color: #ffb54e; + + /* + * The heatmap of sample counts will effectively be sampled from a gradient: + * + * |----------|---------------------------------------| + * cold "cool" hot + * + * The "cold" color will simply be transparent. Therefore, the "cool" + * threshold indicates where the instruction will be fully colored and + * noticeable to the user. + */ + --ig-hot-color: #ff849e; + --ig-cool-color: #ffe546; + --ig-cool-threshold: 0.2; +} + +a.ig-link-normal { + color: inherit; + text-decoration: inherit; +} + +.ig-flex { + display: flex; +} + +.ig-flex-column { + flex-direction: column; +} + +.ig-flex-basis-0 { + flex-basis: 0; +} + +.ig-flex-grow-1 { + flex-grow: 1; +} + +.ig-flex-shrink-0 { + flex-shrink: 0; +} + +.ig-flex-shrink-1 { + flex-shrink: 1; +} + +.ig-items-center { + align-items: center; +} + +.ig-relative { + position: relative; +} + +.ig-absolute { + position: absolute; +} + +.ig-absolute-fill { + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.ig-g1 { + gap: var(--ig-spacing-1); +} + +.ig-g2 { + gap: var(--ig-spacing-2); +} + +.ig-g3 { + gap: var(--ig-spacing-3); +} + +.ig-w1 { + width: var(--ig-size-1); +} + +.ig-w2 { + width: var(--ig-size-2); +} + +.ig-w3 { + width: var(--ig-size-3); +} + +.ig-w4 { + width: var(--ig-size-4); +} + +.ig-w5 { + width: var(--ig-size-5); +} + +.ig-w-100 { + width: 100%; +} + +.ig-ba { + border-style: solid; + border-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bt { + border-top-style: solid; + border-top-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-br { + border-right-style: solid; + border-right-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bb { + border-bottom-style: solid; + border-bottom-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-bl { + border-left-style: solid; + border-left-width: 1px; + border-color: var(--ig-border-color); +} + +.ig-pa1 { + padding: var(--ig-spacing-1); +} + +.ig-pa2 { + padding: var(--ig-spacing-2); +} + +.ig-pa3 { + padding: var(--ig-spacing-3); +} + +.ig-ph1 { + padding-left: var(--ig-spacing-1); + padding-right: var(--ig-spacing-1); +} + +.ig-ph2 { + padding-left: var(--ig-spacing-2); + padding-right: var(--ig-spacing-2); +} + +.ig-ph3 { + padding-left: var(--ig-spacing-3); + padding-right: var(--ig-spacing-3); +} + +.ig-pv1 { + padding-top: var(--ig-spacing-1); + padding-bottom: var(--ig-spacing-1); +} + +.ig-pv2 { + padding-top: var(--ig-spacing-2); + padding-bottom: var(--ig-spacing-2); +} + +.ig-pv3 { + padding-top: var(--ig-spacing-3); + padding-bottom: var(--ig-spacing-3); +} + +.ig-pt1 { + padding-top: var(--ig-spacing-1); +} + +.ig-pt2 { + padding-top: var(--ig-spacing-2); +} + +.ig-pt3 { + padding-top: var(--ig-spacing-3); +} + +.ig-pr1 { + padding-right: var(--ig-spacing-1); +} + +.ig-pr2 { + padding-right: var(--ig-spacing-2); +} + +.ig-pr3 { + padding-right: var(--ig-spacing-3); +} + +.ig-pb1 { + padding-bottom: var(--ig-spacing-1); +} + +.ig-pb2 { + padding-bottom: var(--ig-spacing-2); +} + +.ig-pb3 { + padding-bottom: var(--ig-spacing-3); +} + +.ig-pl1 { + padding-left: var(--ig-spacing-1); +} + +.ig-pl2 { + padding-left: var(--ig-spacing-2); +} + +.ig-pl3 { + padding-left: var(--ig-spacing-3); +} + +.ig-f1 { + font-size: 3rem; +} + +.ig-f2 { + font-size: 2.25rem; +} + +.ig-f3 { + font-size: 1.5rem; +} + +.ig-f4 { + font-size: 1.25rem; +} + +.ig-f5 { + font-size: 1rem; +} + +.ig-f6 { + font-size: .875rem; +} + +.ig-f7 { + font-size: .75rem; +} + +.ig-text-normal { + color: var(--ig-text-color); +} + +.ig-text-dim { + color: var(--ig-text-color-dim); +} + +.ig-tl { + text-align: left; +} + +.ig-tr { + text-align: right; +} + +.ig-tc { + text-align: center; +} + +.ig-bg-white { + background-color: var(--ig-background-light); +} + +.ig-bg-primary { + background-color: var(--ig-background-primary); +} + +.ig-overflow-hidden { + overflow: hidden; +} + +.ig-overflow-auto { + overflow: auto; +} + +.ig-overflow-x-auto { + overflow-x: auto; +} + +.ig-overflow-y-auto { + overflow-y: auto; +} + +.ig-hide-if-empty:empty { + display: none; +} + +/* Non-utility styles */ + +.ig-graph { + color: var(--ig-text-color); + position: absolute; + left: 0; + top: 0; + isolation: isolate; +} + +.ig-block { + position: absolute; + + .ig-block-header { + font-weight: bold; + text-align: center; + background-color: var(--ig-block-header-color); + color: white; + padding: 0 1em; + border: 1px solid var(--ig-border-color); + border-width: 1px 1px 0; + } + + .ig-instructions { + padding: 0.5em; + border: 1px solid var(--ig-border-color); + border-width: 0 1px 1px; + + table { + border-collapse: collapse; + } + + td, + th { + white-space: nowrap; + padding: 0.1em 0.5em; + } + + th { + font-weight: normal; + } + } + + &.ig-selected { + outline: 4px solid var(--ig-block-selected); + } + + &.ig-last-selected { + outline-color: var(--ig-block-last-selected); + } +} + +.ig-block-att-loopheader { + .ig-block-header { + background-color: var(--ig-loop-header-color); + } +} + +.ig-block-att-splitedge { + .ig-instructions { + border-style: dotted; + border-width: 0 2px 2px; + } +} + +.ig-ins-num { + text-align: right; + cursor: pointer; +} + +.ig-ins-type { + text-align: right; +} + +.ig-ins-samples { + font-size: 0.875em; + text-align: right; + cursor: pointer; +} + +.ig-use { + padding: 0 0.25em; + border-radius: 2px; + cursor: pointer; +} + +.ig-edge-label { + position: absolute; + font-size: 0.8em; + line-height: 1; + bottom: -1em; + padding-left: 4px; +} + +.ig-ins-att-RecoveredOnBailout { + color: var(--ig-rob-color); +} + +.ig-ins-att-Movable { + color: var(--ig-movable-color); +} + +.ig-ins-att-Guard { + text-decoration: underline; +} + +.ig-ins-att-InWorklist { + color: var(--ig-in-worklist-color); +} + +.ig-can-flash { + transition: outline-color 1s ease-out; + outline: 3px solid color-mix(in srgb, var(--ig-flash-color) 0%, transparent); +} + +.ig-flash { + transition: outline-color 0s; + outline-color: var(--ig-flash-color); +} + +.ig-hotness { + --ig-hotness: 0; + + --ig-cold-color: color-mix(in srgb, var(--ig-cool-color) 20%, transparent); + background-color: + /* cool <-> hot */ + color-mix(in oklab, + /* cold <-> cool */ + color-mix(in oklab, + /* dead or cold */ + color-mix(in srgb, transparent, var(--ig-cold-color) clamp(0%, calc(var(--ig-hotness) * 100000000%), 100%)), + var(--ig-cool-color) clamp(0%, calc((var(--ig-hotness) / var(--ig-cool-threshold)) * 100%), 100%)), + var(--ig-hot-color) clamp(0%, calc(((var(--ig-hotness) - var(--ig-cool-threshold)) / (1 - var(--ig-cool-threshold))) * 100%), 100%)); +} + +.ig-highlight { + --ig-highlight-color: transparent; + background-color: var(--ig-highlight-color); +}</style> + <style>* { + box-sizing: border-box; +} + +:root { + font-size: 0.875rem; +} + +body { + margin: 0; + background-color: #e5e8ea; + /* Font, and many other styles, taken from the Firefox Profiler/ */ + font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen-Sans, Ubuntu, Noto Sans, Liberation Sans, Cantarell, Helvetica Neue, sans-serif; +} + +#container { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +.tweaks-panel { + position: absolute; + bottom: 0; + right: 0; + padding: 1rem; + border: 1px solid black; + border-width: 1px 0 0 1px; + background-color: white; +}</style> +</head> + +<body> + <script>"use strict";var iongraph=(()=>{var J=Object.defineProperty;var mt=Object.getOwnPropertyDescriptor;var gt=Object.getOwnPropertyNames;var ft=Object.prototype.hasOwnProperty;var bt=(a,t)=>{for(var e in t)J(a,e,{get:t[e],enumerable:!0})},kt=(a,t,e,s)=>{if(t&&typeof t=="object"||typeof t=="function")for(let i of gt(t))!ft.call(a,i)&&i!==e&&J(a,i,{get:()=>t[i],enumerable:!(s=mt(t,i))||s.enumerable});return a};var yt=a=>kt(J({},"__esModule",{value:!0}),a);var Yt={};bt(Yt,{StandaloneUI:()=>et,WebUI:()=>tt});function X(a){a.version===void 0&&(a.version=0);for(let t of a.functions)xt(t,a.version);return a.version=1,a}function xt(a,t){for(let e of a.passes){for(let s of e.mir.blocks)wt(s,t);for(let s of e.lir.blocks)It(s,t)}return a}function wt(a,t){t===0&&(a.ptr=(a.id??a.number)+1,a.id=a.number);for(let e of a.instructions)vt(e,t);return a}function vt(a,t){return t===0&&(a.ptr=a.id),a}function It(a,t){t===0&&(a.ptr=a.id??a.number,a.id=a.number);for(let e of a.instructions)Lt(e,t);return a}function Lt(a,t){return t===0&&(a.ptr=a.id,a.mirPtr=null),a}function q(a,t,e){return Math.max(t,Math.min(e,a))}function R(a,t,e,s){return(a-t)*Math.pow(e,s)+t}function k(a,t,e=!1){if(!a)if(e)console.error(t??"Assertion failed");else throw new Error(t??"Assertion failed")}function L(a,t){return k(a,t),a}function Pt(a){return typeof a=="string"?document.createTextNode(a):a}function Bt(a,t){for(let e of t)e&&a.appendChild(Pt(e))}function x(a,t,e,s){let i=document.createElement(a);if(t&&t.length>0){let c=t.filter(l=>!!l);i.classList.add(...c)}return e?.(i),s&&Bt(i,s),i}var st=Object.prototype.hasOwnProperty;function E(a,t){var e,s;if(a===t)return!0;if(a&&t&&(e=a.constructor)===t.constructor){if(e===Date)return a.getTime()===t.getTime();if(e===RegExp)return a.toString()===t.toString();if(e===Array){if((s=a.length)===t.length)for(;s--&&E(a[s],t[s]););return s===-1}if(!e||typeof a=="object"){s=0;for(e in a)if(st.call(a,e)&&++s&&!st.call(t,e)||!(e in t)||!E(a[e],t[e]))return!1;return Object.keys(t).length===s}}return a!==a&&t!==t}function N(a,t,e={}){let s=t,i=[],c={get(){return s},set(l){s=l;for(let o of i)o(s)},valueOf(){return s},toString(){return String(s)},[Symbol.toPrimitive](l){return l==="string"?String(s):s},onChange(l){i.push(l)},initial:t,name:a,min:e.min??0,max:e.max??100,step:e.step??1};return(e.tweaksObject??K).add(c)}var F=class{constructor(t){this.container=t.container,this.tweaks=[],this.callbacks=[]}add(t){let e=this.tweaks.find(r=>r.name===t.name);if(e)return e;this.tweaks.push(t);let s=document.createElement("div");this.container.appendChild(s),s.style.display="flex",s.style.alignItems="center",s.style.justifyContent="end",s.style.gap="0.5rem";let i=t.name.replace(/[a-zA-Z0-9]/g,"_"),c=document.createElement("label");s.appendChild(c),c.innerText=t.name,c.htmlFor=`tweak-${i}-input`;let l=document.createElement("input");s.appendChild(l),l.type="number",l.value=String(t),l.id=`tweak-${i}-input`,l.style.width="4rem",l.addEventListener("input",()=>{t.set(l.valueAsNumber)});let o=document.createElement("input");s.appendChild(o),o.type="range",o.value=String(t),o.min=String(t.min),o.max=String(t.max),o.step=t.step===0?"any":String(t.step),o.addEventListener("input",()=>{t.set(o.valueAsNumber)});let n=document.createElement("button");return s.appendChild(n),n.innerText="Reset",n.disabled=t.get()===t.initial,n.addEventListener("click",()=>{t.set(t.initial)}),t.onChange(r=>{l.value=String(r),o.value=String(r),n.disabled=t.get()===t.initial;for(let d of this.callbacks)d(t)}),t}onTweak(t){this.callbacks.push(t)}},nt=document.createElement("div");nt.classList.add("tweaks-panel");var K=new F({container:nt});K.onTweak(a=>{window.dispatchEvent(new CustomEvent("tweak",{detail:a}))});window.tweaks=K;var Nt=document.createElement("div"),St=new F({container:Nt}),ot=N("Test Value",3,{tweaksObject:St});ot.set(4);ot=4;var dt=N("Debug?",0,{min:0,max:1}),P=20,C=44,I=16,S=60,T=12,H=36,it=16,D=16,Tt=N("Layout Iterations",2,{min:0,max:6}),rt=N("Nearly Straight Threshold",30,{min:0,max:200}),$t=N("Nearly Straight Iterations",8,{min:0,max:10}),at=N("Stop At Pass",30,{min:0,max:30}),Mt=1.5,Et=.01,Ct=1,Ht=.1,z=40;function _(a){return a.mir.attributes.includes("loopheader")}function Dt(a){return a.loopHeight!==void 0}function $(a){if(k(a),Dt(a))return a;throw new Error(`Block ${a.id} is not a pseudo LoopHeader`)}var j=1,lt=2,U=4,O=0,W=1,w=new Proxy(console,{get(a,t){let e=a[t];return typeof e!="function"?e:+dt?e.bind(a):()=>{}}}),V=class{constructor(t,e,s={}){this.viewport=t;let i=t.getBoundingClientRect();this.viewportSize={x:i.width,y:i.height},this.graphContainer=document.createElement("div"),this.graphContainer.classList.add("ig-graph"),this.graphContainer.style.transformOrigin="top left",this.viewport.appendChild(this.graphContainer),this.sampleCounts=s.sampleCounts,this.maxSampleCounts=[0,0],this.heatmapMode=O;for(let[n,r]of this.sampleCounts?.totalLineHits??[])this.maxSampleCounts[O]=Math.max(this.maxSampleCounts[O],r);for(let[n,r]of this.sampleCounts?.selfLineHits??[])this.maxSampleCounts[W]=Math.max(this.maxSampleCounts[W],r);this.size={x:0,y:0},this.numLayers=0,this.zoom=1,this.translation={x:0,y:0},this.animating=!1,this.targetZoom=1,this.targetTranslation={x:0,y:0},this.startMousePos={x:0,y:0},this.lastMousePos={x:0,y:0},this.selectedBlockPtrs=new Set,this.lastSelectedBlockPtr=0,this.nav={visited:[],currentIndex:-1,siblings:[]},this.highlightedInstructions=[],this.instructionPalette=s.instructionPalette??[0,1,2,3,4].map(n=>`var(--ig-highlight-${n})`),this.blocks=e.mir.blocks.map(n=>{let r={ptr:n.ptr,id:n.id,mir:n,lir:e.lir.blocks.find(d=>d.id===n.id)??null,preds:[],succs:[],el:void 0,size:{x:0,y:0},layer:-1,loopID:-1,layoutNode:void 0};if(r.mir.attributes.includes("loopheader")){let d=r;d.loopHeight=0,d.parentLoop=null,d.outgoingEdges=[]}return k(r.ptr,"blocks must always have non-null ptrs"),r}),this.blocksByID=new Map,this.blocksByPtr=new Map,this.insPtrsByID=new Map,this.insIDsByPtr=new Map,this.loops=[];for(let n of this.blocks){this.blocksByID.set(n.id,n),this.blocksByPtr.set(n.ptr,n);for(let r of n.mir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id);if(n.lir)for(let r of n.lir.instructions)this.insPtrsByID.set(r.id,r.ptr),this.insIDsByPtr.set(r.ptr,r.id)}for(let n of this.blocks)if(n.preds=n.mir.predecessors.map(r=>L(this.blocksByID.get(r))),n.succs=n.mir.successors.map(r=>L(this.blocksByID.get(r))),_(n)){let r=n.preds.filter(d=>d.mir.attributes.includes("backedge"));k(r.length===1),n.backedge=r[0]}for(let n of this.blocks)n.el=this.renderBlock(n);for(let n of this.blocks)n.size={x:n.el.clientWidth,y:n.el.clientHeight};let[c,l,o]=this.layout();this.render(c,l,o),this.addEventListeners()}layout(){let[t,e]=this.findLayoutRoots();w.log("Layout roots:",t.map(l=>l.id));for(let l of[...t,...e]){let o=l;o.loopHeight=0,o.parentLoop=null,o.outgoingEdges=[],Object.defineProperty(o,"backedge",{get(){throw new Error("Accessed .backedge on a pseudo loop header! Don't do that.")},configurable:!0})}for(let l of t)w.group("findLoops"),this.findLoops(l),w.groupEnd();for(let l of t)w.group("layer"),this.layer(l),w.groupEnd();for(let l of e)l.layer=0,l.loopID=l.id;let s=this.makeLayoutNodes();this.straightenEdges(s);let i=this.finagleJoints(s),c=this.verticalize(s,i);return[s,c,i]}findLayoutRoots(){let t=[],e=[],s=this.blocks.filter(i=>i.preds.length===0);for(let i of s){let c=i;if(i.mir.attributes.includes("osr")){k(i.succs.length>0);let l=i.succs[0];c=l;for(let o=0;;o++){if(o>=1e7)throw new Error("likely infinite loop");let n=c.preds.filter(r=>!r.mir.attributes.includes("osr")&&!r.mir.attributes.includes("backedge"));if(n.length===0)break;c=n[0]}c!==l?e.push(i):c=i}t.includes(c)||t.push(c)}return[t,e]}findLoops(t,e=null){if(e===null&&(e=[t.id]),!(t.loopID>=0)){if(w.log("block:",t.id,t.mir.loopDepth,"loopIDsByDepth:",e),w.log(t.mir.attributes),_(t)){let s=e[e.length-1],i=$(this.blocksByID.get(s));t.parentLoop=i,e=[...e,t.id],w.log("Block",t.id,"is true loop header, loopIDsByDepth is now",e)}if(t.mir.loopDepth>e.length-1&&(t.mir.loopDepth=e.length-1,w.log("Block",t.id,"has been forced back to loop depth",t.mir.loopDepth)),t.mir.loopDepth<e.length-1&&(e=e.slice(0,t.mir.loopDepth+1),w.log("Block",t.id,"has low loop depth, therefore we exited a loop. loopIDsByDepth:",e)),t.loopID=e[t.mir.loopDepth],!t.mir.attributes.includes("backedge"))for(let s of t.succs)this.findLoops(s,e)}}layer(t,e=0){if(w.log("block",t.id,"layer",e),t.mir.attributes.includes("backedge")){t.layer=t.succs[0].layer;return}if(e<=t.layer)return;t.layer=Math.max(t.layer,e),this.numLayers=Math.max(t.layer+1,this.numLayers);let s=$(this.blocksByID.get(t.loopID));for(;s;)s.loopHeight=Math.max(s.loopHeight,t.layer-s.layer+1),s=s.parentLoop;for(let i of t.succs)i.mir.loopDepth<t.mir.loopDepth?$(this.blocksByID.get(t.loopID)).outgoingEdges.push(i):this.layer(i,e+1);if(_(t))for(let i of t.outgoingEdges)this.layer(i,e+t.loopHeight)}makeLayoutNodes(){w.group("makeLayoutNodes");function t(o,n,r){o.dstNodes[n]=r,r.srcNodes.includes(o)||r.srcNodes.push(o),w.log("connected",o.id,"to",r.id)}let e;{let o={};for(let n of this.blocks)o[n.layer]||(o[n.layer]=[]),o[n.layer].push(n);e=Object.entries(o).map(([n,r])=>[Number(n),r]).sort((n,r)=>n[0]-r[0]).map(([n,r])=>r)}let s=0,i=e.map(()=>[]),c=[],l=new Map;for(let[o,n]of e.entries()){w.group("layer",o,"blocks",n.map(u=>u.id));let r=[];for(let u of n)for(let h=c.length-1;h>=0;h--){let p=c[h];p.dstBlock===u&&(r.unshift(p),c.splice(h,1))}let d=new Map;for(let u of c){let h,p=d.get(u.dstBlock.id);if(p)t(u.src,u.srcPort,p),h=p;else{let m={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:u.dstBlock,jointOffsets:[],flags:0};t(u.src,u.srcPort,m),i[o].push(m),d.set(u.dstBlock.id,m),h=m,w.log("Created dummy",h.id,"on the way to block",u.dstBlock.id)}u.src=h,u.srcPort=0}let b=[];for(let u of n){let h=$(this.blocksByID.get(u.loopID));for(;_(h);){let p=b.find(f=>f.loopID===h.id);p?p.block=u:b.push({loopID:h.id,block:u});let m=h.parentLoop;if(!m)break;h=m}}let g=[];for(let u of n){let h={id:s++,pos:{x:P,y:P},size:u.size,block:u,srcNodes:[],dstNodes:[],jointOffsets:[],flags:0};for(let p of r)p.dstBlock===u&&t(p.src,p.srcPort,h);i[o].push(h),u.layoutNode=h;for(let p of b.filter(m=>m.block===u)){let m=$(this.blocksByID.get(p.loopID)).backedge,f={id:s++,pos:{x:P,y:P},size:{x:0,y:0},block:null,srcNodes:[],dstNodes:[],dstBlock:m,jointOffsets:[],flags:0},y=l.get(m);y?t(f,0,y):(f.flags|=U,t(f,0,m.layoutNode)),i[o].push(f),l.set(m,f)}if(u.mir.attributes.includes("backedge"))t(u.layoutNode,0,u.succs[0].layoutNode);else for(let[p,m]of u.succs.entries())m.mir.attributes.includes("backedge")?g.push({src:h,srcPort:p,dstBlock:m}):c.push({src:h,srcPort:p,dstBlock:m})}for(let u of g){let h=L(l.get(u.dstBlock));t(u.src,u.srcPort,h)}w.groupEnd()}w.log("Pruning backedge dummies");{let o=[];for(let r of At(i))r.srcNodes.length===0&&o.push(r);let n=new Set;for(let r of o){let d=r;for(;d.block===null&&d.srcNodes.length===0;)Ot(d),n.add(d),k(d.dstNodes.length===1),d=d.dstNodes[0]}for(let r of i)for(let d=r.length-1;d>=0;d--)n.has(r[d])&&r.splice(d,1)}w.log("Marking leftmost and rightmost dummies");for(let o of i){for(let n=0;n<o.length&&o[n].block===null;n++)o[n].flags|=j;for(let n=o.length-1;n>=0&&o[n].block===null;n--)o[n].flags|=lt}w.log("Verifying integrity of all nodes");for(let o of i)for(let n of o){n.block?k(n.dstNodes.length===n.block.succs.length,`expected node ${n.id} for block ${n.block.id} to have ${n.block.succs.length} destination nodes, but got ${n.dstNodes.length} instead`):k(n.dstNodes.length===1,`expected dummy node ${n.id} to have only one destination node, but got ${n.dstNodes.length} instead`);for(let r=0;r<n.dstNodes.length;r++)k(n.dstNodes[r]!==void 0,`dst slot ${r} of node ${n.id} was undefined`)}return w.groupEnd(),i}straightenEdges(t){let e=g=>{for(let u=0;u<g.length-1;u++){let h=g[u],p=g[u+1],m=h.block===null&&p.block!==null,f=h.pos.x+h.size.x+(m?I:0)+C;p.pos.x=Math.max(p.pos.x,f)}},s=()=>{for(let g of t)for(let u of g){if(u.block===null)continue;let h=u.block.loopID!==null?$(this.blocksByID.get(u.block.loopID)):null;if(h){let p=h.layoutNode;u.pos.x=Math.max(u.pos.x,p.pos.x)}}},i=()=>{let g=new Map;for(let u of Q(t)){let h=u.dstBlock,p=u.pos.x;g.set(h,Math.max(g.get(h)??0,p))}for(let u of Q(t)){let h=u.dstBlock,p=g.get(h);k(p,`no position for backedge ${h.id}`),u.pos.x=p}for(let u of t)e(u)},c=()=>{let g=new Map;for(let u of t){let h=0,p=0;for(;h<u.length;h++)if(!(u[h].flags&j)){p=u[h].pos.x;break}for(h-=1,p-=C+I;h>=0;h--){let m=u[h];k(m.block===null&&m.flags&j);let f=p;for(let y of m.srcNodes){let v=y.pos.x+y.dstNodes.indexOf(m)*S;v<f&&(f=v)}m.pos.x=f,p=m.pos.x-C,g.set(m.dstBlock,Math.min(g.get(m.dstBlock)??1/0,f))}}for(let u of Q(t)){if(!(u.flags&j))continue;let h=g.get(u.dstBlock);k(h,`no position for run to block ${u.dstBlock.id}`),u.pos.x=h}},l=()=>{for(let g=0;g<t.length-1;g++){let u=t[g];e(u);let h=-1;for(let p of u)for(let[m,f]of p.dstNodes.entries()){let y=t[g+1].indexOf(f);if(y>h&&f.srcNodes[0]===p){let v=I+S*m,B=I,Z=f.pos.x;f.pos.x=Math.max(f.pos.x,p.pos.x+v-B),f.pos.x!==Z&&(h=y)}}}},o=()=>{for(let g of t){for(let u=g.length-1;u>=0;u--){let h=g[u];if(!h.block||h.block.mir.attributes.includes("backedge"))continue;let p=[];for(let m of h.srcNodes){let f=I+m.dstNodes.indexOf(h)*S,y=I;p.push(m.pos.x+f-(h.pos.x+y))}for(let[m,f]of h.dstNodes.entries()){if(f.block===null&&f.dstBlock.mir.attributes.includes("backedge"))continue;let y=I+m*S,v=I;p.push(f.pos.x+v-(h.pos.x+y))}if(!p.includes(0)){p=p.filter(m=>m>0).sort((m,f)=>m-f);for(let m of p){let f=!1;for(let y=u+1;y<g.length;y++){let v=g[y];if(v.flags<)continue;let B=h.pos.x+m,Z=h.pos.x+m+h.size.x,ht=v.pos.x-C,pt=v.pos.x+v.size.x+C;Z>=ht&&B<=pt&&(f=!0)}if(!f){h.pos.x+=m;break}}}}e(g)}},n=()=>{for(let g=t.length-1;g>=0;g--){let u=t[g];e(u);for(let h of u)for(let p of h.srcNodes){if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}},r=()=>{for(let g=0;g<t.length;g++){let u=t[g];e(u);for(let h of u){if(h.dstNodes.length===0)continue;let p=h.dstNodes[0];if(p.block!==null)continue;Math.abs(p.pos.x-h.pos.x)<=rt&&(p.pos.x=Math.max(p.pos.x,h.pos.x),h.pos.x=Math.max(p.pos.x,h.pos.x))}}};function d(g,u){let h=[];for(let p=0;p<u;p++)for(let m of g)h.push(m);return h}let b=[...d([l,s,i],Tt),i,...d([n,r],$t),o,i,c];k(b.length<=(at.initial??1/0),`STOP_AT_PASS was too small - should be at least ${b.length}`),w.group("Running passes");for(let[g,u]of b.entries())g<at&&(w.log(u.name??u.toString()),u());w.groupEnd()}finagleJoints(t){let e=[];for(let s of t){let i=[];for(let r of s)if(r.jointOffsets=new Array(r.dstNodes.length).fill(0),!r.block?.mir.attributes.includes("backedge"))for(let[d,b]of r.dstNodes.entries()){let g=r.pos.x+I+S*d,u=b.pos.x+I;Math.abs(u-g)<2*T||i.push({x1:g,x2:u,src:r,srcPort:d,dst:b})}i.sort((r,d)=>r.x1-d.x1);let c=[],l=[];t:for(let r of i){let d=r.x2-r.x1>=0?c:l,b=null;for(let g=d.length-1;g>=0;g--){let u=d[g],h=!1;for(let p of u){if(r.dst===p.dst){u.push(r);continue t}let m=Math.min(r.x1,r.x2),f=Math.max(r.x1,r.x2),y=Math.min(p.x1,p.x2),v=Math.max(p.x1,p.x2);if(f>=y&&m<=v){h=!0;break}}if(h)break;b=u}b?b.push(r):d.push([r])}let o=Math.max(0,c.length+l.length-1)*it,n=-o/2;for(let r of[...c.reverse(),...l]){for(let d of r)d.src.jointOffsets[d.srcPort]=n;n+=it}e.push(o)}return k(e.length===t.length),e}verticalize(t,e){let s=new Array(t.length),i=P;for(let c=0;c<t.length;c++){let l=t[c],o=0;for(let n of l)n.pos.y=i,o=Math.max(o,n.size.y);s[c]=o,i+=o+H+e[c]+H}return s}renderBlock(t){let e=document.createElement("div");this.graphContainer.appendChild(e),e.classList.add("ig-block","ig-bg-white");for(let o of t.mir.attributes)e.classList.add(`ig-block-att-${o}`);e.setAttribute("data-ig-block-ptr",`${t.ptr}`),e.setAttribute("data-ig-block-id",`${t.id}`);let s="";t.mir.attributes.includes("loopheader")?s=" (loop header)":t.mir.attributes.includes("backedge")?s=" (backedge)":t.mir.attributes.includes("splitedge")&&(s=" (split edge)");let i=document.createElement("div");i.classList.add("ig-block-header"),i.innerText=`Block ${t.id}${s}`,e.appendChild(i);let c=document.createElement("div");c.classList.add("ig-instructions"),e.appendChild(c);let l=document.createElement("table");if(t.lir){l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + ${this.sampleCounts?` + <col style="width: 1px"> + <col style="width: 1px"> + `:""} + </colgroup> + ${this.sampleCounts?` + <thead> + <tr> + <th></th> + <th></th> + <th class="ig-f6">Total</th> + <th class="ig-f6">Self</th> + </tr> + </thead> + `:""} + `;for(let o of t.lir.instructions)l.appendChild(this.renderLIRInstruction(o))}else{l.innerHTML=` + <colgroup> + <col style="width: 1px"> + <col style="width: auto"> + <col style="width: 1px"> + </colgroup> + `;for(let o of t.mir.instructions)l.appendChild(this.renderMIRInstruction(o))}if(c.appendChild(l),t.succs.length===2)for(let[o,n]of[1,0].entries()){let r=document.createElement("div");r.innerText=`${n}`,r.classList.add("ig-edge-label"),r.style.left=`${I+S*o}px`,e.appendChild(r)}return i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",o=>{o.stopPropagation(),o.shiftKey||this.selectedBlockPtrs.clear(),this.setSelection([],t.ptr)}),e}render(t,e,s){for(let n of t)for(let r of n)if(r.block!==null){let d=r.block;d.el.style.left=`${r.pos.x}px`,d.el.style.top=`${r.pos.y}px`}let i=0,c=0;for(let n of t)for(let r of n)i=Math.max(i,r.pos.x+r.size.x+P),c=Math.max(c,r.pos.y+r.size.y+P);let l=document.createElementNS("http://www.w3.org/2000/svg","svg");this.graphContainer.appendChild(l);let o=(n,r)=>{for(let d of n)i=Math.max(i,d+P);for(let d of r)c=Math.max(c,d+P)};for(let n=0;n<t.length;n++){let r=t[n];for(let d of r){d.block||k(d.dstNodes.length===1,`dummy nodes must have exactly one destination, but dummy ${d.id} had ${d.dstNodes.length}`),k(d.dstNodes.length===d.jointOffsets.length,"must have a joint offset for each destination");for(let[b,g]of d.dstNodes.entries()){let u=d.pos.x+I+S*b,h=d.pos.y+d.size.y;if(d.block?.mir.attributes.includes("backedge")){let p=d.block.succs[0],m=d.pos.x,f=d.pos.y+D,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=jt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(d.flags&U){let p=L(g.block),m=d.pos.x+I,f=d.pos.y+D+T,y=p.layoutNode.pos.x+p.size.x,v=p.layoutNode.pos.y+D,B=zt(m,f,y,v);l.appendChild(B),o([m,y],[f,v])}else if(g.block===null&&g.dstBlock.mir.attributes.includes("backedge")){let p=g.pos.x+I,m=g.pos.y+(g.flags&U?D+T:0);if(d.block===null){let f=h-H,y=Ft(u,h,p,m,f,!1);l.appendChild(y),o([u,p],[h,m,f])}else{let f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=_t(u,h,p,m,f);l.appendChild(y),o([u,p],[h,m,f])}}else{let p=g.pos.x+I,m=g.pos.y,f=h-d.size.y+e[n]+H+s[n]/2+d.jointOffsets[b],y=Rt(u,h,p,m,f,g.block!==null);l.appendChild(y),o([u,p],[h,m,f])}}}}if(l.setAttribute("width",`${i}`),l.setAttribute("height",`${c}`),this.size={x:i,y:c},+dt)for(let n of t)for(let r of n){let d=document.createElement("div");d.innerHTML=`${r.id}<br><- ${r.srcNodes.map(b=>b.id)}<br>-> ${r.dstNodes.map(b=>b.id)}<br>${r.flags}`,d.style.position="absolute",d.style.border="1px solid black",d.style.backgroundColor="white",d.style.left=`${r.pos.x}px`,d.style.top=`${r.pos.y}px`,d.style.whiteSpace="nowrap",this.graphContainer.appendChild(d)}this.updateHighlightedInstructions(),this.updateHotness()}renderMIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-mir","ig-can-flash",...t.attributes.map(o=>`ig-ins-att-${o}`)),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=`v${t.id}`,s.appendChild(i);let c=document.createElement("td");c.innerHTML=e.replace(/(v)(\d+)/g,(o,n,r)=>`<span class="ig-use ig-highlightable" data-ig-use="${r}">${n}${r}</span>`),s.appendChild(c);let l=document.createElement("td");return l.classList.add("ig-ins-type"),l.innerText=t.type==="None"?"":t.type,s.appendChild(l),i.addEventListener("pointerdown",o=>{o.preventDefault(),o.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),c.querySelectorAll(".ig-use").forEach(o=>{o.addEventListener("pointerdown",n=>{n.preventDefault(),n.stopPropagation()}),o.addEventListener("click",n=>{let r=parseInt(L(o.getAttribute("data-ig-use")),10);this.jumpToInstruction(r,{zoom:1})})}),s}renderLIRInstruction(t){let e=t.opcode.replace("->","\u2192").replace("<-","\u2190"),s=document.createElement("tr");s.classList.add("ig-ins","ig-ins-lir","ig-hotness"),s.setAttribute("data-ig-ins-ptr",`${t.ptr}`),s.setAttribute("data-ig-ins-id",`${t.id}`);let i=document.createElement("td");i.classList.add("ig-ins-num"),i.innerText=String(t.id),s.appendChild(i);let c=document.createElement("td");if(c.innerText=e,s.appendChild(c),this.sampleCounts){let l=this.sampleCounts?.totalLineHits.get(t.id)??0,o=this.sampleCounts?.selfLineHits.get(t.id)??0,n=document.createElement("td");n.classList.add("ig-ins-samples"),n.classList.toggle("ig-text-dim",l===0),n.innerText=`${l}`,n.title="Color by total count",s.appendChild(n);let r=document.createElement("td");r.classList.add("ig-ins-samples"),r.classList.toggle("ig-text-dim",o===0),r.innerText=`${o}`,r.title="Color by self count",s.appendChild(r);for(let[d,b]of[n,r].entries())b.addEventListener("pointerdown",g=>{g.preventDefault(),g.stopPropagation()}),b.addEventListener("click",()=>{k(d===O||d===W),this.heatmapMode=d,this.updateHotness()})}return i.addEventListener("pointerdown",l=>{l.preventDefault(),l.stopPropagation()}),i.addEventListener("click",()=>{this.toggleInstructionHighlight(t.ptr)}),s}renderSelection(){this.graphContainer.querySelectorAll(".ig-block").forEach(t=>{let e=parseInt(L(t.getAttribute("data-ig-block-ptr")),10);t.classList.toggle("ig-selected",this.selectedBlockPtrs.has(e)),t.classList.toggle("ig-last-selected",this.lastSelectedBlockPtr===e)})}removeNonexistentHighlights(){this.highlightedInstructions=this.highlightedInstructions.filter(t=>this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`))}updateHighlightedInstructions(){for(let t of this.highlightedInstructions)k(this.highlightedInstructions.filter(e=>e.ptr===t.ptr).length===1,`instruction ${t.ptr} was highlighted more than once`);this.graphContainer.querySelectorAll(".ig-ins, .ig-use").forEach(t=>{Vt(t)});for(let t of this.highlightedInstructions){let e=this.instructionPalette[t.paletteColor%this.instructionPalette.length],s=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-ptr="${t.ptr}"]`);if(s){ct(s,e);let i=this.insIDsByPtr.get(t.ptr);this.graphContainer.querySelectorAll(`.ig-use[data-ig-use="${i}"]`).forEach(c=>{ct(c,e)})}}}updateHotness(){this.graphContainer.querySelectorAll(".ig-ins-lir").forEach(t=>{k(t.classList.contains("ig-hotness"));let e=parseInt(L(t.getAttribute("data-ig-ins-id")),10),s=0;this.sampleCounts&&(s=((this.heatmapMode===O?this.sampleCounts.totalLineHits:this.sampleCounts.selfLineHits).get(e)??0)/this.maxSampleCounts[this.heatmapMode]),t.style.setProperty("--ig-hotness",`${s}`)})}addEventListeners(){this.viewport.addEventListener("wheel",e=>{e.preventDefault();let s=this.zoom;if(e.ctrlKey){s=Math.max(Ht,Math.min(Ct,this.zoom*Math.pow(Mt,-e.deltaY*Et)));let c=s/this.zoom-1;this.zoom=s;let{x:l,y:o}=this.viewport.getBoundingClientRect(),n=e.clientX-l-this.translation.x,r=e.clientY-o-this.translation.y;this.translation.x-=n*c,this.translation.y-=r*c}else this.translation.x-=e.deltaX,this.translation.y-=e.deltaY;let i=this.clampTranslation(this.translation,s);this.translation.x=i.x,this.translation.y=i.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerdown",e=>{e.pointerType==="mouse"&&!(e.button===0||e.button===1)||(e.preventDefault(),this.viewport.setPointerCapture(e.pointerId),this.startMousePos={x:e.clientX,y:e.clientY},this.lastMousePos={x:e.clientX,y:e.clientY},this.animating=!1)}),this.viewport.addEventListener("pointermove",e=>{if(!this.viewport.hasPointerCapture(e.pointerId))return;let s=e.clientX-this.lastMousePos.x,i=e.clientY-this.lastMousePos.y;this.translation.x+=s,this.translation.y+=i,this.lastMousePos={x:e.clientX,y:e.clientY};let c=this.clampTranslation(this.translation,this.zoom);this.translation.x=c.x,this.translation.y=c.y,this.animating=!1,this.updatePanAndZoom()}),this.viewport.addEventListener("pointerup",e=>{this.viewport.releasePointerCapture(e.pointerId);let s=2,i=this.startMousePos.x-e.clientX,c=this.startMousePos.y-e.clientY;Math.abs(i)<=s&&Math.abs(c)<=s&&this.setSelection([]),this.animating=!1}),new ResizeObserver(e=>{k(e.length===1);let s=e[0].contentRect;this.viewportSize.x=s.width,this.viewportSize.y=s.height}).observe(this.viewport)}setSelection(t,e=0){this.setSelectionRaw(t,e),e?this.nav={visited:[e],currentIndex:0,siblings:[e]}:this.nav={visited:[],currentIndex:-1,siblings:[]}}setSelectionRaw(t,e){this.selectedBlockPtrs.clear();for(let s of[...t,e])this.blocksByPtr.has(s)&&this.selectedBlockPtrs.add(s);this.lastSelectedBlockPtr=this.blocksByPtr.has(e)?e:0,this.renderSelection()}navigate(t){let e=this.lastSelectedBlockPtr;if(t==="down"||t==="up")if(e){let s=L(this.blocksByPtr.get(e)),i=(t==="down"?s.succs:s.preds).map(l=>l.ptr);s.ptr!==this.nav.visited[this.nav.currentIndex]&&(this.nav.visited=[s.ptr],this.nav.currentIndex=0);let c=this.nav.currentIndex+(t==="down"?1:-1);if(0<=c&&c<this.nav.visited.length)this.nav.currentIndex=c,this.nav.siblings=i;else{let l=i[0];l!==void 0&&(t==="down"?(this.nav.visited.push(l),this.nav.currentIndex+=1,k(this.nav.currentIndex===this.nav.visited.length-1)):(this.nav.visited.unshift(l),k(this.nav.currentIndex===0)),this.nav.siblings=i)}this.setSelectionRaw([],this.nav.visited[this.nav.currentIndex])}else{let s=[...this.blocks].sort((n,r)=>n.id-r.id),i=s.filter(n=>n.preds.length===0),c=s.filter(n=>n.succs.length===0),l=t==="down"?i:c,o=l[0];k(o),this.setSelectionRaw([],o.ptr),this.nav={visited:[o.ptr],currentIndex:0,siblings:l.map(n=>n.ptr)}}else if(e!==void 0){let s=this.nav.siblings.indexOf(e);k(s>=0,"currently selected node should be in siblings array");let i=s+(t==="right"?1:-1);0<=i&&i<this.nav.siblings.length&&this.setSelectionRaw([],this.nav.siblings[i])}k(this.nav.visited.length===0||this.nav.siblings.includes(this.nav.visited[this.nav.currentIndex]),"expected currently visited node to be in the siblings array"),k(this.lastSelectedBlockPtr===0||this.nav.siblings.includes(this.lastSelectedBlockPtr),"expected currently selected block to be in siblings array")}toggleInstructionHighlight(t,e){this.removeNonexistentHighlights();let s=this.highlightedInstructions.findIndex(c=>c.ptr===t),i=s>=0;if(e!==void 0&&(i=!e),i)s>=0&&this.highlightedInstructions.splice(s,1);else if(s<0){let c=0;for(;;){if(this.highlightedInstructions.find(l=>l.paletteColor===c)){c+=1;continue}break}this.highlightedInstructions.push({ptr:t,paletteColor:c})}this.updateHighlightedInstructions()}clampTranslation(t,e){let s=z-this.size.x*e,i=this.viewportSize.x-z,c=z-this.size.y*e,l=this.viewportSize.y-z,o=q(t.x,s,i),n=q(t.y,c,l);return{x:o,y:n}}updatePanAndZoom(){let t=this.clampTranslation(this.translation,this.zoom);this.graphContainer.style.transform=`translate(${t.x}px, ${t.y}px) scale(${this.zoom})`}graph2viewport(t,e=this.translation,s=this.zoom){return{x:t.x*s+e.x,y:t.y*s+e.y}}viewport2graph(t,e=this.translation,s=this.zoom){return{x:(t.x-e.x)/s,y:(t.y-e.y)/s}}async goToGraphCoordinates(t,{zoom:e=this.zoom,animate:s=!0}){let i={x:-t.x*e,y:-t.y*e};if(!s){this.animating=!1,this.translation.x=i.x,this.translation.y=i.y,this.zoom=e,this.updatePanAndZoom(),await new Promise(l=>setTimeout(l,0));return}if(this.targetTranslation=i,this.targetZoom=e,this.animating)return;this.animating=!0;let c=performance.now();for(;this.animating;){let l=await new Promise(h=>requestAnimationFrame(h)),o=(l-c)/1e3;c=l;let n=1,r=.01,d=1e-6,b=this.targetTranslation.x-this.translation.x,g=this.targetTranslation.y-this.translation.y,u=this.targetZoom-this.zoom;if(this.translation.x=R(this.translation.x,this.targetTranslation.x,d,o),this.translation.y=R(this.translation.y,this.targetTranslation.y,d,o),this.zoom=R(this.zoom,this.targetZoom,d,o),this.updatePanAndZoom(),Math.abs(b)<=n&&Math.abs(g)<=n&&Math.abs(u)<=r){this.translation.x=this.targetTranslation.x,this.translation.y=this.targetTranslation.y,this.zoom=this.targetZoom,this.animating=!1,this.updatePanAndZoom();break}}await new Promise(l=>setTimeout(l,0))}jumpToBlock(t,{zoom:e=this.zoom,animate:s=!0,viewportPos:i}={}){let c=this.blocksByPtr.get(t);if(!c)return Promise.resolve();let l;return i?l={x:c.layoutNode.pos.x-i.x/e,y:c.layoutNode.pos.y-i.y/e}:l=this.graphPosToCenterRect(c.layoutNode.pos,c.layoutNode.size,e),this.goToGraphCoordinates(l,{zoom:e,animate:s})}async jumpToInstruction(t,{zoom:e=this.zoom,animate:s=!0}){let i=this.graphContainer.querySelector(`.ig-ins[data-ig-ins-id="${t}"]`);if(!i)return;let c=i.getBoundingClientRect(),l=this.graphContainer.getBoundingClientRect(),o=(c.x-l.x)/this.zoom,n=(c.y-l.y)/this.zoom,r=c.width/this.zoom,d=c.height/this.zoom,b=this.graphPosToCenterRect({x:o,y:n},{x:r,y:d},e);i.classList.add("ig-flash"),await this.goToGraphCoordinates(b,{zoom:e,animate:s}),i.classList.remove("ig-flash")}graphPosToCenterRect(t,e,s){let i=this.viewportSize.x/s,c=this.viewportSize.y/s,l=Math.max(20/s,(i-e.x)/2),o=Math.max(20/s,(c-e.y)/2),n=t.x-l,r=t.y-o;return{x:n,y:r}}exportState(){let t={translation:this.translation,zoom:this.zoom,heatmapMode:this.heatmapMode,highlightedInstructions:this.highlightedInstructions,selectedBlockPtrs:this.selectedBlockPtrs,lastSelectedBlockPtr:this.lastSelectedBlockPtr,viewportPosOfSelectedBlock:void 0};return this.lastSelectedBlockPtr&&(t.viewportPosOfSelectedBlock=this.graph2viewport(L(this.blocksByPtr.get(this.lastSelectedBlockPtr)).layoutNode.pos)),t}restoreState(t,e){this.translation.x=t.translation.x,this.translation.y=t.translation.y,this.zoom=t.zoom,this.heatmapMode=t.heatmapMode,this.highlightedInstructions=t.highlightedInstructions,this.setSelection(Array.from(t.selectedBlockPtrs),t.lastSelectedBlockPtr),this.updatePanAndZoom(),this.updateHotness(),this.updateHighlightedInstructions(),e.preserveSelectedBlockPosition&&this.jumpToBlock(this.lastSelectedBlockPtr,{zoom:this.zoom,animate:!1,viewportPos:t.viewportPosOfSelectedBlock})}};function Ot(a){for(let t of a.dstNodes){let e=t.srcNodes.indexOf(a);k(e!==-1),t.srcNodes.splice(e,1)}}function*Q(a){for(let t of a)for(let e of t)e.block===null&&(yield e)}function*At(a){for(let t of a)for(let e of t)e.block===null&&e.dstBlock.mir.attributes.includes("backedge")&&(yield e)}function Rt(a,t,e,s,i,c,l=1){let o=T;k(t+o<=i&&i<s-o,`downward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i-o} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${e} ${i+o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,180);r.appendChild(b)}return r}function Ft(a,t,e,s,i,c,l=1){let o=T;k(s+o<=i&&i<=t-o,`upward arrow: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${o} `,!0),l%2===1&&(a+=.5,e+=.5,i+=.5);let n="";if(n+=`M ${a} ${t} `,Math.abs(e-a)<2*o)n+=`C ${a} ${t+(s-t)/3} ${e} ${t+2*(s-t)/3} ${e} ${s} `;else{let b=Math.sign(e-a);n+=`L ${a} ${i+o} `,n+=`A ${o} ${o} 0 0 ${b>0?1:0} ${a+o*b} ${i} `,n+=`L ${e-o*b} ${i} `,n+=`A ${o} ${o} 0 0 ${b>0?0:1} ${e} ${i-o} `,n+=`L ${e} ${s} `}let r=document.createElementNS("http://www.w3.org/2000/svg","g"),d=document.createElementNS("http://www.w3.org/2000/svg","path");if(d.setAttribute("d",n),d.setAttribute("fill","none"),d.setAttribute("stroke","black"),d.setAttribute("stroke-width",`${l} `),r.appendChild(d),c){let b=G(e,s,0);r.appendChild(b)}return r}function zt(a,t,e,s,i=1){let c=T;k(t-c>=s&&a-c>=e,`to backedge: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, r = ${c} `,!0),i%2===1&&(a+=.5,s+=.5);let l="";l+=`M ${a} ${t} `,l+=`A ${c} ${c} 0 0 0 ${a-c} ${s} `,l+=`L ${e} ${s} `;let o=document.createElementNS("http://www.w3.org/2000/svg","g"),n=document.createElementNS("http://www.w3.org/2000/svg","path");n.setAttribute("d",l),n.setAttribute("fill","none"),n.setAttribute("stroke","black"),n.setAttribute("stroke-width",`${i} `),o.appendChild(n);let r=G(e,s,270);return o.appendChild(r),o}function _t(a,t,e,s,i,c=1){let l=T;k(t+l<=i&&a<=e&&s<=t,`block to backedge dummy: x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s}, ym = ${i}, r = ${l} `,!0),c%2===1&&(a+=.5,e+=.5,i+=.5);let o="";o+=`M ${a} ${t} `,o+=`L ${a} ${i-l} `,o+=`A ${l} ${l} 0 0 0 ${a+l} ${i} `,o+=`L ${e-l} ${i} `,o+=`A ${l} ${l} 0 0 0 ${e} ${i-l} `,o+=`L ${e} ${s} `;let n=document.createElementNS("http://www.w3.org/2000/svg","g"),r=document.createElementNS("http://www.w3.org/2000/svg","path");return r.setAttribute("d",o),r.setAttribute("fill","none"),r.setAttribute("stroke","black"),r.setAttribute("stroke-width",`${c} `),n.appendChild(r),n}function jt(a,t,e,s,i=1){k(e<a&&s===t,`x1 = ${a}, y1 = ${t}, x2 = ${e}, y2 = ${s} `,!0),i%2===1&&(t+=.5,s+=.5);let c="";c+=`M ${a} ${t} `,c+=`L ${e} ${s} `;let l=document.createElementNS("http://www.w3.org/2000/svg","g"),o=document.createElementNS("http://www.w3.org/2000/svg","path");o.setAttribute("d",c),o.setAttribute("fill","none"),o.setAttribute("stroke","black"),o.setAttribute("stroke-width",`${i} `),l.appendChild(o);let n=G(e,s,270);return l.appendChild(n),l}function G(a,t,e,s=5){let i=document.createElementNS("http://www.w3.org/2000/svg","path");return i.setAttribute("d",`M 0 0 L ${-s} ${s*1.5} L ${s} ${s*1.5} Z`),i.setAttribute("transform",`translate(${a}, ${t}) rotate(${e})`),i}function ct(a,t){a.classList.add("ig-highlight"),a.style.setProperty("--ig-highlight-color",t)}function Vt(a){a.classList.remove("ig-highlight"),a.style.setProperty("--ig-highlight-color","transparent")}var A=class{constructor(t,{func:e,pass:s=0,sampleCounts:i}){this.graph=null,this.func=e,this.passNumber=s,this.sampleCounts=i,this.keyPasses=[null,null,null,null];{let c=null;for(let[l,o]of e.passes.entries())o.mir.blocks.length>0&&(this.keyPasses[0]===null&&(this.keyPasses[0]=l),o.lir.blocks.length===0&&(this.keyPasses[1]=l)),o.lir.blocks.length>0&&(c?.lir.blocks.length===0&&(this.keyPasses[2]=l),this.keyPasses[3]=l),c=o}this.redundantPasses=[];{let c=null;for(let[l,o]of e.passes.entries()){if(c===null){c=o;continue}E(c.mir,o.mir)&&E(c.lir,o.lir)&&this.redundantPasses.push(l),c=o}}this.viewport=x("div",["ig-flex-grow-1","ig-overflow-hidden"],c=>{c.style.position="relative"}),this.sidebarLinks=e.passes.map((c,l)=>x("a",["ig-link-normal","ig-pv1","ig-ph2","ig-flex","ig-g2"],o=>{o.href="#",o.addEventListener("click",n=>{n.preventDefault(),this.switchPass(l)})},[x("div",["ig-w1","ig-tr","ig-f6","ig-text-dim"],o=>{o.style.paddingTop="0.08rem"},[`${l}`]),x("div",[this.redundantPasses.includes(l)&&"ig-text-dim"],()=>{},[c.name])])),this.container=x("div",["ig-absolute","ig-absolute-fill","ig-flex"],()=>{},[x("div",["ig-w5","ig-br","ig-flex-shrink-0","ig-overflow-y-auto","ig-bg-white"],()=>{},[...this.sidebarLinks]),this.viewport]),t.appendChild(this.container),this.keydownHandler=this.keydownHandler.bind(this),this.tweakHandler=this.tweakHandler.bind(this),window.addEventListener("keydown",this.keydownHandler),window.addEventListener("tweak",this.tweakHandler),this.update()}destroy(){this.container.remove(),window.removeEventListener("keydown",this.keydownHandler),window.removeEventListener("tweak",this.tweakHandler)}update(){for(let[s,i]of this.sidebarLinks.entries())i.classList.toggle("ig-bg-primary",this.passNumber===s);let t=this.graph?.exportState();this.viewport.innerHTML="",this.graph=null;let e=this.func.passes[this.passNumber];if(e)try{this.graph=new V(this.viewport,e,{sampleCounts:this.sampleCounts}),t&&this.graph.restoreState(t,{preserveSelectedBlockPosition:!0})}catch(s){this.viewport.innerHTML="An error occurred while laying out the graph. See console.",console.error(s)}}switchPass(t){this.passNumber=t,this.update()}keydownHandler(t){switch(t.key){case"w":case"s":this.graph?.navigate(t.key==="s"?"down":"up"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"a":case"d":this.graph?.navigate(t.key==="d"?"right":"left"),this.graph?.jumpToBlock(this.graph.lastSelectedBlockPtr);break;case"f":for(let e=this.passNumber+1;e<this.func.passes.length;e++)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"r":for(let e=this.passNumber-1;e>=0;e--)if(!this.redundantPasses.includes(e)){this.switchPass(e);break}break;case"1":case"2":case"3":case"4":{let e=["1","2","3","4"].indexOf(t.key),s=this.keyPasses[e];typeof s=="number"&&this.switchPass(s)}break;case"c":{let e=this.graph?.blocksByPtr.get(this.graph?.lastSelectedBlockPtr??-1);e&&this.graph?.jumpToBlock(e.ptr,{zoom:1})}break}}tweakHandler(){this.update()}};var M=new URL(window.location.toString()).searchParams,Gt=M.has("func")?parseInt(M.get("func"),10):void 0,ut=M.has("pass")?parseInt(M.get("pass"),10):void 0,Y=class{constructor(t){this.exportButton=null,this.ionjson=null,this.funcIndex=Gt??0,this.funcSelected=t.funcSelected,this.funcSelector=x("div",[],()=>{},["Function",x("input",["ig-w3"],e=>{e.type="number",e.min="1",e.addEventListener("input",()=>{this.switchFunc(parseInt(e.value,10)-1)})},[])," / ",x("span",["num-functions"])]),this.funcSelectorNone=x("div",[],()=>{},["No functions to display."]),this.funcName=x("div"),this.root=x("div",["ig-bb","ig-flex","ig-bg-white"],()=>{},[x("div",["ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center","ig-br","ig-hide-if-empty"],()=>{},[t.browse&&x("div",[],()=>{},[x("input",[],e=>{e.type="file",e.addEventListener("change",s=>{let i=s.target;i.files?.length&&this.fileSelected(i.files[0])})})]),this.funcSelector,this.funcSelectorNone]),x("div",["ig-flex-grow-1","ig-pv2","ig-ph3","ig-flex","ig-g2","ig-items-center"],()=>{},[this.funcName,x("div",["ig-flex-grow-1"]),t.export&&x("div",[],()=>{},[x("button",[],e=>{this.exportButton=e,e.addEventListener("click",()=>{this.exportStandalone()})},["Export"])])])]),this.update()}async fileSelected(t){let e=JSON.parse(await t.text());this.ionjson=X(e),this.switchFunc(0),this.update()}switchIonJSON(t){this.ionjson=t,this.switchFunc(this.funcIndex)}switchFunc(t){t=Math.max(0,Math.min(this.numFunctions()-1,t)),this.funcIndex=isNaN(t)?0:t,this.funcSelected(this.ionjson?.functions[this.funcIndex]??null),this.update()}numFunctions(){return this.ionjson?.functions.length??0}update(){let t=0<=this.funcIndex&&this.funcIndex<this.numFunctions();this.funcSelector.hidden=this.numFunctions()<=1,this.funcSelectorNone.hidden=!(this.ionjson&&this.numFunctions()===0);let e=this.funcSelector.querySelector("input");e.max=`${this.numFunctions()}`,e.value=`${this.funcIndex+1}`,this.funcSelector.querySelector(".num-functions").innerHTML=`${this.numFunctions()}`,this.funcName.hidden=!t,this.funcName.innerText=`${this.ionjson?.functions[this.funcIndex].name??""}`,this.exportButton&&(this.exportButton.disabled=!this.ionjson||!t)}async exportStandalone(){let t=L(this.ionjson),e=t.functions[this.funcIndex].name,s={version:1,functions:[t.functions[this.funcIndex]]},c=(await(await fetch("./standalone.html")).text()).replace(/\{\{\s*IONJSON\s*\}\}/,JSON.stringify(s)),l=URL.createObjectURL(new Blob([c],{type:"text/html;charset=utf-8"})),o=document.createElement("a");o.href=l,o.download=`iongraph-${e}.html`,document.body.appendChild(o),o.click(),o.remove(),URL.revokeObjectURL(l)}},tt=class{constructor(){this.menuBar=new Y({browse:!0,export:!0,funcSelected:t=>this.switchFunc(t)}),this.func=null,this.sampleCountsFromFile=void 0,this.graph=null,this.loadStuffFromQueryParams(),this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer]),this.update()}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut,sampleCounts:this.sampleCountsFromFile}))}loadStuffFromQueryParams(){(async()=>{let t=M.get("file");if(t){let s=await(await fetch(t)).json(),i=X(s);this.menuBar.switchIonJSON(i)}})(),(async()=>{let t=M.get("sampleCounts");if(t){let s=await(await fetch(t)).json();this.sampleCountsFromFile={selfLineHits:new Map(s.selfLineHits),totalLineHits:new Map(s.totalLineHits)},this.update()}})()}switchFunc(t){this.func=t,this.update()}},et=class{constructor(){this.menuBar=new Y({funcSelected:t=>this.switchFunc(t)}),this.func=null,this.graph=null,this.graphContainer=x("div",["ig-relative","ig-flex-basis-0","ig-flex-grow-1","ig-overflow-hidden"]),this.root=x("div",["ig-absolute","ig-absolute-fill","ig-flex","ig-flex-column"],()=>{},[this.menuBar.root,this.graphContainer])}update(){this.graph&&this.graph.destroy(),this.func&&(this.graph=new A(this.graphContainer,{func:this.func,pass:ut}))}setIonJSON(t){this.menuBar.switchIonJSON(t)}switchFunc(t){this.func=t,this.update()}};return yt(Yt);})(); +</script> + <script>window.__exportedIonJSON = {{ IONJSON }}</script> + <script> + const ui = new iongraph.StandaloneUI(); + document.body.appendChild(ui.root); + ui.setIonJSON(window.__exportedIonJSON); + </script> +</body> diff --git a/tool/zjit_iongraph.rb b/tool/zjit_iongraph.rb new file mode 100755 index 0000000000..0cb7701614 --- /dev/null +++ b/tool/zjit_iongraph.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +require 'json' +require 'logger' + +LOGGER = Logger.new($stderr) + +def run_ruby *cmd + # Find the first --zjit* option and add --zjit-dump-hir-iongraph after it + zjit_index = cmd.find_index { |arg| arg.start_with?("--zjit") } + raise "No --zjit option found in command" unless zjit_index + cmd.insert(zjit_index + 1, "--zjit-dump-hir-iongraph") + pid = Process.spawn(*cmd) + _, status = Process.wait2(pid) + if status.exitstatus != 0 + LOGGER.warn("Command failed with exit status #{status.exitstatus}") + end + pid +end + +usage = "Usage: zjit_iongraph.rb <path_to_ruby> <options>" +RUBY = ARGV[0] || raise(usage) +OPTIONS = ARGV[1..] +pid = run_ruby(RUBY, *OPTIONS) +functions = Dir["/tmp/zjit-iongraph-#{pid}/fun*.json"].map do |path| + JSON.parse(File.read(path)) +end + +if functions.empty? + LOGGER.warn("No iongraph functions found for PID #{pid}") +end + +json = JSON.dump({version: 1, functions: functions}) +# Get zjit_iongraph.html from the sibling file next to this script +html = File.read(File.join(File.dirname(__FILE__), "zjit_iongraph.html")) +html.sub!("{{ IONJSON }}", json) +output_path = "zjit_iongraph_#{pid}.html" +File.write(output_path, html) +puts "Wrote iongraph to #{output_path}" |